You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2017/07/14 10:23:18 UTC
[01/11] ignite git commit: Corrected fix for REST processor wrt
authentication
Repository: ignite
Updated Branches:
refs/heads/master 7338445ac -> abe8e67e6
Corrected fix for REST processor wrt authentication
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/707c454a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/707c454a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/707c454a
Branch: refs/heads/master
Commit: 707c454ad9c3b4132e2d0a20d15dc1eb2ed295b0
Parents: 478d3b5
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Wed Jul 12 10:53:46 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Jul 12 10:53:46 2017 +0300
----------------------------------------------------------------------
.../processors/rest/GridRestProcessor.java | 45 ++++++--------------
1 file changed, 12 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/707c454a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
index fd5583d..9842883 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
@@ -243,23 +243,21 @@ public class GridRestProcessor extends GridProcessorAdapter {
SecurityContext secCtx0 = ses.secCtx;
- if (ctx.state().publicApiActiveState() || !isClusterActivateTaskRequest(req)) {
- try {
- if (secCtx0 == null)
- ses.secCtx = secCtx0 = authenticate(req);
+ try {
+ if (secCtx0 == null)
+ ses.secCtx = secCtx0 = authenticate(req);
- authorize(req, secCtx0);
- }
- catch (SecurityException e) {
- assert secCtx0 != null;
+ authorize(req, secCtx0);
+ }
+ catch (SecurityException e) {
+ assert secCtx0 != null;
- GridRestResponse res = new GridRestResponse(STATUS_SECURITY_CHECK_FAILED, e.getMessage());
+ GridRestResponse res = new GridRestResponse(STATUS_SECURITY_CHECK_FAILED, e.getMessage());
- return new GridFinishedFuture<>(res);
- }
- catch (IgniteCheckedException e) {
- return new GridFinishedFuture<>(new GridRestResponse(STATUS_AUTH_FAILED, e.getMessage()));
- }
+ return new GridFinishedFuture<>(res);
+ }
+ catch (IgniteCheckedException e) {
+ return new GridFinishedFuture<>(new GridRestResponse(STATUS_AUTH_FAILED, e.getMessage()));
}
}
@@ -321,25 +319,6 @@ public class GridRestProcessor extends GridProcessorAdapter {
}
/**
- * We skip authentication for activate cluster request.
- * It's necessary workaround to make possible cluster activation through Visor,
- * as security checks require working caches.
- *
- * @param req Request.
- */
- private boolean isClusterActivateTaskRequest(GridRestRequest req) {
- if (req instanceof GridRestTaskRequest) {
- GridRestTaskRequest taskReq = (GridRestTaskRequest)req;
-
- if (VisorGatewayTask.class.getCanonicalName().equals(taskReq.taskName()) &&
- taskReq.params().contains(VisorChangeGridActiveStateTask.class.getCanonicalName()))
- return true;
- }
-
- return false;
- }
-
- /**
* @param req Request.
* @return Not null session.
* @throws IgniteCheckedException If failed.
[09/11] ignite git commit: IGNITE-5479 - Cleanup public API for
PersistentStoreConfiguration
Posted by vo...@apache.org.
IGNITE-5479 - Cleanup public API for PersistentStoreConfiguration
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/689b1b6e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/689b1b6e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/689b1b6e
Branch: refs/heads/master
Commit: 689b1b6e9c3e723cf394c7ff2427097b21d96ce3
Parents: 21964fb
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Thu Jul 13 10:12:01 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Jul 13 10:12:01 2017 +0300
----------------------------------------------------------------------
.../configuration/MemoryConfiguration.java | 4 +-
.../MemoryPolicyConfiguration.java | 4 +-
.../PersistentStoreConfiguration.java | 61 ++++++++++----------
.../GridCacheDatabaseSharedManager.java | 23 +++++---
.../IgniteCacheDatabaseSharedManager.java | 21 ++++---
.../wal/FileWriteAheadLogManager.java | 6 +-
.../utils/PlatformConfigurationUtils.java | 4 +-
.../node/VisorPersistentStoreConfiguration.java | 36 ++++++------
.../persistence/db/wal/IgnitePdsWalTlbTest.java | 2 +-
.../Cache/PersistentStoreTest.cs | 8 ++-
.../Config/spring-test.xml | 4 ++
.../IgniteConfigurationTest.cs | 8 ++-
.../PersistentStoreConfiguration.cs | 31 ++++++----
modules/web-console/backend/app/mongo.js | 2 +-
.../generator/ConfigurationGenerator.js | 8 +--
.../generator/defaults/Cluster.service.js | 2 +-
.../configuration/clusters/persistence.pug | 2 +-
17 files changed, 128 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java
index 5cf6cb7..f83d7d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryConfiguration.java
@@ -64,7 +64,7 @@ public class MemoryConfiguration implements Serializable {
/** Default memory policy start size (256 MB). */
@SuppressWarnings("UnnecessaryBoxing")
- public static final Long DFLT_MEMORY_POLICY_INITIAL_SIZE = new Long(256L * 1024 * 1024);
+ public static final long DFLT_MEMORY_POLICY_INITIAL_SIZE = 256L * 1024 * 1024;
/** Fraction of available memory to allocate for default MemoryPolicy. */
private static final double DFLT_MEMORY_POLICY_FRACTION = 0.8;
@@ -222,6 +222,8 @@ public class MemoryConfiguration implements Serializable {
if (maxSize < DFLT_MEMORY_POLICY_INITIAL_SIZE)
memPlc.setInitialSize(maxSize);
+ else
+ memPlc.setInitialSize(DFLT_MEMORY_POLICY_INITIAL_SIZE);
memPlc.setMaxSize(maxSize);
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/core/src/main/java/org/apache/ignite/configuration/MemoryPolicyConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryPolicyConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryPolicyConfiguration.java
index b496901..dff8b2b 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/MemoryPolicyConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/MemoryPolicyConfiguration.java
@@ -78,7 +78,7 @@ public final class MemoryPolicyConfiguration implements Serializable {
private String name = DFLT_MEM_PLC_DEFAULT_NAME;
/** Memory policy start size. */
- private Long initialSize = MemoryConfiguration.DFLT_MEMORY_POLICY_INITIAL_SIZE;
+ private long initialSize;
/** Memory policy maximum size. */
private long maxSize = MemoryConfiguration.DFLT_MEMORY_POLICY_MAX_SIZE;
@@ -172,7 +172,7 @@ public final class MemoryPolicyConfiguration implements Serializable {
*
* @return Memory policy start size.
*/
- public Long getInitialSize() {
+ public long getInitialSize() {
return initialSize;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
index 4792483..e8a0ff4 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
@@ -44,10 +44,6 @@ public class PersistentStoreConfiguration implements Serializable {
/** Default length of interval over which rate-based metric is calculated. */
public static final int DFLT_RATE_TIME_INTERVAL_MILLIS = 60_000;
- /** */
- @SuppressWarnings("UnnecessaryBoxing")
- public static final Long DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE = new Long(256L * 1024 * 1024);
-
/** Default number of checkpointing threads. */
public static final int DFLT_CHECKPOINTING_THREADS = 1;
@@ -70,7 +66,7 @@ public class PersistentStoreConfiguration implements Serializable {
public static final int DFLT_WAL_FLUSH_FREQ = 2000;
/** Default wal fsync delay. */
- public static final int DFLT_WAL_FSYNC_DELAY = 1;
+ public static final int DFLT_WAL_FSYNC_DELAY = 1000;
/** Default wal record iterator buffer size. */
public static final int DFLT_WAL_RECORD_ITERATOR_BUFFER_SIZE = 64 * 1024 * 1024;
@@ -90,11 +86,11 @@ public class PersistentStoreConfiguration implements Serializable {
/** Checkpointing frequency. */
private long checkpointingFreq = DFLT_CHECKPOINTING_FREQ;
- /** Lock wait time. */
- private int lockWaitTime = DFLT_LOCK_WAIT_TIME;
+ /** Lock wait time, in milliseconds. */
+ private long lockWaitTime = DFLT_LOCK_WAIT_TIME;
/** */
- private Long checkpointingPageBufSize = DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE;
+ private long checkpointingPageBufSize;
/** */
private int checkpointingThreads = DFLT_CHECKPOINTING_THREADS;
@@ -124,10 +120,10 @@ public class PersistentStoreConfiguration implements Serializable {
private int tlbSize = DFLT_TLB_SIZE;
/** Wal flush frequency in milliseconds. */
- private int walFlushFreq = DFLT_WAL_FLUSH_FREQ;
+ private long walFlushFreq = DFLT_WAL_FLUSH_FREQ;
/** Wal fsync delay. */
- private int walFsyncDelay = DFLT_WAL_FSYNC_DELAY;
+ private long walFsyncDelay = DFLT_WAL_FSYNC_DELAY;
/** Wal record iterator buffer size. */
private int walRecordIterBuffSize = DFLT_WAL_RECORD_ITERATOR_BUFFER_SIZE;
@@ -200,9 +196,10 @@ public class PersistentStoreConfiguration implements Serializable {
/**
* Gets amount of memory allocated for a checkpointing temporary buffer.
*
- * @return checkpointing page buffer size in bytes.
+ * @return Checkpointing page buffer size in bytes or {@code 0} for Ignite
+ * to choose the buffer size automatically.
*/
- public Long getCheckpointingPageBufferSize() {
+ public long getCheckpointingPageBufferSize() {
return checkpointingPageBufSize;
}
@@ -211,7 +208,8 @@ public class PersistentStoreConfiguration implements Serializable {
* copies of pages that are being written to disk and being update in parallel while the checkpointing is in
* progress.
*
- * @param checkpointingPageBufSize checkpointing page buffer size in bytes.
+ * @param checkpointingPageBufSize Checkpointing page buffer size in bytes or {@code 0} for Ignite to
+ * choose the buffer size automatically.
* @return {@code this} for chaining.
*/
public PersistentStoreConfiguration setCheckpointingPageBufferSize(long checkpointingPageBufSize) {
@@ -243,21 +241,23 @@ public class PersistentStoreConfiguration implements Serializable {
}
/**
- * Time out in second, while wait and try get file lock for start persist manager.
+ * Time out in milliseonds to wait when acquiring persistence store lock file before failing the
+ * local node.
*
- * @return Time for wait.
+ * @return Lock wait time in milliseconds.
*/
- public int getLockWaitTime() {
+ public long getLockWaitTime() {
return lockWaitTime;
}
/**
- * Time out in milliseconds, while wait and try get file lock for start persist manager.
+ * Time out in milliseconds to wait when acquiring persistence store lock file before failing the
+ * local node.
*
- * @param lockWaitTime Lock wait time.
+ * @param lockWaitTime Lock wait time in milliseconds.
* @return {@code this} for chaining.
*/
- public PersistentStoreConfiguration setLockWaitTime(int lockWaitTime) {
+ public PersistentStoreConfiguration setLockWaitTime(long lockWaitTime) {
this.lockWaitTime = lockWaitTime;
return this;
@@ -474,19 +474,22 @@ public class PersistentStoreConfiguration implements Serializable {
}
/**
- * Property define how often will be fsync, in milliseconds.
- * In background mode, exist thread which do fsync by timeout.
+ * This property define how often WAL will be fsync-ed in {@code BACKGROUND} mode. Ignored for
+ * all other WAL modes.
*
- * @return Flush frequency.
+ * @return WAL flush frequency, in milliseconds.
*/
- public int getWalFlushFrequency() {
+ public long getWalFlushFrequency() {
return walFlushFreq;
}
/**
- * @param walFlushFreq Wal flush frequency, in milliseconds.
+ * This property define how often WAL will be fsync-ed in {@code BACKGROUND} mode. Ignored for
+ * all other WAL modes.
+ *
+ * @param walFlushFreq WAL flush frequency, in milliseconds.
*/
- public PersistentStoreConfiguration setWalFlushFrequency(int walFlushFreq) {
+ public PersistentStoreConfiguration setWalFlushFrequency(long walFlushFreq) {
this.walFlushFreq = walFlushFreq;
return this;
@@ -495,15 +498,15 @@ public class PersistentStoreConfiguration implements Serializable {
/**
* Gets the fsync delay, in nanoseconds.
*/
- public int getWalFsyncDelay() {
+ public long getWalFsyncDelayNanos() {
return walFsyncDelay <= 0 ? DFLT_WAL_FSYNC_DELAY : walFsyncDelay;
}
/**
- * @param walFsyncDelay Wal fsync delay, in nanoseconds.
+ * @param walFsyncDelayNanos Wal fsync delay, in nanoseconds.
*/
- public PersistentStoreConfiguration setWalFsyncDelay(int walFsyncDelay) {
- this.walFsyncDelay = walFsyncDelay;
+ public PersistentStoreConfiguration setWalFsyncDelayNanos(long walFsyncDelayNanos) {
+ walFsyncDelay = walFsyncDelayNanos;
return this;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 9f2067a..d147f36 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -151,6 +151,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** */
public static final String IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC = "IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC";
+ /** Default checkpointing page buffer size (may be adjusted by Ignite). */
+ public static final Long DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE = 256L * 1024 * 1024;
+
/** Skip sync. */
private final boolean skipSync = IgniteSystemProperties.getBoolean(IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC);
@@ -277,7 +280,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
private FileLockHolder fileLockHolder;
/** Lock wait time. */
- private final int lockWaitTime;
+ private final long lockWaitTime;
/** */
private Map<Integer, Map<Integer, T2<Long, WALPointer>>> reservedForExchange;
@@ -375,10 +378,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
- * @throws IgniteCheckedException If failed.
+ *
*/
- private void initDataBase() throws IgniteCheckedException {
- Long cpBufSize = persistenceCfg.getCheckpointingPageBufferSize();
+ private void initDataBase() {
+ long cpBufSize = persistenceCfg.getCheckpointingPageBufferSize();
if (persistenceCfg.getCheckpointingThreads() > 1)
asyncRunner = new ThreadPoolExecutor(
@@ -391,7 +394,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
// Intentionally use identity comparison to check if configuration default has changed.
//noinspection NumberEquality
- if (cpBufSize == PersistentStoreConfiguration.DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE) {
+ if (cpBufSize == 0L) {
+ cpBufSize = DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE;
+
MemoryConfiguration memCfg = cctx.kernalContext().config().getMemoryConfiguration();
assert memCfg != null;
@@ -2928,10 +2933,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
- * @param lockWaitTime During which time thread will try capture file lock.
+ * @param lockWaitTimeMillis During which time thread will try capture file lock.
* @throws IgniteCheckedException If failed to capture file lock.
*/
- public void tryLock(int lockWaitTime) throws IgniteCheckedException {
+ public void tryLock(long lockWaitTimeMillis) throws IgniteCheckedException {
assert lockFile != null;
FileChannel ch = lockFile.getChannel();
@@ -2965,7 +2970,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
String content = null;
// Try to get lock, if not available wait 1 sec and re-try.
- for (int i = 0; i < lockWaitTime; i += 1000) {
+ for (int i = 0; i < lockWaitTimeMillis; i += 1000) {
try {
lock = ch.tryLock(0, 1, false);
if (lock != null && lock.isValid()) {
@@ -2989,7 +2994,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
if (content == null)
content = readContent();
- failMsg = "Failed to acquire file lock during " + (lockWaitTime / 1000) +
+ failMsg = "Failed to acquire file lock during " + (lockWaitTimeMillis / 1000) +
" sec, (locked by " + content + "): " + file.getAbsolutePath();
}
catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index c5f174c..c503fb4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -295,15 +295,6 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
}
/**
- * @param dbCfg Database configuration.
- * @param memPlcCfg MemoryPolicy configuration.
- * @param memMetrics MemoryMetrics instance.
- */
- private MemoryPolicy createDefaultMemoryPolicy(MemoryConfiguration dbCfg, MemoryPolicyConfiguration memPlcCfg, MemoryMetricsImpl memMetrics) {
- return initMemory(dbCfg, memPlcCfg, memMetrics);
- }
-
- /**
* @param sysCacheInitSize Initial size of PageMemory to be created for system cache.
* @param sysCacheMaxSize Maximum size of PageMemory to be created for system cache.
*
@@ -454,6 +445,14 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
* @throws IgniteCheckedException If config is invalid.
*/
private void checkPolicySize(MemoryPolicyConfiguration plcCfg) throws IgniteCheckedException {
+ boolean dfltInitSize = false;
+
+ if (plcCfg.getInitialSize() == 0) {
+ plcCfg.setInitialSize(DFLT_MEMORY_POLICY_INITIAL_SIZE);
+
+ dfltInitSize = true;
+ }
+
if (plcCfg.getInitialSize() < MIN_PAGE_MEMORY_SIZE)
throw new IgniteCheckedException("MemoryPolicy must have size more than 10MB (use " +
"MemoryPolicyConfiguration.initialSize property to set correct size in bytes) " +
@@ -461,8 +460,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
);
if (plcCfg.getMaxSize() < plcCfg.getInitialSize()) {
- // We will know for sure if initialSize has been changed if we compare Longs by "==".
- if (plcCfg.getInitialSize() == DFLT_MEMORY_POLICY_INITIAL_SIZE) {
+ // If initial size was not set, use the max size.
+ if (dfltInitSize) {
plcCfg.setInitialSize(plcCfg.getMaxSize());
LT.warn(log, "MemoryPolicy maxSize=" + U.readableSize(plcCfg.getMaxSize(), true) +
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 5c112fb..897f903 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -124,7 +124,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
private final int tlbSize;
/** WAL flush frequency. Makes sense only for {@link WALMode#BACKGROUND} log WALMode. */
- private final int flushFreq;
+ private final long flushFreq;
/** Fsync delay. */
private final long fsyncDelay;
@@ -229,7 +229,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
mode = psCfg.getWalMode();
tlbSize = psCfg.getTlbSize();
flushFreq = psCfg.getWalFlushFrequency();
- fsyncDelay = psCfg.getWalFsyncDelay();
+ fsyncDelay = psCfg.getWalFsyncDelayNanos();
alwaysWriteFullPages = psCfg.isAlwaysWriteFullPages();
ioFactory = psCfg.getFileIOFactory();
walAutoArchiveAfterInactivity = psCfg.getWalAutoArchiveAfterInactivity();
@@ -1666,7 +1666,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** Condition activated each time writeBuffer() completes. Used to wait previously flushed write to complete */
private final Condition writeComplete = lock.newCondition();
- /** Condition for timed wait of several threads, see {@link PersistentStoreConfiguration#getWalFsyncDelay()} */
+ /** Condition for timed wait of several threads, see {@link PersistentStoreConfiguration#getWalFsyncDelayNanos()} */
private final Condition fsync = lock.newCondition();
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
index 03b30db..8c9f5e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java
@@ -1499,7 +1499,7 @@ public class PlatformConfigurationUtils {
.setWalMode(WALMode.fromOrdinal(in.readInt()))
.setTlbSize(in.readInt())
.setWalFlushFrequency((int) in.readLong())
- .setWalFsyncDelay(in.readInt())
+ .setWalFsyncDelayNanos(in.readLong())
.setWalRecordIteratorBufferSize(in.readInt())
.setAlwaysWriteFullPages(in.readBoolean())
.setMetricsEnabled(in.readBoolean())
@@ -1531,7 +1531,7 @@ public class PlatformConfigurationUtils {
w.writeInt(cfg.getWalMode().ordinal());
w.writeInt(cfg.getTlbSize());
w.writeLong(cfg.getWalFlushFrequency());
- w.writeInt(cfg.getWalFsyncDelay());
+ w.writeLong(cfg.getWalFsyncDelayNanos());
w.writeInt(cfg.getWalRecordIteratorBufferSize());
w.writeBoolean(cfg.isAlwaysWriteFullPages());
w.writeBoolean(cfg.isMetricsEnabled());
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistentStoreConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistentStoreConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistentStoreConfiguration.java
index 3559845..128f43a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistentStoreConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorPersistentStoreConfiguration.java
@@ -39,10 +39,10 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject {
private long checkpointingFreq;
/** Lock wait time. */
- private int lockWaitTime;
+ private long lockWaitTime;
/** */
- private Long checkpointingPageBufSize;
+ private long checkpointingPageBufSize;
/** */
private int checkpointingThreads;
@@ -72,10 +72,10 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject {
private int tlbSize;
/** Wal flush frequency. */
- private int walFlushFreq;
+ private long walFlushFreq;
- /** Wal fsync delay. */
- private int walFsyncDelay;
+ /** Wal fsync delay in nanoseconds. */
+ private long walFsyncDelay;
/** Wal record iterator buffer size. */
private int walRecordIterBuffSize;
@@ -114,7 +114,7 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject {
walMode = cfg.getWalMode();
tlbSize = cfg.getTlbSize();
walFlushFreq = cfg.getWalFlushFrequency();
- walFsyncDelay = cfg.getWalFsyncDelay();
+ walFsyncDelay = cfg.getWalFsyncDelayNanos();
walRecordIterBuffSize = cfg.getWalRecordIteratorBufferSize();
alwaysWriteFullPages = cfg.isAlwaysWriteFullPages();
subIntervals = cfg.getSubIntervals();
@@ -138,7 +138,7 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject {
/**
* @return Checkpointing page buffer size in bytes.
*/
- public Long getCheckpointingPageBufferSize() {
+ public long getCheckpointingPageBufferSize() {
return checkpointingPageBufSize;
}
@@ -152,7 +152,7 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject {
/**
* @return Time for wait.
*/
- public int getLockWaitTime() {
+ public long getLockWaitTime() {
return lockWaitTime;
}
@@ -229,14 +229,14 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject {
/**
* @return Flush frequency.
*/
- public int getWalFlushFrequency() {
+ public long getWalFlushFrequency() {
return walFlushFreq;
}
/**
* Gets the fsync delay, in nanoseconds.
*/
- public int getWalFsyncDelay() {
+ public long getWalFsyncDelayNanos() {
return walFsyncDelay;
}
@@ -258,8 +258,8 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject {
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
U.writeString(out, persistenceStorePath);
out.writeLong(checkpointingFreq);
- out.writeInt(lockWaitTime);
- out.writeObject(checkpointingPageBufSize);
+ out.writeLong(lockWaitTime);
+ out.writeLong(checkpointingPageBufSize);
out.writeInt(checkpointingThreads);
out.writeInt(walHistSize);
out.writeInt(walSegments);
@@ -269,8 +269,8 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject {
out.writeBoolean(metricsEnabled);
U.writeEnum(out, walMode);
out.writeInt(tlbSize);
- out.writeInt(walFlushFreq);
- out.writeInt(walFsyncDelay);
+ out.writeLong(walFlushFreq);
+ out.writeLong(walFsyncDelay);
out.writeInt(walRecordIterBuffSize);
out.writeBoolean(alwaysWriteFullPages);
out.writeInt(subIntervals);
@@ -281,8 +281,8 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject {
@Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
persistenceStorePath = U.readString(in);
checkpointingFreq = in.readLong();
- lockWaitTime = in.readInt();
- checkpointingPageBufSize = (Long)in.readObject();
+ lockWaitTime = in.readLong();
+ checkpointingPageBufSize = in.readLong();
checkpointingThreads = in.readInt();
walHistSize = in.readInt();
walSegments = in.readInt();
@@ -292,8 +292,8 @@ public class VisorPersistentStoreConfiguration extends VisorDataTransferObject {
metricsEnabled = in.readBoolean();
walMode = WALMode.fromOrdinal(in.readByte());
tlbSize = in.readInt();
- walFlushFreq = in.readInt();
- walFsyncDelay = in.readInt();
+ walFlushFreq = in.readLong();
+ walFsyncDelay = in.readLong();
walRecordIterBuffSize = in.readInt();
alwaysWriteFullPages = in.readBoolean();
subIntervals = in.readInt();
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java
index a1a7286..fd0fd34 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java
@@ -31,7 +31,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import static org.apache.ignite.configuration.PersistentStoreConfiguration.DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE;
+import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs
index 3320dd7..d321639 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/PersistentStoreTest.cs
@@ -59,7 +59,8 @@ namespace Apache.Ignite.Core.Tests.Cache
PersistentStorePath = Path.Combine(_tempDir, "Store"),
WalStorePath = Path.Combine(_tempDir, "WalStore"),
WalArchivePath = Path.Combine(_tempDir, "WalArchive"),
- MetricsEnabled = true
+ MetricsEnabled = true,
+ CheckpointingPageBufferSize = 1024 * 1024 // TODO: Use default (IGNITE-5717)
}
};
@@ -116,7 +117,10 @@ namespace Apache.Ignite.Core.Tests.Cache
{
var cfg = new IgniteConfiguration(GetTestConfiguration())
{
- PersistentStoreConfiguration = new PersistentStoreConfiguration()
+ PersistentStoreConfiguration = new PersistentStoreConfiguration
+ {
+ CheckpointingPageBufferSize = 1024 * 1024 // TODO: Use default (IGNITE-5717)
+ }
};
using (var ignite = Ignition.Start(cfg))
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/spring-test.xml
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/spring-test.xml b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/spring-test.xml
index dd0669a..31fa3b3 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/spring-test.xml
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Config/spring-test.xml
@@ -56,5 +56,9 @@
</property>
</bean>
</property>
+
+ <property name="persistentStoreConfiguration">
+ <bean class="org.apache.ignite.configuration.PersistentStoreConfiguration"/>
+ </property>
</bean>
</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs
index 81fd226..93d6af3 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs
@@ -316,6 +316,9 @@ namespace Apache.Ignite.Core.Tests
Assert.AreEqual(MemoryPolicyConfiguration.DefaultMaxSize, plc.MaxSize);
Assert.AreEqual(MemoryPolicyConfiguration.DefaultSubIntervals, plc.SubIntervals);
Assert.AreEqual(MemoryPolicyConfiguration.DefaultRateTimeInterval, plc.RateTimeInterval);
+
+ // Check PersistentStoreConfiguration defaults.
+ CheckDefaultProperties(resCfg.PersistentStoreConfiguration);
}
}
@@ -532,8 +535,7 @@ namespace Apache.Ignite.Core.Tests
Assert.AreEqual(PersistentStoreConfiguration.DefaultTlbSize, cfg.TlbSize);
Assert.AreEqual(PersistentStoreConfiguration.DefaultCheckpointingFrequency, cfg.CheckpointingFrequency);
Assert.AreEqual(PersistentStoreConfiguration.DefaultCheckpointingThreads, cfg.CheckpointingThreads);
- Assert.AreEqual(PersistentStoreConfiguration.DefaultCheckpointingPageBufferSize,
- cfg.CheckpointingPageBufferSize);
+ Assert.AreEqual(default(long), cfg.CheckpointingPageBufferSize);
Assert.AreEqual(PersistentStoreConfiguration.DefaultLockWaitTime, cfg.LockWaitTime);
Assert.AreEqual(PersistentStoreConfiguration.DefaultWalFlushFrequency, cfg.WalFlushFrequency);
Assert.AreEqual(PersistentStoreConfiguration.DefaultWalFsyncDelayNanos, cfg.WalFsyncDelayNanos);
@@ -546,6 +548,8 @@ namespace Apache.Ignite.Core.Tests
Assert.IsFalse(cfg.MetricsEnabled);
Assert.AreEqual(PersistentStoreConfiguration.DefaultSubIntervals, cfg.SubIntervals);
Assert.AreEqual(PersistentStoreConfiguration.DefaultRateTimeInterval, cfg.RateTimeInterval);
+ Assert.AreEqual(PersistentStoreConfiguration.DefaultWalStorePath, cfg.WalStorePath);
+ Assert.AreEqual(PersistentStoreConfiguration.DefaultWalArchivePath, cfg.WalArchivePath);
}
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs
index 43b17ac..cac6cc8 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/PersistentStore/PersistentStoreConfiguration.cs
@@ -30,11 +30,6 @@ namespace Apache.Ignite.Core.PersistentStore
public class PersistentStoreConfiguration
{
/// <summary>
- /// Default value for <see cref="CheckpointingPageBufferSize"/>.
- /// </summary>
- public const long DefaultCheckpointingPageBufferSize = 256L * 1024 * 1024;
-
- /// <summary>
/// Default value for <see cref="CheckpointingThreads"/>.
/// </summary>
public const int DefaultCheckpointingThreads = 1;
@@ -82,7 +77,7 @@ namespace Apache.Ignite.Core.PersistentStore
/// <summary>
/// Default value for <see cref="WalFsyncDelayNanos"/>.
/// </summary>
- public const int DefaultWalFsyncDelayNanos = 1;
+ public const long DefaultWalFsyncDelayNanos = 1000;
/// <summary>
/// The default sub intervals.
@@ -97,11 +92,20 @@ namespace Apache.Ignite.Core.PersistentStore
public static readonly TimeSpan DefaultRateTimeInterval = TimeSpan.FromSeconds(60);
/// <summary>
+ /// Default value for <see cref="WalStorePath"/>.
+ /// </summary>
+ public const string DefaultWalStorePath = "db/wal";
+
+ /// <summary>
+ /// Default value for <see cref="WalArchivePath"/>.
+ /// </summary>
+ public const string DefaultWalArchivePath = "db/wal/archive";
+
+ /// <summary>
/// Initializes a new instance of the <see cref="PersistentStoreConfiguration"/> class.
/// </summary>
public PersistentStoreConfiguration()
{
- CheckpointingPageBufferSize = DefaultCheckpointingPageBufferSize;
CheckpointingThreads = DefaultCheckpointingThreads;
CheckpointingFrequency = DefaultCheckpointingFrequency;
LockWaitTime = DefaultLockWaitTime;
@@ -114,6 +118,8 @@ namespace Apache.Ignite.Core.PersistentStore
WalFsyncDelayNanos = DefaultWalFsyncDelayNanos;
RateTimeInterval = DefaultRateTimeInterval;
SubIntervals = DefaultSubIntervals;
+ WalArchivePath = DefaultWalArchivePath;
+ WalStorePath = DefaultWalStorePath;
}
/// <summary>
@@ -137,7 +143,7 @@ namespace Apache.Ignite.Core.PersistentStore
WalMode = (WalMode)reader.ReadInt();
TlbSize = reader.ReadInt();
WalFlushFrequency = reader.ReadLongAsTimespan();
- WalFsyncDelayNanos = reader.ReadInt();
+ WalFsyncDelayNanos = reader.ReadLong();
WalRecordIteratorBufferSize = reader.ReadInt();
AlwaysWriteFullPages = reader.ReadBoolean();
MetricsEnabled = reader.ReadBoolean();
@@ -166,7 +172,7 @@ namespace Apache.Ignite.Core.PersistentStore
writer.WriteInt((int)WalMode);
writer.WriteInt(TlbSize);
writer.WriteTimeSpanAsLong(WalFlushFrequency);
- writer.WriteInt(WalFsyncDelayNanos);
+ writer.WriteLong(WalFsyncDelayNanos);
writer.WriteInt(WalRecordIteratorBufferSize);
writer.WriteBoolean(AlwaysWriteFullPages);
writer.WriteBoolean(MetricsEnabled);
@@ -188,8 +194,9 @@ namespace Apache.Ignite.Core.PersistentStore
/// <summary>
/// Gets or sets the size of the checkpointing page buffer.
+ /// <para />
+ /// Default is <c>0</c>: Ignite will choose buffer size automatically.
/// </summary>
- [DefaultValue(DefaultCheckpointingPageBufferSize)]
public long CheckpointingPageBufferSize { get; set; }
/// <summary>
@@ -227,12 +234,14 @@ namespace Apache.Ignite.Core.PersistentStore
/// <summary>
/// Gets or sets the path to the directory where WAL (Write Ahead Log) is stored.
/// </summary>
+ [DefaultValue(DefaultWalStorePath)]
public string WalStorePath { get; set; }
/// <summary>
/// Gets or sets the path to the directory where WAL (Write Ahead Log) archive is stored.
/// Every WAL segment will be fully copied to this directory before it can be reused for WAL purposes.
/// </summary>
+ [DefaultValue(DefaultWalArchivePath)]
public string WalArchivePath { get; set; }
/// <summary>
@@ -256,7 +265,7 @@ namespace Apache.Ignite.Core.PersistentStore
/// Gets or sets the WAL (Write Ahead Log) fsync (disk sync) delay, in nanoseconds
/// </summary>
[DefaultValue(DefaultWalFsyncDelayNanos)]
- public int WalFsyncDelayNanos { get; set; }
+ public long WalFsyncDelayNanos { get; set; }
/// <summary>
/// Gets or sets the size of the WAL (Write Ahead Log) record iterator buffer, in bytes.
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/web-console/backend/app/mongo.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index aa11e0b..57af928 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -1006,7 +1006,7 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
walSegmentSize: Number,
walHistorySize: Number,
walFlushFrequency: Number,
- walFsyncDelay: Number,
+ walFsyncDelayNanos: Number,
walRecordIteratorBufferSize: Number,
lockWaitTime: Number,
rateTimeInterval: Number,
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 8c9b14c..8299b9b 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -1477,17 +1477,17 @@ export default class IgniteConfigurationGenerator {
.boolProperty('metricsEnabled')
.boolProperty('alwaysWriteFullPages')
.intProperty('checkpointingFrequency')
- .intProperty('checkpointingPageBufferSize')
+ .longProperty('checkpointingPageBufferSize')
.intProperty('checkpointingThreads')
.stringProperty('walStorePath')
.stringProperty('walArchivePath')
.intProperty('walSegments')
.intProperty('walSegmentSize')
.intProperty('walHistorySize')
- .intProperty('walFlushFrequency')
- .intProperty('walFsyncDelay')
+ .longProperty('walFlushFrequency')
+ .longProperty('walFsyncDelayNanos')
.intProperty('walRecordIteratorBufferSize')
- .intProperty('lockWaitTime')
+ .longProperty('lockWaitTime')
.intProperty('rateTimeInterval')
.intProperty('tlbSize')
.intProperty('subIntervals');
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
index a4680bf..1c7fc44 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cluster.service.js
@@ -340,7 +340,7 @@ const DFLT_CLUSTER = {
walSegmentSize: 67108864,
walHistorySize: 20,
walFlushFrequency: 2000,
- walFsyncDelay: 1,
+ walFsyncDelayNanos: 1000,
walRecordIteratorBufferSize: 67108864,
lockWaitTime: 10000,
rateTimeInterval: 60000,
http://git-wip-us.apache.org/repos/asf/ignite/blob/689b1b6e/modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug
index 7fd0966..50a569b 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug
@@ -62,7 +62,7 @@ include /app/helpers/jade/mixins
+number('WAL flush frequency:', `${model}.walFlushFrequency`, '"PersistenceWalFlushFrequency"', enabled, '2000', '1',
'How often will be fsync, in milliseconds. In background mode, exist thread which do fsync by timeout')
.settings-row
- +number('WAL fsync delay:', `${model}.walFsyncDelay`, '"PersistenceWalFsyncDelay"', enabled, '1', '1', 'WAL fsync delay, in nanoseconds')
+ +number('WAL fsync delay:', `${model}.walFsyncDelayNanos`, '"PersistenceWalFsyncDelay"', enabled, '1000', '1', 'WAL fsync delay, in nanoseconds')
.settings-row
+number('WAL record iterator buffer size:', `${model}.walRecordIteratorBufferSize`, '"PersistenceWalRecordIteratorBufferSize"', enabled, '67108864', '1',
'How many bytes iterator read from disk(for one reading), during go ahead WAL')
[04/11] ignite git commit: Removed CacheBinaryAutoStoreExample and
relevant bean "h2-example-db" from example-default.xml because example
duplicated existing CacheAutoStoreExample.
Posted by vo...@apache.org.
Removed CacheBinaryAutoStoreExample and relevant bean "h2-example-db" from example-default.xml because example duplicated existing CacheAutoStoreExample.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8afdc7ba
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8afdc7ba
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8afdc7ba
Branch: refs/heads/master
Commit: 8afdc7baae73ecba67e0735baa97d03f2c4fc715
Parents: 5859b19
Author: devozerov <vo...@gridgain.com>
Authored: Wed Jul 12 13:51:43 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Jul 12 13:51:43 2017 +0300
----------------------------------------------------------------------
examples/config/example-default.xml | 7 -
.../store/auto/CacheBinaryAutoStoreExample.java | 170 -------------------
.../datagrid/store/auto/package-info.java | 22 ---
3 files changed, 199 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8afdc7ba/examples/config/example-default.xml
----------------------------------------------------------------------
diff --git a/examples/config/example-default.xml b/examples/config/example-default.xml
index 6bd6f16..e6c359d 100644
--- a/examples/config/example-default.xml
+++ b/examples/config/example-default.xml
@@ -28,13 +28,6 @@
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util
http://www.springframework.org/schema/util/spring-util.xsd">
-
- <!-- Datasource for sample in-memory H2 database. -->
- <bean id="h2-example-db" class="org.h2.jdbcx.JdbcDataSource">
- <property name="URL" value="jdbc:h2:tcp://localhost/mem:ExampleDb" />
- <property name="user" value="sa" />
- </bean>
-
<bean abstract="true" id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
<!-- Set to true to enable distributed class loading for examples, default is false. -->
<property name="peerClassLoadingEnabled" value="true"/>
http://git-wip-us.apache.org/repos/asf/ignite/blob/8afdc7ba/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/CacheBinaryAutoStoreExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/CacheBinaryAutoStoreExample.java b/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/CacheBinaryAutoStoreExample.java
deleted file mode 100644
index c80c87b..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/CacheBinaryAutoStoreExample.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.binary.datagrid.store.auto;
-
-import java.sql.Types;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStore;
-import org.apache.ignite.cache.store.jdbc.CacheJdbcPojoStoreFactory;
-import org.apache.ignite.cache.store.jdbc.JdbcType;
-import org.apache.ignite.cache.store.jdbc.JdbcTypeField;
-import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.examples.model.Person;
-import org.apache.ignite.examples.util.DbH2ServerStartup;
-import org.apache.ignite.transactions.Transaction;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
-
-/**
- * Demonstrates usage of cache with underlying persistent store configured.
- * <p>
- * This example uses {@link CacheJdbcPojoStore} as a persistent store.
- * <p>
- * To start the example, you should:
- * <ul>
- * <li>Start H2 database TCP server using {@link DbH2ServerStartup}.</li>
- * <li>Start a few nodes using {@link ExampleNodeStartup} or by starting remote nodes as specified below.</li>
- * <li>Start example using {@link CacheBinaryAutoStoreExample}.</li>
- * </ul>
- * <p>
- * Remote nodes should always be started with special configuration file which
- * contains H2 data source bean descriptor: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.
- * <p>
- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will
- * start node with {@code examples/config/example-ignite.xml} configuration.
- */
-public class CacheBinaryAutoStoreExample {
- /** Global person ID to use across entire example. */
- private static final Long id = 25121642L;
-
- /** Cache name. */
- public static final String CACHE_NAME = CacheBinaryAutoStoreExample.class.getSimpleName();
-
- /**
- * Configure cache with store.
- */
- private static CacheConfiguration<Long, Person> cacheConfiguration() {
- CacheJdbcPojoStoreFactory<Long, Person> storeFactory = new CacheJdbcPojoStoreFactory<>();
-
- storeFactory.setDataSourceBean("h2-example-db");
- storeFactory.setDialect(new H2Dialect());
-
- JdbcType jdbcType = new JdbcType();
-
- jdbcType.setCacheName(CACHE_NAME);
- jdbcType.setDatabaseSchema("PUBLIC");
- jdbcType.setDatabaseTable("PERSON");
-
- jdbcType.setKeyType("java.lang.Long");
- jdbcType.setKeyFields(new JdbcTypeField(Types.BIGINT, "ID", Long.class, "id"));
-
- jdbcType.setValueType("org.apache.ignite.examples.model.Person");
- jdbcType.setValueFields(
- new JdbcTypeField(Types.BIGINT, "ID", Long.class, "id"),
- new JdbcTypeField(Types.VARCHAR, "FIRST_NAME", String.class, "firstName"),
- new JdbcTypeField(Types.VARCHAR, "LAST_NAME", String.class, "lastName")
- );
-
- storeFactory.setTypes(jdbcType);
-
- CacheConfiguration<Long, Person> cfg = new CacheConfiguration<>(CACHE_NAME);
-
- cfg.setCacheStoreFactory(storeFactory);
-
- // Set atomicity as transaction, since we are showing transactions in the example.
- cfg.setAtomicityMode(TRANSACTIONAL);
-
- // This option will allow to start remote nodes without having user classes in classpath.
- cfg.setStoreKeepBinary(true);
-
- cfg.setReadThrough(true);
- cfg.setWriteThrough(true);
-
- return cfg;
- }
-
- /**
- * Executes example.
- *
- * @param args Command line arguments, none required.
- * @throws Exception If example execution failed.
- */
- public static void main(String[] args) throws Exception {
- // To start ignite with desired configuration uncomment the appropriate line.
- try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- System.out.println();
- System.out.println(">>> Populate database with data...");
- DbH2ServerStartup.populateDatabase();
-
- System.out.println();
- System.out.println(">>> Cache auto store example started...");
-
- try (IgniteCache<Long, Person> cache = ignite.getOrCreateCache(cacheConfiguration())) {
- try (Transaction tx = ignite.transactions().txStart()) {
- Person val = cache.get(id);
-
- System.out.println(">>> Read value: " + val);
-
- val = cache.getAndPut(id, new Person(id, 1L, "Isaac", "Newton", 100.10, "English physicist and mathematician"));
-
- System.out.println(">>> Overwrote old value: " + val);
-
- val = cache.get(id);
-
- System.out.println(">>> Read value: " + val);
-
- System.out.println(">>> Update salary in transaction...");
-
- val.salary *= 2;
-
- cache.put(id, val);
-
- tx.commit();
- }
-
- System.out.println(">>> Read value after commit: " + cache.get(id));
-
- cache.clear();
-
- System.out.println(">>> ------------------------------------------");
- System.out.println(">>> Load data to cache from DB with custom SQL...");
-
- // Load cache on all data nodes with custom SQL statement.
- cache.loadCache(null, "java.lang.Long", "select * from PERSON where id <= 3");
-
- System.out.println(">>> Loaded cache entries: " + cache.size());
-
- cache.clear();
-
- // Load cache on all data nodes with default SQL statement.
- System.out.println(">>> Load ALL data to cache from DB...");
- cache.loadCache(null);
-
- System.out.println(">>> Loaded cache entries: " + cache.size());
- }
- finally {
- // Distributed cache could be removed from cluster only by #destroyCache() call.
- ignite.destroyCache(CACHE_NAME);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8afdc7ba/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/package-info.java b/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/package-info.java
deleted file mode 100644
index 153f210..0000000
--- a/examples/src/main/java/org/apache/ignite/examples/binary/datagrid/store/auto/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains automatic JDBC store example.
- */
-package org.apache.ignite.examples.binary.datagrid.store.auto;
[05/11] ignite git commit: Fixed page acquire during checkpoint
Posted by vo...@apache.org.
Fixed page acquire during checkpoint
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c6ee085b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c6ee085b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c6ee085b
Branch: refs/heads/master
Commit: c6ee085b8a1321ce7fa15f8adf74fa7a01f7a445
Parents: 8afdc7b
Author: Dmitriy Govorukhin <dm...@gmail.com>
Authored: Wed Jul 12 14:22:03 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Jul 12 14:22:03 2017 +0300
----------------------------------------------------------------------
.../persistence/pagemem/PageMemoryImpl.java | 30 ++++++++++++++++----
1 file changed, 24 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c6ee085b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index e4428a2..47381d7 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -803,9 +803,14 @@ public class PageMemoryImpl implements PageMemoryEx {
int tag;
+ boolean tmpBuffer = false;
+
seg.readLock().lock();
try {
+ if (!isInCheckpoint(fullId))
+ return null;
+
tag = seg.partTag(fullId.groupId(), PageIdUtils.partId(fullId.pageId()));
relPtr = seg.loadedPages.get(
@@ -826,6 +831,8 @@ public class PageMemoryImpl implements PageMemoryEx {
// Pin the page until page will not be copied.
if (PageHeader.tempBufferPointer(absPtr) == INVALID_REL_PTR)
PageHeader.acquirePage(absPtr);
+ else
+ tmpBuffer = true;
}
}
finally {
@@ -869,7 +876,7 @@ public class PageMemoryImpl implements PageMemoryEx {
}
}
else
- return copyPageForCheckpoint(absPtr, fullId, tmpBuf, tracker) ? tag : null;
+ return copyPageForCheckpoint(absPtr, fullId, tmpBuf, tmpBuffer, tracker) ? tag : null;
}
/**
@@ -877,19 +884,24 @@ public class PageMemoryImpl implements PageMemoryEx {
* @param fullId Full id.
* @param tmpBuf Tmp buffer.
*/
- private boolean copyPageForCheckpoint(long absPtr, FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) {
+ private boolean copyPageForCheckpoint(
+ long absPtr,
+ FullPageId fullId,
+ ByteBuffer tmpBuf,
+ boolean tmpBuffer,
+ CheckpointMetricsTracker tracker
+ ) {
assert absPtr != 0;
+ assert PageHeader.isAcquired(absPtr);
rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
try {
long tmpRelPtr = PageHeader.tempBufferPointer(absPtr);
- if (!clearCheckpoint(fullId)){
- assert tmpRelPtr == INVALID_REL_PTR;
+ boolean success = clearCheckpoint(fullId);
- return false;
- }
+ assert success : "Page was pin when we resolve abs pointer, it can not be evicted";
if (tmpRelPtr != INVALID_REL_PTR){
PageHeader.tempBufferPointer(absPtr, INVALID_REL_PTR);
@@ -907,6 +919,12 @@ public class PageMemoryImpl implements PageMemoryEx {
// We pinned the page when allocated the temp buffer, release it now.
PageHeader.releasePage(absPtr);
+
+ // Need release again because we pin page when resolve abs pointer,
+ // and page did not have tmp buffer page.
+ if (!tmpBuffer)
+ PageHeader.releasePage(absPtr);
+
}
else {
copyInBuffer(absPtr, tmpBuf);
[06/11] ignite git commit: IGNITE-5740 - Added transaction load
timing benchmark
Posted by vo...@apache.org.
IGNITE-5740 - Added transaction load timing benchmark
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0cb6ac06
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0cb6ac06
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0cb6ac06
Branch: refs/heads/master
Commit: 0cb6ac06adddd43ac72c707b29d7216bd4cb711a
Parents: c6ee085
Author: Oleg Ostanin <oo...@gridgain.com>
Authored: Wed Jul 12 15:57:40 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Jul 12 15:57:40 2017 +0300
----------------------------------------------------------------------
.../yardstick/IgniteBenchmarkArguments.java | 33 +++++
.../cache/IgnitePutTxLoadBenchmark.java | 119 +++++++++++++++++++
2 files changed, 152 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0cb6ac06/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 5ec6c54..594fa1f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -209,6 +209,18 @@ public class IgniteBenchmarkArguments {
private int pageSize = MemoryConfiguration.DFLT_PAGE_SIZE;
/** */
+ @Parameter(names = {"-sl", "--stringLength"}, description = "Test string length")
+ private int stringLength = 500;
+
+ /** */
+ @Parameter(names = {"-wt", "--warningTime"}, description = "Warning time interval for printing log")
+ private long warningTime = 500;
+
+ /** */
+ @Parameter(names = {"-prb", "--printRollBacks"}, description = "Print rollBacks")
+ private boolean printRollBacks;
+
+ /** */
@Parameter(names = {"-prt", "--partitions"}, description = "Number of cache partitions")
private int partitions = 10;
@@ -507,6 +519,27 @@ public class IgniteBenchmarkArguments {
}
/**
+ * @return Test string length.
+ */
+ public int getStringLength() {
+ return stringLength;
+ }
+
+ /**
+ * @return Warning time interval.
+ */
+ public long getWarningTime() {
+ return warningTime;
+ }
+
+ /**
+ * @return Flag for printing rollbacks.
+ */
+ public boolean printRollBacks() {
+ return printRollBacks;
+ }
+
+ /**
* @return Number of partitioned caches.
*/
public int partitionedCachesNumber() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/0cb6ac06/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxLoadBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxLoadBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxLoadBenchmark.java
new file mode 100644
index 0000000..7ac7c3a
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgnitePutTxLoadBenchmark.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.IgniteTransactions;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionMetrics;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Ignite benchmark that performs transactional load put operations.
+ */
+public class IgnitePutTxLoadBenchmark extends IgniteCacheAbstractBenchmark<Integer, Object> {
+ /** */
+ private ArrayList<IgniteCache<Object, Object>> cacheList;
+
+ /** */
+ private String val;
+
+ /** */
+ private Random random;
+
+ /** {@inheritDoc} */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ if (!IgniteSystemProperties.getBoolean("SKIP_MAP_CHECK"))
+ ignite().compute().broadcast(new WaitMapExchangeFinishCallable());
+
+ cacheList = new ArrayList<>(args.cachesCount());
+
+ for (int i = 0; i < args.cachesCount(); i++)
+ cacheList.add(ignite().cache("tx-" + i));
+
+ val = createVal(args.getStringLength());
+
+ random = new Random();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ IgniteTransactions transactions = ignite().transactions();
+
+ long startTime;
+ long endTime;
+
+ try (Transaction tx = transactions.txStart(args.txConcurrency(), args.txIsolation())) {
+ ArrayList<Long> keyList = new ArrayList<>(args.scaleFactor());
+
+ for (int i = 0; i < args.scaleFactor(); i++)
+ keyList.add(random.nextLong());
+
+ Collections.sort(keyList);
+
+ for (int i = 0; i < args.scaleFactor(); i++){
+ IgniteCache<Object, Object> curCache = cacheList.get(random.nextInt(cacheList.size()));
+ curCache.put(keyList.get(i), val);
+ }
+
+ startTime = System.currentTimeMillis();
+
+ tx.commit();
+
+ endTime = System.currentTimeMillis();
+
+ }
+
+ TransactionMetrics tm = transactions.metrics();
+
+ if (endTime - startTime > args.getWarningTime())
+ BenchmarkUtils.println("Transaction commit time = " + (tm.commitTime() - startTime));
+
+ if (tm.txRollbacks() > 0 && args.printRollBacks())
+ BenchmarkUtils.println("Transaction rollbacks = " + tm.txRollbacks());
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteCache<Integer, Object> cache() {
+ return ignite().cache("tx");
+ }
+
+ /**
+ * Creates String val
+ * @param lgth String length
+ * @return String for inserting in cache
+ */
+ private String createVal(int lgth){
+ StringBuilder sb = new StringBuilder(lgth);
+
+ for(int i = 0; i < lgth; i++)
+ sb.append('x');
+
+ return sb.toString();
+ }
+}
[11/11] ignite git commit: Merge branch 'ignite-2.1'
Posted by vo...@apache.org.
Merge branch 'ignite-2.1'
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/abe8e67e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/abe8e67e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/abe8e67e
Branch: refs/heads/master
Commit: abe8e67e63018305cf0c2065282380b0cf9b762c
Parents: 7338445 3c1749d
Author: devozerov <vo...@gridgain.com>
Authored: Fri Jul 14 13:22:54 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Jul 14 13:22:54 2017 +0300
----------------------------------------------------------------------
RELEASE_NOTES.txt | 33 ++++
examples/config/example-default.xml | 7 -
.../store/auto/CacheBinaryAutoStoreExample.java | 170 -------------------
.../datagrid/store/auto/package-info.java | 22 ---
.../configuration/MemoryConfiguration.java | 4 +-
.../MemoryPolicyConfiguration.java | 4 +-
.../PersistentStoreConfiguration.java | 61 +++----
.../ignite/internal/pagemem/FullPageId.java | 6 +-
.../ignite/internal/pagemem/PageIdUtils.java | 14 +-
.../pagemem/store/IgnitePageStoreManager.java | 1 +
.../internal/pagemem/store/PageStore.java | 2 +
.../delta/MetaPageUpdateLastAllocatedIndex.java | 2 +-
.../processors/cache/GridCacheProcessor.java | 6 +
.../GridDhtPartitionsExchangeFuture.java | 8 +
.../cache/persistence/DbCheckpointListener.java | 7 +-
.../FullPageIdIterableComparator.java | 51 ------
.../GridCacheDatabaseSharedManager.java | 86 +++++++---
.../persistence/GridCacheOffheapManager.java | 56 +++---
.../IgniteCacheDatabaseSharedManager.java | 21 ++-
.../cache/persistence/file/FilePageStore.java | 2 +-
.../persistence/file/FilePageStoreManager.java | 1 +
.../cache/persistence/pagemem/PageMemoryEx.java | 8 +-
.../persistence/pagemem/PageMemoryImpl.java | 38 +++--
.../persistence/partstate/GroupPartitionId.java | 145 ++++++++++++++++
.../partstate/PagesAllocationRange.java | 68 ++++++++
.../partstate/PartitionAllocationMap.java | 113 ++++++++++++
.../snapshot/IgniteCacheSnapshotManager.java | 17 +-
.../cache/persistence/tree/io/PageMetaIO.java | 27 +--
.../persistence/tree/io/TrackingPageIO.java | 8 +-
.../persistence/tree/util/PageHandler.java | 3 +-
.../wal/FileWriteAheadLogManager.java | 6 +-
.../utils/PlatformConfigurationUtils.java | 4 +-
.../processors/rest/GridRestProcessor.java | 45 ++---
.../node/VisorPersistentStoreConfiguration.java | 36 ++--
.../persistence/db/wal/IgnitePdsWalTlbTest.java | 2 +-
.../Cache/PersistentStoreTest.cs | 8 +-
.../Config/spring-test.xml | 4 +
.../IgniteConfigurationTest.cs | 8 +-
.../PersistentStoreConfiguration.cs | 31 ++--
modules/web-console/backend/app/mongo.js | 2 +-
.../generator/ConfigurationGenerator.js | 8 +-
.../generator/defaults/Cluster.service.js | 2 +-
.../configuration/clusters/persistence.pug | 2 +-
.../web-console/frontend/views/sql/sql.tpl.pug | 4 +-
.../yardstick/IgniteBenchmarkArguments.java | 33 ++++
.../cache/IgnitePutTxLoadBenchmark.java | 119 +++++++++++++
46 files changed, 830 insertions(+), 475 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/abe8e67e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/abe8e67e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
[03/11] ignite git commit: AI 2.1 release notes.
Posted by vo...@apache.org.
AI 2.1 release notes.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5859b192
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5859b192
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5859b192
Branch: refs/heads/master
Commit: 5859b192ba28d53e1bccb01ce3005821e26b5347
Parents: f382826
Author: devozerov <vo...@gridgain.com>
Authored: Wed Jul 12 12:46:42 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Jul 12 12:46:42 2017 +0300
----------------------------------------------------------------------
RELEASE_NOTES.txt | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5859b192/RELEASE_NOTES.txt
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index b64aced..92286cf 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,6 +1,39 @@
Apache Ignite Release Notes
===========================
+Apache Ignite In-Memory Data Fabric 2.1
+---------------------------------------
+Ignite:
+* Persistent cache store
+* Added IgniteFuture.listenAsync() and IgniteFuture.chainAsync() mehtods
+* Deprecated IgniteConfiguration.marshaller
+* Updated Lucene dependency to version 5.5.2
+* Machine learning: implemented K-means clusterization algorithm optimized for distributed storages
+* SQL: CREATE TABLE and DROP TABLE commands support
+* SQL: New thin JDBC driver
+* SQL: Improved performance of certain queries, when affinity node can be calculated in advance
+* SQL: Fixed return type of AVG() function
+* SQL: BLOB type support added to thick JDBC driver
+* SQL: Improved LocalDate, LocalTime and LocalDateTime support for Java 8
+* SQL: Added FieldsQueryCursor interface to get fields metadata for SqlFieldsQuery
+* ODBC: Implemented DML statement batching
+* Massive performance and stability improvements
+
+Ignite.NET:
+* Automatic remote assembly loading
+* NuGet-based standalone node deployment
+* Added conditional data removeal via LINQ DeleteAll
+* Added TimestampAttribute to control DateTime serialization mode
+* Added local collections joins support to LINQ.
+
+Ignite CPP:
+* Added Compute::Call and Compute::Broadcast methods
+
+Web Console:
+* Implemented support for UNIQUE indexes for key fields on import model from RDBMS
+* Added option to show full stack trace on Queries screen
+* Added PK alias generation on Models screen.
+
Apache Ignite In-Memory Data Fabric 2.0
---------------------------------------
Ignite:
[10/11] ignite git commit: IGNITE-5744 Ignore non user caches when
automatically choosing a queryable cache inside JDBC driver
Posted by vo...@apache.org.
IGNITE-5744 Ignore non user caches when automatically choosing a queryable cache inside JDBC driver
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3c1749da
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3c1749da
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3c1749da
Branch: refs/heads/master
Commit: 3c1749da82e663500e45a34369eac48dbbc62bdc
Parents: 689b1b6
Author: Alexander Paschenko <al...@gmail.com>
Authored: Thu Jul 13 11:25:55 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Jul 13 11:25:55 2017 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/cache/GridCacheProcessor.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3c1749da/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 85772d8..f7c6e11 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1668,6 +1668,9 @@ public class GridCacheProcessor extends GridProcessorAdapter {
public IgniteCacheProxy<?, ?> getOrStartPublicCache(boolean start, boolean inclLoc) throws IgniteCheckedException {
// Try to find started cache first.
for (Map.Entry<String, GridCacheAdapter<?, ?>> e : caches.entrySet()) {
+ if (!e.getValue().context().userCache())
+ continue;
+
CacheConfiguration ccfg = e.getValue().configuration();
String cacheName = ccfg.getName();
@@ -1680,6 +1683,9 @@ public class GridCacheProcessor extends GridProcessorAdapter {
for (Map.Entry<String, DynamicCacheDescriptor> e : cachesInfo.registeredCaches().entrySet()) {
DynamicCacheDescriptor desc = e.getValue();
+ if (!desc.cacheType().userCache())
+ continue;
+
CacheConfiguration ccfg = desc.cacheConfiguration();
if (ccfg.getCacheMode() != LOCAL) {
[02/11] ignite git commit: IGNITE-5639 Added duration for empty
result set.
Posted by vo...@apache.org.
IGNITE-5639 Added duration for empty result set.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f3828261
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f3828261
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f3828261
Branch: refs/heads/master
Commit: f3828261b30c12d5aa181914033afe46c787f87e
Parents: 707c454
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Wed Jul 12 14:57:50 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Jul 12 14:57:50 2017 +0700
----------------------------------------------------------------------
modules/web-console/frontend/views/sql/sql.tpl.pug | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f3828261/modules/web-console/frontend/views/sql/sql.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/views/sql/sql.tpl.pug b/modules/web-console/frontend/views/sql/sql.tpl.pug
index f215c43..0b8011a 100644
--- a/modules/web-console/frontend/views/sql/sql.tpl.pug
+++ b/modules/web-console/frontend/views/sql/sql.tpl.pug
@@ -196,7 +196,7 @@ mixin paragraph-scan
.col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
.error(ng-switch-when='error') Error: {{paragraph.error.message}}
- .empty(ng-switch-when='empty') Result set is empty
+ .empty(ng-switch-when='empty') Result set is empty. Duration: #[b {{paragraph.duration | duration}}]
.table(ng-switch-when='table')
+table-result-heading-scan
+table-result-body
@@ -247,7 +247,7 @@ mixin paragraph-query
label Error: {{paragraph.error.message}}
br
a(ng-show='paragraph.resultType() === "error"' ng-click='showStackTrace(paragraph)') Show more
- .empty(ng-switch-when='empty') Result set is empty
+ .empty(ng-switch-when='empty') Result set is empty. Duration: #[b {{paragraph.duration | duration}}]
.table(ng-switch-when='table')
+table-result-heading-query
+table-result-body
[08/11] ignite git commit: Optimized snapshot progress tracking
Posted by vo...@apache.org.
Optimized snapshot progress tracking
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/21964fb5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/21964fb5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/21964fb5
Branch: refs/heads/master
Commit: 21964fb5f6fb6fee891283332202cbc9ed5ac3f3
Parents: 3787181
Author: Dmitry Pavlov <dp...@gmail.com>
Authored: Wed Jul 12 18:59:10 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Jul 12 18:59:10 2017 +0300
----------------------------------------------------------------------
.../ignite/internal/pagemem/FullPageId.java | 6 +-
.../ignite/internal/pagemem/PageIdUtils.java | 14 +-
.../pagemem/store/IgnitePageStoreManager.java | 1 +
.../internal/pagemem/store/PageStore.java | 2 +
.../delta/MetaPageUpdateLastAllocatedIndex.java | 2 +-
.../cache/persistence/DbCheckpointListener.java | 7 +-
.../FullPageIdIterableComparator.java | 51 -------
.../GridCacheDatabaseSharedManager.java | 63 +++++---
.../persistence/GridCacheOffheapManager.java | 56 +++----
.../cache/persistence/file/FilePageStore.java | 2 +-
.../persistence/file/FilePageStoreManager.java | 1 +
.../cache/persistence/pagemem/PageMemoryEx.java | 8 +-
.../persistence/pagemem/PageMemoryImpl.java | 10 +-
.../persistence/partstate/GroupPartitionId.java | 145 +++++++++++++++++++
.../partstate/PagesAllocationRange.java | 68 +++++++++
.../partstate/PartitionAllocationMap.java | 113 +++++++++++++++
.../snapshot/IgniteCacheSnapshotManager.java | 17 ++-
.../cache/persistence/tree/io/PageMetaIO.java | 27 ++--
.../persistence/tree/io/TrackingPageIO.java | 8 +-
.../persistence/tree/util/PageHandler.java | 3 +-
20 files changed, 466 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
index 00f52c1..9e24943 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/FullPageId.java
@@ -21,7 +21,7 @@ import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
/**
- * Compound object used to address a page in the global page space.
+ * Compound object used to address a page in the global page space.
* <h3>Page ID structure</h3>
* <p>
* Generally, a full page ID consists of a cache ID and page ID. A page ID consists of
@@ -49,13 +49,13 @@ import org.apache.ignite.internal.util.typedef.internal.U;
* Effective page ID is page ID with zeroed bits used for page ID rotation.
*/
public class FullPageId {
- /** */
+ /** Page ID. */
private final long pageId;
/** */
private final long effectivePageId;
- /** */
+ /** Cache group ID. */
private final int grpId;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
index 92f427a..6f4ba93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
@@ -49,10 +49,10 @@ public final class PageIdUtils {
/** */
public static final long TAG_MASK = ~(-1L << TAG_SIZE);
- /** */
+ /** Page Index is a monotonically growing number within each partition */
public static final long PART_ID_MASK = ~(-1L << PART_ID_SIZE);
- /** */
+ /** Flags mask. Flags consists from a number of reserved bits, and page type (data/index page) */
public static final long FLAG_MASK = ~(-1L << FLAG_SIZE);
/** */
@@ -92,10 +92,10 @@ public final class PageIdUtils {
}
/**
- * Extracts a page index from the given pageId.
+ * Extracts a page index from the given page ID.
*
- * @param pageId Page id.
- * @return Page ID.
+ * @param pageId Page ID.
+ * @return Page index.
*/
public static int pageIndex(long pageId) {
return (int)(pageId & PAGE_IDX_MASK); // 4 bytes
@@ -150,7 +150,9 @@ public final class PageIdUtils {
/**
* @param partId Partition ID.
- * @return Part ID constructed from the given cache ID and partition ID.
+ * @param flag Flags (a number of reserved bits, and page type (data/index page))
+ * @param pageIdx Page index, monotonically growing number within each partition
+ * @return Page ID constructed from the given pageIdx and partition ID, see {@link FullPageId}
*/
public static long pageId(int partId, byte flag, int pageIdx) {
long pageId = flag & FLAG_MASK;
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index a1b766f..eaa85ad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -77,6 +77,7 @@ public interface IgnitePageStoreManager extends GridCacheSharedManager, IgniteCh
*
* @param grpId Cache group ID of the evicted partition.
* @param partId Partition ID.
+ * @param tag Partition tag (growing 1-based partition file version).
* @throws IgniteCheckedException If failed to handle partition destroy callback.
*/
public void onPartitionDestroyed(int grpId, int partId, int tag) throws IgniteCheckedException;
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
index be83704..4698a6b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
@@ -70,6 +70,8 @@ public interface PageStore {
*
* @param pageId Page ID.
* @param pageBuf Page buffer to write.
+ * @param tag Partition file version, 1-based incrementing counter. For outdated pages {@code tag} has lower value,
+ * and write does nothing
* @throws IgniteCheckedException If page writing failed (IO error occurred).
*/
public void write(long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException;
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java
index 60aebde..11b2a67 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java
@@ -44,7 +44,7 @@ public class MetaPageUpdateLastAllocatedIndex extends PageDeltaRecord {
PageMetaIO io = PageMetaIO.VERSIONS.forVersion(PageIO.getVersion(pageAddr));
- io.setLastPageCount(pageAddr, lastAllocatedIdx);
+ io.setLastAllocatedPageCount(pageAddr, lastAllocatedIdx);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java
index 0b28b6a..1c438b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java
@@ -17,9 +17,8 @@
package org.apache.ignite.internal.processors.cache.persistence;
-import java.util.Map;
import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap;
/**
*
@@ -35,9 +34,9 @@ public interface DbCheckpointListener {
public boolean nextSnapshot();
/**
- *
+ * @return Partition allocation statistic map
*/
- public Map<T2<Integer, Integer>, T2<Integer, Integer>> partitionStatMap();
+ public PartitionAllocationMap partitionStatMap();
/**
* @param cacheOrGrpName Cache or group name.
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java
deleted file mode 100644
index c056c52..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence;
-
-import java.io.Serializable;
-import java.util.Comparator;
-import org.apache.ignite.internal.util.typedef.T2;
-
-/**
- *
- */
-public class FullPageIdIterableComparator implements Comparator<T2<Integer, Integer>>, Serializable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- public static final FullPageIdIterableComparator INSTANCE = new FullPageIdIterableComparator();
-
- /** {@inheritDoc} */
- @Override public int compare(T2<Integer, Integer> o1, T2<Integer, Integer> o2) {
- if (o1.get1() < o2.get1())
- return -1;
-
- if (o1.get1() > o2.get1())
- return 1;
-
- if (o1.get2() < o2.get2())
- return -1;
-
- if (o1.get2() > o2.get2())
- return 1;
-
- return 0;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 5136731..9f2067a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -40,7 +40,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
-import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -52,6 +51,7 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
@@ -109,6 +109,7 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStor
import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperation;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
@@ -263,8 +264,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** */
private boolean stopping;
- /** Checkpoint runner thread pool. */
- private ExecutorService asyncRunner;
+ /** Checkpoint runner thread pool. If null tasks are to be run in single thread */
+ @Nullable private ExecutorService asyncRunner;
/** Buffer for the checkpoint threads. */
private ThreadLocal<ByteBuffer> threadBuf;
@@ -1916,6 +1917,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
asyncRunner == null ? 1 : chp.cpPages.collectionsSize());
tracker.onPagesWriteStart();
+ final AtomicInteger writtenPagesCtr = new AtomicInteger();
+ final int totalPagesToWriteCnt = chp.cpPages.size();
if (asyncRunner != null) {
for (int i = 0; i < chp.cpPages.collectionsSize(); i++) {
@@ -1923,7 +1926,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
tracker,
chp.cpPages.innerCollection(i),
updStores,
- doneWriteFut
+ doneWriteFut,
+ writtenPagesCtr,
+ totalPagesToWriteCnt
);
try {
@@ -1937,7 +1942,12 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
else {
// Single-threaded checkpoint.
- Runnable write = new WriteCheckpointPages(tracker, chp.cpPages, updStores, doneWriteFut);
+ Runnable write = new WriteCheckpointPages(tracker,
+ chp.cpPages,
+ updStores,
+ doneWriteFut,
+ writtenPagesCtr,
+ totalPagesToWriteCnt);
write.run();
}
@@ -2092,15 +2102,15 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
curCpProgress = curr;
}
- final NavigableMap<T2<Integer, Integer>, T2<Integer, Integer>> map =
- new TreeMap<>(FullPageIdIterableComparator.INSTANCE);
+ final PartitionAllocationMap map = new PartitionAllocationMap();
DbCheckpointListener.Context ctx0 = new DbCheckpointListener.Context() {
@Override public boolean nextSnapshot() {
return curr.nextSnapshot;
}
- @Override public Map<T2<Integer, Integer>, T2<Integer, Integer>> partitionStatMap() {
+ /** {@inheritDoc} */
+ @Override public PartitionAllocationMap partitionStatMap() {
return map;
}
@@ -2278,14 +2288,12 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
}
- /**
- *
- */
+ /** Pages write task */
private class WriteCheckpointPages implements Runnable {
/** */
private CheckpointMetricsTracker tracker;
- /** */
+ /** Collection of page IDs to write under this task. Overall pages to write may be greater than this collection*/
private Collection<FullPageId> writePageIds;
/** */
@@ -2294,19 +2302,34 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** */
private CountDownFuture doneFut;
+ /** Counter for all written pages. May be shared between several workers */
+ private AtomicInteger writtenPagesCntr;
+
+ /** Total pages to write, counter may be greater than {@link #writePageIds} size*/
+ private final int totalPagesToWrite;
+
/**
- * @param writePageIds Write page IDs.
+ * Creates task for write pages
+ * @param tracker
+ * @param writePageIds Collection of page IDs to write.
+ * @param updStores
+ * @param doneFut
+ * @param writtenPagesCntr all written pages counter, may be shared between several write tasks
+ * @param totalPagesToWrite total pages to be written under this checkpoint
*/
private WriteCheckpointPages(
- CheckpointMetricsTracker tracker,
- Collection<FullPageId> writePageIds,
- GridConcurrentHashSet<PageStore> updStores,
- CountDownFuture doneFut
- ) {
+ final CheckpointMetricsTracker tracker,
+ final Collection<FullPageId> writePageIds,
+ final GridConcurrentHashSet<PageStore> updStores,
+ final CountDownFuture doneFut,
+ @NotNull final AtomicInteger writtenPagesCntr,
+ final int totalPagesToWrite) {
this.tracker = tracker;
this.writePageIds = writePageIds;
this.updStores = updStores;
this.doneFut = doneFut;
+ this.writtenPagesCntr = writtenPagesCntr;
+ this.totalPagesToWrite = totalPagesToWrite;
}
/** {@inheritDoc} */
@@ -2354,7 +2377,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
tmpWriteBuf.rewind();
}
- snapshotMgr.onPageWrite(fullId, tmpWriteBuf);
+ int curWrittenPages = writtenPagesCntr.incrementAndGet();
+
+ snapshotMgr.onPageWrite(fullId, tmpWriteBuf, curWrittenPages, totalPagesToWrite);
tmpWriteBuf.rewind();
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index bd902fb..6e6b7df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -52,6 +52,9 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.PagesAllocationRange;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionCountersIO;
@@ -63,7 +66,6 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.lang.GridCursor;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
@@ -207,9 +209,9 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
try {
- long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
+ long partMetaPageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
- if (pageAddr == 0L) {
+ if (partMetaPageAddr == 0L) {
U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage +
", saveMeta=" + saveMeta + ", beforeDestroy=" + beforeDestroy + ", size=" + size +
", updCntr=" + updCntr + ", state=" + state + ']');
@@ -220,21 +222,21 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
boolean changed = false;
try {
- PagePartitionMetaIO io = PageIO.getPageIO(pageAddr);
+ PagePartitionMetaIO io = PageIO.getPageIO(partMetaPageAddr);
- changed |= io.setUpdateCounter(pageAddr, updCntr);
- changed |= io.setGlobalRemoveId(pageAddr, rmvId);
- changed |= io.setSize(pageAddr, size);
+ changed |= io.setUpdateCounter(partMetaPageAddr, updCntr);
+ changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId);
+ changed |= io.setSize(partMetaPageAddr, size);
if (state != null)
- changed |= io.setPartitionState(pageAddr, (byte)state.ordinal());
+ changed |= io.setPartitionState(partMetaPageAddr, (byte)state.ordinal());
else
assert grp.isLocal() : grp.cacheOrGroupName();
long cntrsPageId;
if (grp.sharedGroup()) {
- cntrsPageId = io.getCountersPageId(pageAddr);
+ cntrsPageId = io.getCountersPageId(partMetaPageAddr);
byte[] data = serializeCacheSizes(store.cacheSizes());
@@ -247,7 +249,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (init && items > 0) {
cntrsPageId = pageMem.allocatePage(grpId, store.partId(), PageIdAllocator.FLAG_DATA);
- io.setCountersPageId(pageAddr, cntrsPageId);
+ io.setCountersPageId(partMetaPageAddr, cntrsPageId);
changed = true;
}
@@ -301,7 +303,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (needSnapshot) {
pageCnt = this.ctx.pageStore().pages(grpId, store.partId());
- io.setCandidatePageCount(pageAddr, pageCnt);
+ io.setCandidatePageCount(partMetaPageAddr, pageCnt);
if (saveMeta) {
long metaPageId = pageMem.metaPageId(grpId);
@@ -345,13 +347,13 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (partMap.containsKey(store.partId()) &&
partMap.get(store.partId()) == GridDhtPartitionState.OWNING)
- addPartition(ctx.partitionStatMap(), pageAddr, io, grpId, store.partId(),
+ addPartition(ctx.partitionStatMap(), partMetaPageAddr, io, grpId, store.partId(),
this.ctx.pageStore().pages(grpId, store.partId()));
changed = true;
}
else
- pageCnt = io.getCandidatePageCount(pageAddr);
+ pageCnt = io.getCandidatePageCount(partMetaPageAddr);
if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null))
wal.log(new MetaPageUpdatePartitionDataRecord(
@@ -397,27 +399,29 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
/**
* @param map Map to add values to.
- * @param pageAddr page address
+ * @param metaPageAddr Meta page address
* @param io Page Meta IO
* @param cacheId Cache ID.
- * @param partition Partition ID.
- * @param pages Number of pages to add.
+ * @param partId Partition ID. Or {@link PageIdAllocator#INDEX_PARTITION} for index partition
+ * @param currAllocatedPageCnt total number of pages allocated for partition <code>[partition, cacheId]</code>
*/
private static void addPartition(
- Map<T2<Integer, Integer>, T2<Integer, Integer>> map,
- long pageAddr,
- PageMetaIO io,
- int cacheId,
- int partition,
- int pages
+ final PartitionAllocationMap map,
+ final long metaPageAddr,
+ final PageMetaIO io,
+ final int cacheId,
+ final int partId,
+ final int currAllocatedPageCnt
) {
- if (pages <= 1)
+ if (currAllocatedPageCnt <= 1)
return;
- assert PageIO.getPageId(pageAddr) != 0;
+ assert PageIO.getPageId(metaPageAddr) != 0;
- int lastAllocatedIdx = io.getLastPageCount(pageAddr);
- map.put(new T2<>(cacheId, partition), new T2<>(lastAllocatedIdx, pages));
+ int lastAllocatedPageCnt = io.getLastAllocatedPageCount(metaPageAddr);
+ map.put(
+ new GroupPartitionId(cacheId, partId),
+ new PagesAllocationRange(lastAllocatedPageCnt, currAllocatedPageCnt));
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index c827e96..a7ca13c 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -77,7 +77,7 @@ public class FilePageStore implements PageStore {
/** */
private volatile boolean recover;
- /** */
+ /** Partition file version, 1-based incrementing counter. For outdated pages tag has low value, and write does nothing */
private volatile int tag;
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index af20136..e2ad070 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -318,6 +318,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
* @param cacheId Cache ID to write.
* @param pageId Page ID.
* @param pageBuf Page buffer.
+ * @param tag Partition tag (growing 1-based partition file version). Used to validate page is not outdated
* @return PageStore to which the page has been written.
* @throws IgniteCheckedException If IO error occurred.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
index 7c63d41..53e21b7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
@@ -115,19 +115,19 @@ public interface PageMemoryEx extends PageMemory {
*
* @param pageId Page ID to get byte buffer for. The page ID must be present in the collection returned by
* the {@link #beginCheckpoint()} method call.
- * @param tmpBuf Temporary buffer to write changes into.
+ * @param outBuf Temporary buffer to write changes into.
* @param tracker Checkpoint metrics tracker.
- * @return {@code True} if data were read, {@code false} otherwise (data already saved to storage).
+ * @return {@code Partition tag} if data was read, {@code null} otherwise (data already saved to storage).
* @throws IgniteException If failed to obtain page data.
*/
- @Nullable public Integer getForCheckpoint(FullPageId pageId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker);
+ @Nullable public Integer getForCheckpoint(FullPageId pageId, ByteBuffer outBuf, CheckpointMetricsTracker tracker);
/**
* Marks partition as invalid / outdated.
*
* @param cacheId Cache ID.
* @param partId Partition ID.
- * @return New partition tag.
+ * @return New partition tag (growing 1-based partition file version).
*/
public int invalidate(int cacheId, int partId);
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 47381d7..1b4cf81 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -792,8 +792,8 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public Integer getForCheckpoint(FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) {
- assert tmpBuf.remaining() == pageSize();
+ @Override public Integer getForCheckpoint(FullPageId fullId, ByteBuffer outBuf, CheckpointMetricsTracker tracker) {
+ assert outBuf.remaining() == pageSize();
Segment seg = segment(fullId.groupId(), fullId.pageId());
@@ -876,7 +876,7 @@ public class PageMemoryImpl implements PageMemoryEx {
}
}
else
- return copyPageForCheckpoint(absPtr, fullId, tmpBuf, tmpBuffer, tracker) ? tag : null;
+ return copyPageForCheckpoint(absPtr, fullId, outBuf, tmpBuffer, tracker) ? tag : null;
}
/**
@@ -1565,7 +1565,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** */
private final int maxDirtyPages;
- /** */
+ /** Maps partition (cacheId, partId) to its tag. Tag is 1-based incrementing partition file counter */
private final Map<T2<Integer, Integer>, Integer> partTagMap = new HashMap<>();
/**
@@ -1903,7 +1903,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/**
* @param grpId Cache group ID.
* @param partId Partition ID.
- * @return Partition tag.
+ * @return Partition tag. Growing 1 based partition file version
*/
private int partTag(int grpId, int partId) {
assert getReadHoldCount() > 0 || getWriteHoldCount() > 0;
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/GroupPartitionId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/GroupPartitionId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/GroupPartitionId.java
new file mode 100644
index 0000000..dbdf670
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/GroupPartitionId.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.partstate;
+
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Pair of cache group ID with partition ID. Immutable, comparable class, may be used as key in maps
+ */
+public class GroupPartitionId implements Comparable<GroupPartitionId> {
+ /** Index for super(meta) page. There is always such page for iterated cache partition */
+ private static final int METAPAGE_IDX = 0;
+
+ /** Cache group ID. */
+ private final int grpId;
+
+ /** Partition ID. */
+ private final int partId;
+
+ /**
+ * Creates group-partition tuple.
+ *
+ * @param grpId Group ID.
+ * @param partId Partition ID.
+ */
+ public GroupPartitionId(final int grpId, final int partId) {
+ this.grpId = grpId;
+ this.partId = partId;
+ }
+
+ /**
+ * @param partId Partition ID.
+ * @return flag to be used for partition
+ */
+ private static byte getFlagByPartId(final int partId) {
+ return partId == PageIdAllocator.INDEX_PARTITION ? PageMemory.FLAG_IDX : PageMemory.FLAG_DATA;
+ }
+
+ /**
+ * @return cache ID
+ */
+ public int getGroupId() {
+ return grpId;
+ }
+
+ /**
+ * @return Partition ID
+ */
+ public int getPartitionId() {
+ return partId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(GroupPartitionId.class, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ GroupPartitionId key = (GroupPartitionId)o;
+
+ if (grpId != key.grpId)
+ return false;
+
+ return partId == key.partId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int result = grpId;
+
+ result = 31 * result + partId;
+
+ return result;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compareTo(@NotNull GroupPartitionId o) {
+ if (getGroupId() < o.getGroupId())
+ return -1;
+
+ if (getGroupId() > o.getGroupId())
+ return 1;
+
+ if (getPartitionId() < o.getPartitionId())
+ return -1;
+
+ if (getPartitionId() > o.getPartitionId())
+ return 1;
+ return 0;
+ }
+
+ /**
+ * @param pageIdx Page Index, monotonically growing number within each partition
+ * @return page ID (64 bits) constructed from partition ID and given index
+ */
+ private long createPageId(final int pageIdx) {
+ final int partId = getPartitionId();
+
+ return PageIdUtils.pageId(partId, getFlagByPartId(partId), pageIdx);
+ }
+
+ /**
+ * Returns Full page ID. For index 0 will return super-page of next partition
+ *
+ * @param pageIdx Page Index, monotonically growing number within each partition
+ * @return FullPageId consists of cache ID (32 bits) and page ID (64 bits).
+ */
+ @NotNull private FullPageId createFullPageId(final int pageIdx) {
+ return new FullPageId(createPageId(pageIdx), getGroupId());
+ }
+
+ /**
+ * @return will return super-page (metapage) of this partition
+ */
+ @NotNull public FullPageId createFirstPageFullId() {
+ return createFullPageId(METAPAGE_IDX);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
new file mode 100644
index 0000000..e7170c3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.partstate;
+
+/**
+ * Range of pages allocated.
+ * Contains currently allocated page count and previously observed page count.
+ * May be used for tracking history of recent allocation for partition <code>[partition, cacheId]</code>
+ */
+public class PagesAllocationRange {
+ /**
+ * Previously observed total number of allocated pages. May be stored using PageMetaIO.
+ * Used to separate newly allocated pages with previously observed state
+ * Minimum value is 0. Can't be greater than {@link #currAllocatedPageCnt}
+ */
+ private final int lastAllocatedPageCnt;
+
+ /** Total current number of pages allocated, minimum value is 0. */
+ private final int currAllocatedPageCnt;
+
+ /**
+ * Creates pages range
+ *
+ * @param lastAllocatedPageCnt Last allocated pages count.
+ * @param currAllocatedPageCnt Currently allocated pages count.
+ */
+ public PagesAllocationRange(final int lastAllocatedPageCnt, final int currAllocatedPageCnt) {
+ this.lastAllocatedPageCnt = lastAllocatedPageCnt;
+ this.currAllocatedPageCnt = currAllocatedPageCnt;
+ }
+
+ /**
+ * @return Total current number of pages allocated, minimum value is 0.
+ */
+ public int getCurrAllocatedPageCnt() {
+ return currAllocatedPageCnt;
+ }
+
+ /**
+ * @return Previously observed total number of allocated pages.
+ */
+ public int getLastAllocatedPageCnt() {
+ return lastAllocatedPageCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "PagesAllocationRange{" +
+ "lastAllocatedPageCnt=" + lastAllocatedPageCnt +
+ ", currAllocatedPageCnt=" + currAllocatedPageCnt +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
new file mode 100644
index 0000000..9ed4000
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.partstate;
+
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Information structure with partitions state.
+ * Page counts map.
+ */
+public class PartitionAllocationMap {
+ /** Maps following pairs: (groupId, partId) -> (lastAllocatedCount, allocatedCount) */
+ private final NavigableMap<GroupPartitionId, PagesAllocationRange> map = new TreeMap<>();
+
+ /**
+ * Returns the value to which the specified key is mapped,
+ * or {@code null} if this map contains no mapping for the key.
+ *
+ * @param key to get
+ * @return value or null
+ */
+ @Nullable public PagesAllocationRange get(GroupPartitionId key) {
+ return map.get(key);
+ }
+
+ /**
+ * Extracts partition information from full page ID
+ *
+ * @param fullId page related to some cache
+ * @return pair of cache ID and partition ID
+ */
+ @NotNull public static GroupPartitionId createCachePartId(@NotNull final FullPageId fullId) {
+ return new GroupPartitionId(fullId.groupId(), PageIdUtils.partId(fullId.pageId()));
+ }
+
+ /** @return <tt>true</tt> if this map contains no key-value mappings */
+ public boolean isEmpty() {
+ return map.isEmpty();
+ }
+
+ /** @return the number of key-value mappings in this map. */
+ public int size() {
+ return map.size();
+ }
+
+ /** @return keys (all caches partitions) */
+ public Set<GroupPartitionId> keySet() {
+ return map.keySet();
+ }
+
+ /** @return values (allocation ranges) */
+ public Iterable<PagesAllocationRange> values() {
+ return map.values();
+ }
+
+ /** @return Returns the first (lowest) key currently in this map. */
+ public GroupPartitionId firstKey() {
+ return map.firstKey();
+ }
+
+ /**
+ * Returns next (higher) key for provided cache and partition or null
+ *
+ * @param key cache and partition to search
+ * @return first found key which is greater than provided
+ */
+ @Nullable public GroupPartitionId nextKey(@NotNull final GroupPartitionId key) {
+ return map.navigableKeySet().higher(key);
+ }
+
+ /** @return set view of the mappings contained in this map, sorted in ascending key order */
+ public Set<Map.Entry<GroupPartitionId, PagesAllocationRange>> entrySet() {
+ return map.entrySet();
+ }
+
+ /** @return <tt>true</tt> if this map contains a mapping for the specified key */
+ public boolean containsKey(GroupPartitionId key) {
+ return map.containsKey(key);
+ }
+
+ /**
+ * @param key key with which the specified value is to be associated
+ * @param val value to be associated with the specified key
+ * @return the previous value associated with <tt>key</tt>, or <tt>null</tt> if there was no mapping for
+ * <tt>key</tt>.
+ */
+ public PagesAllocationRange put(GroupPartitionId key, PagesAllocationRange val) {
+ return map.put(key, val);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
index 0a27bcd..50e6515 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.cache.persistence.snapshot;
import java.nio.ByteBuffer;
-import java.util.NavigableMap;
import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.events.DiscoveryEvent;
@@ -29,8 +28,8 @@ import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap;
import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport;
-import org.apache.ignite.internal.util.typedef.T2;
import org.jetbrains.annotations.Nullable;
/**
@@ -64,12 +63,13 @@ public class IgniteCacheSnapshotManager<T extends SnapshotOperation> extends Gri
/**
* @param snapshotOperation current snapshot operation.
+ * @param map (cacheId, partId) -> (lastAllocatedIndex, count)
*
* @return {@code true} if next operation must be snapshot, {@code false} if checkpoint must be executed.
*/
public boolean onMarkCheckPointBegin(
T snapshotOperation,
- NavigableMap<T2<Integer, Integer>, T2<Integer, Integer>> map
+ PartitionAllocationMap map
) throws IgniteCheckedException {
return false;
}
@@ -107,9 +107,16 @@ public class IgniteCacheSnapshotManager<T extends SnapshotOperation> extends Gri
}
/**
- * @param fullId Full id.
+ * @param fullId Full page id.
+ * @param tmpWriteBuf buffer
+ * @param writtenPages Overall pages written, negative value means there is no progress tracked
+ * @param totalPages Overall pages count to be written, should be positive
*/
- public void onPageWrite(FullPageId fullId, ByteBuffer tmpWriteBuf) {
+ public void onPageWrite(
+ final FullPageId fullId,
+ final ByteBuffer tmpWriteBuf,
+ final int writtenPages,
+ final int totalPages) {
// No-op.
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
index ac482e8..becd3e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
@@ -43,10 +43,10 @@ public class PageMetaIO extends PageIO {
/** Last successful full snapshot tag offset. */
private static final int LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF = NEXT_SNAPSHOT_TAG_OFF + 8;
- /** Last allocated index offset. */
+ /** Last allocated pages count offset. */
private static final int LAST_PAGE_COUNT_OFF = LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF + 8;
- /** Candidate allocated index offset. */
+ /** Candidate allocated page count offset. */
private static final int CANDIDATE_PAGE_COUNT_OFF = LAST_PAGE_COUNT_OFF + 4;
/** End of page meta. */
@@ -82,7 +82,7 @@ public class PageMetaIO extends PageIO {
setLastSuccessfulSnapshotId(pageAddr, 0);
setNextSnapshotTag(pageAddr, 1);
setLastSuccessfulSnapshotTag(pageAddr, 0);
- setLastPageCount(pageAddr, 0);
+ setLastAllocatedPageCount(pageAddr, 0);
setCandidatePageCount(pageAddr, 0);
}
@@ -179,24 +179,31 @@ public class PageMetaIO extends PageIO {
}
/**
- * @param pageAddr Page address.
- * @param pageCnt Last allocated index.
+ * Sets last allocated pages count, used to save and observe previous allocated count
+ *
+ * @param pageAddr Meta Page address.
+ * @param pageCnt Last allocated pages count to set
*/
- public void setLastPageCount(long pageAddr, int pageCnt) {
+ public void setLastAllocatedPageCount(final long pageAddr, final int pageCnt) {
PageUtils.putInt(pageAddr, LAST_PAGE_COUNT_OFF, pageCnt);
}
/**
- * @param buf Buffer.
+ * Gets last allocated pages count from given buffer
+ *
+ * @param buf Buffer to read data from.
*/
- public int getLastPageCount(@NotNull ByteBuffer buf) {
+ public int getLastAllocatedPageCount(@NotNull final ByteBuffer buf) {
return buf.getInt(LAST_PAGE_COUNT_OFF);
}
/**
- * @param pageAddr Page address.
+ * Gets last allocated pages count by provided address
+ *
+ * @param pageAddr Meta page address.
+ * @return Last allocated page count
*/
- public int getLastPageCount(long pageAddr) {
+ public int getLastAllocatedPageCount(final long pageAddr) {
return PageUtils.getInt(pageAddr, LAST_PAGE_COUNT_OFF);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
index 2263130..2051778 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.persistence.tree.io;
import java.nio.ByteBuffer;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.jetbrains.annotations.Nullable;
/**
* We use dedicated page for tracking pages updates.
@@ -182,6 +183,7 @@ public class TrackingPageIO extends PageIO {
* @param buf Buffer.
* @param pageId Page id.
* @param curSnapshotTag Snapshot tag.
+ * @param lastSuccessfulSnapshotTag Last successful snapshot id.
* @param pageSize Page size.
*/
public boolean wasChanged(ByteBuffer buf, long pageId, long curSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
@@ -265,10 +267,12 @@ public class TrackingPageIO extends PageIO {
* @param buf Buffer.
* @param start Start.
* @param curSnapshotTag Snapshot id.
+ * @param lastSuccessfulSnapshotTag Last successful snapshot id.
* @param pageSize Page size.
- * @return set pageId if it was changed or next closest one, if there is no changed page null will be returned
+ * @return set pageId if it was changed or next closest one, if there is no changed page {@code null} will be returned
*/
- public Long findNextChangedPage(ByteBuffer buf, long start, long curSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
+ @Nullable public Long findNextChangedPage(ByteBuffer buf, long start, long curSnapshotTag,
+ long lastSuccessfulSnapshotTag, int pageSize) {
validateSnapshotId(buf, curSnapshotTag + 1, lastSuccessfulSnapshotTag, pageSize);
int cntOfPage = countOfPageToTrack(pageSize);
http://git-wip-us.apache.org/repos/asf/ignite/blob/21964fb5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
index a87525a..3316980 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.persistence.tree.util;
import java.nio.ByteBuffer;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageSupport;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
@@ -440,7 +441,7 @@ public abstract class PageHandler<X, R> {
* @return {@code true} If we need to make a delta WAL record for the change in this page.
*/
public static boolean isWalDeltaRecordNeeded(
- PageMemory pageMem,
+ PageSupport pageMem,
int cacheId,
long pageId,
long page,
[07/11] ignite git commit: More verbose logging
Posted by vo...@apache.org.
More verbose logging
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/37871813
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/37871813
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/37871813
Branch: refs/heads/master
Commit: 3787181310597b7a6e633e745ba08209abd038a9
Parents: 0cb6ac0
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Wed Jul 12 18:28:57 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Jul 12 18:28:57 2017 +0300
----------------------------------------------------------------------
.../dht/preloader/GridDhtPartitionsExchangeFuture.java | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/37871813/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 5760f87..3b6fe91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1008,11 +1008,19 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
*/
private void tryToPerformLocalSnapshotOperation() {
try {
+ long start = U.currentTimeMillis();
+
IgniteInternalFuture fut = cctx.snapshot()
.tryStartLocalSnapshotOperation(discoEvt);
if (fut != null)
fut.get();
+
+ long end = U.currentTimeMillis();
+
+ if (log.isInfoEnabled())
+ log.info("Snapshot initialization completed [topVer=" + exchangeId().topologyVersion() +
+ ", time=" + (end - start) + "ms]");
}
catch (IgniteCheckedException e) {
U.error(log, "Error while starting snapshot operation", e);