You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2018/01/17 10:35:13 UTC
[01/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Repository: ignite
Updated Branches:
refs/heads/master 43a62cdba -> 6f7aba852
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
index f3268d9..2da1aa3 100644
--- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
@@ -18,6 +18,7 @@
package org.apache.ignite.development.utils;
import java.io.File;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
@@ -30,6 +31,7 @@ import org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO;
import org.apache.ignite.internal.processors.query.h2.database.io.H2LeafIO;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.logger.NullLogger;
+import org.jetbrains.annotations.Nullable;
/**
* Print WAL log data in human-readable form.
@@ -41,7 +43,7 @@ public class IgniteWalConverter {
public static void main(String[] args) throws Exception {
if (args.length < 2)
throw new IllegalArgumentException("\nYou need to provide:\n" +
- "\t1. Size of pages (1024, 2048, etc).\n" +
+ "\t1. Size of pages, which was selected for file store (1024, 2048, 4096, etc).\n" +
"\t2. Path to dir with wal files.\n" +
"\t3. (Optional) Path to dir with archive wal files.");
@@ -49,6 +51,9 @@ public class IgniteWalConverter {
H2ExtrasInnerIO.register();
H2ExtrasLeafIO.register();
+ boolean printRecords = IgniteSystemProperties.getBoolean("PRINT_RECORDS", false);
+ boolean printStat = IgniteSystemProperties.getBoolean("PRINT_STAT", true);
+
final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(),
Integer.parseInt(args[0]),
null,
@@ -62,11 +67,20 @@ public class IgniteWalConverter {
if (workFiles == null)
throw new IllegalArgumentException("No .wal files in dir: " + args[1]);
+ @Nullable final WalStat stat = printStat ? new WalStat() : null;
+
try (WALIterator stIt = factory.iteratorWorkFiles(workFiles)) {
while (stIt.hasNextX()) {
IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
- System.out.println("[W] " + next.get2());
+ final WALPointer pointer = next.get1();
+ final WALRecord record = next.get2();
+
+ if (stat != null)
+ stat.registerRecord(record, pointer, true);
+
+ if (printRecords)
+ System.out.println("[W] " + record);
}
}
@@ -77,9 +91,21 @@ public class IgniteWalConverter {
while (stIt.hasNextX()) {
IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
- System.out.println("[A] " + next.get2());
+ final WALPointer pointer = next.get1();
+ final WALRecord record = next.get2();
+
+ if (stat != null)
+ stat.registerRecord(record, pointer, false);
+
+ if (printRecords)
+ System.out.println("[A] " + record);
}
}
}
+
+ System.err.flush();
+
+ if (stat != null)
+ System.out.println("Statistic collected:\n" + stat.toString());
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/RecordSizeCountStat.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/RecordSizeCountStat.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/RecordSizeCountStat.java
new file mode 100644
index 0000000..1d0a1c0
--- /dev/null
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/RecordSizeCountStat.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.development.utils;
+
+/**
+ * Statistic for record size, used to accumulate information about several record and calculating average.
+ */
+public class RecordSizeCountStat {
+ /** Sum of sizes. */
+ private long size = -1;
+
+ /** Count of all records. */
+ private int cnt;
+
+ /**
+ * @param size record size. Negative value means size is unknown for current occurrence. Any negative value resets
+ * accumulated statistics.
+ */
+ void occurrence(int size) {
+ if (size >= 0) {
+ if (this.size < 0)
+ this.size = 0;
+
+ this.size += size;
+ }
+ else {
+ if (this.size > 0)
+ this.size = -1;
+ }
+
+ cnt++;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return (size >= 0 ? size : "") +
+ "\t " + cnt +
+ "\t " + ((size >= 0) ? size / cnt : "");
+ }
+
+ /**
+ * @return average size of one record, as double.
+ */
+ private double averageD() {
+ return 1.0 * size / cnt;
+ }
+
+ /**
+ * @return average size of one record, printable version.
+ */
+ String averageStr() {
+ return cnt == 0 ? "" : String.format("%.2f", averageD());
+ }
+
+ /**
+ * @return Count of all records.
+ */
+ int getCount() {
+ return cnt;
+ }
+
+ /**
+ * @param val other statistic value to reduce with.
+ */
+ void add(RecordSizeCountStat val) {
+ cnt += val.cnt;
+ size += val.size;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/TxWalStat.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/TxWalStat.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/TxWalStat.java
new file mode 100644
index 0000000..a45b8ea
--- /dev/null
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/TxWalStat.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.development.utils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+
+/**
+ * Transactions statistics for WAL.
+ */
+public class TxWalStat {
+ /** Maximum element in popular combinations discovery map before eviction. */
+ private static final int POPULAR_COMBINATION_MAP_MAX_SIZE = 50000;
+
+ /** Usages count, when evict prohibited from popular combinations discovery map. */
+ private static final int USAGES_CNT_EVICT_PROHIBITED = 10;
+
+ /** Transactions in preparing/committing state. When commit is finished, TX is removed from this collection. */
+ private final Map<GridCacheVersion, TxOccurrence> opened = new HashMap<>();
+
+ /** Field for calculating average number of primary nodes involved in Tx. */
+ private final RecordSizeCountStat avgPrimaryNodes = new RecordSizeCountStat();
+
+ /** Field for calculating average number of total nodes involved in Tx. */
+ private final RecordSizeCountStat avgTotalNodes = new RecordSizeCountStat();
+
+ /** Tx statistics: Entries updated count -> count of such Txes. */
+ private final Map<Integer, Integer> txEntriesUpdated = new TreeMap<>();
+
+ /** Tx statistics: Caches Involved count -> count of such Txes. */
+ private final Map<Integer, Integer> txCachesInvolved = new TreeMap<>();
+
+ /** Cache IDs combination involved in Tx. Popular combination search map, with possible eviction. */
+ private final LruMap<String> cacheIdsInTx = new LruMap<>(POPULAR_COMBINATION_MAP_MAX_SIZE,
+ USAGES_CNT_EVICT_PROHIBITED);
+
+ /**
+ * Cache IDs combination involved in Tx, weighted using primary nodes in Tx. Used to search popular combinations
+ * mostly involved into highly distributive transactions.
+ */
+ private final LruMap<String> cacheIdsWeightedNodesInTx = new LruMap<>(POPULAR_COMBINATION_MAP_MAX_SIZE,
+ USAGES_CNT_EVICT_PROHIBITED * 90);
+ /**
+ * Cache IDs combination involved in Tx, weighted using total nodes in Tx. Used to search popular combinations
+ * mostly involved into highly distributive transactions.
+ */
+ private final LruMap<String> cacheIdsWeightedTotalNodesInTx = new LruMap<>(POPULAR_COMBINATION_MAP_MAX_SIZE,
+ USAGES_CNT_EVICT_PROHIBITED * 150);
+
+ /**
+ * @param key key (parameter) value found.
+ * @param map map to save increment.
+ * @param <K> type of key.
+ */
+ private static <K> void incrementStat(K key, Map<K, Integer> map) {
+ incrementStat(key, map, 1);
+ }
+
+ /**
+ * @param key key (parameter) value found.
+ * @param map map to save increment.
+ * @param increment value to increment statistic, 1 or weight of current occurrence.
+ * @param <K> type of key.
+ */
+ private static <K> void incrementStat(K key, Map<K, Integer> map, int increment) {
+ Integer val = map.get(key);
+ int recordStat = val == null ? 0 : val;
+
+ recordStat += increment;
+ map.put(key, recordStat);
+ }
+
+ /**
+ * Handles TX prepare: creates TX in {@link #opened} map.
+ * @param nearXidVer near Xid Version. Global transaction identifier within cluster, assigned by transaction
+ * coordinator.
+ * @param nodes primary nodes registered in prepare record.
+ * @param totalNodes all nodes (primary & backup) in prepare record.
+ */
+ void onTxPrepareStart(GridCacheVersion nearXidVer, int nodes, int totalNodes) {
+ txComputeIfAbsent(nearXidVer, nodes, totalNodes);
+ }
+
+ /**
+ * @param nearXidVer near Xid Version. Global transaction identifier within cluster, assigned by transaction
+ * coordinator.
+ * @param nodes primary nodes registered in prepare record.
+ * @param totalNodes all nodes (primary & backup) in prepare record.
+ * @return tx occurrence to accumulate entries into.
+ */
+ private TxOccurrence txComputeIfAbsent(GridCacheVersion nearXidVer, int nodes, int totalNodes) {
+ TxOccurrence occurrence = opened.get(nearXidVer);
+
+ if (occurrence == null)
+ occurrence = new TxOccurrence(nodes, totalNodes);
+
+ opened.put(nearXidVer, occurrence);
+
+ return occurrence;
+ }
+
+ /**
+ * Handles commit or rollback transaction. Finished statistics accumulation.
+ *
+ * @param nearXidVer near Xid Version. Global transaction identifier within cluster, assigned by transaction
+ * coordinator.
+ * @param commit tx committed, flag indicating TX successes.
+ */
+ void onTxEnd(GridCacheVersion nearXidVer, boolean commit) {
+ TxOccurrence occurrence = opened.remove(nearXidVer);
+
+ if (occurrence == null)
+ return;
+
+ if (!commit)
+ return;
+
+ if (occurrence.nodes > 0 && occurrence.totalNodes > 0) {
+ avgPrimaryNodes.occurrence(occurrence.nodes);
+ avgTotalNodes.occurrence(occurrence.totalNodes);
+ }
+
+ incrementStat(occurrence.entriesUpdated, txEntriesUpdated);
+
+ incrementStat(occurrence.caches.size(), txCachesInvolved);
+
+ if (!occurrence.caches.isEmpty()) {
+ final String sortedCachesKey = occurrence.caches.toString();
+
+ incrementStat(sortedCachesKey, cacheIdsInTx.map, 1);
+
+ if (occurrence.nodes > 0)
+ incrementStat(sortedCachesKey, cacheIdsWeightedNodesInTx.map, occurrence.nodes);
+
+ if (occurrence.totalNodes > 0)
+ incrementStat(sortedCachesKey, cacheIdsWeightedTotalNodesInTx.map, occurrence.totalNodes);
+ }
+ }
+
+ /**
+ * Handles Data entry from data record. Entries not under transaction are ignored.
+ *
+ * @param entry object updated.
+ */
+ void onDataEntry(DataEntry entry) {
+ final GridCacheVersion ver = entry.nearXidVersion();
+
+ if (ver == null)
+ return;
+
+ txComputeIfAbsent(ver, -1, -1).onDataEntry(entry);
+ }
+
+ /**
+ * @param sb buffer.
+ * @param mapName display name of map.
+ * @param map values.
+ */
+ private void printSizeCountMap(StringBuilder sb, String mapName, Map<?, Integer> map) {
+ sb.append(mapName).append(": \n");
+ sb.append("key\tcount");
+ sb.append("\n");
+
+ final List<? extends Map.Entry<?, Integer>> entries = new ArrayList<>(map.entrySet());
+
+ Collections.sort(entries, new Comparator<Map.Entry<?, Integer>>() {
+ @Override public int compare(Map.Entry<?, Integer> o1, Map.Entry<?, Integer> o2) {
+ return -Integer.compare(o1.getValue(), o2.getValue());
+ }
+ });
+
+ int othersCnt = 0;
+ int othersSum = 0;
+ int cnt = 0;
+
+ for (Map.Entry<?, Integer> next : entries) {
+ if (cnt < WalStat.DISPLAY_MAX) {
+ sb.append(next.getKey()).append("\t").append(next.getValue()).append("\t");
+ sb.append("\n");
+ }
+ else {
+ othersCnt++;
+ othersSum += next.getValue();
+ }
+ cnt++;
+ }
+
+ if (othersCnt > 0) {
+ sb.append("... other ").append(othersCnt).append(" values").append("\t").append(othersSum).append("\t");
+ sb.append("\n");
+ }
+
+ sb.append("\n");
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ final StringBuilder sb = new StringBuilder();
+ sb.append("Tx stat: remained Opened: \t").append(opened.size()).append("\n").append("\n");
+
+ sb.append("Tx stat: Average Primary Node Count: \t").append(avgPrimaryNodes.averageStr()).append("\n").append("\n");
+
+ sb.append("Tx stat: Average Total Node Count: \t").append(avgTotalNodes.averageStr()).append("\n").append("\n");
+
+ printSizeCountMap(sb, "Tx stat: Entries updated", txEntriesUpdated);
+ printSizeCountMap(sb, "Tx stat: Caches involved", txCachesInvolved);
+ printSizeCountMap(sb, "Tx stat: Caches list in TX, evicted = " + cacheIdsInTx.evicted, cacheIdsInTx.map);
+
+ printSizeCountMap(sb, "Tx stat: Caches list in TX; weighted by primary Nodes, evicted = "
+ + cacheIdsWeightedNodesInTx.evicted, cacheIdsWeightedNodesInTx.map);
+
+ printSizeCountMap(sb, "Tx stat: Caches list in TX; weighted by total Nodes, evicted = "
+ + cacheIdsWeightedNodesInTx.evicted, cacheIdsWeightedNodesInTx.map);
+ return sb.toString();
+ }
+
+ /**
+ * Tx in prepare or in commit state, used to accumulate statistic.
+ */
+ private static class TxOccurrence {
+ /** Primary nodes count from TX record. */
+ private int nodes;
+ /** Primary + backup nodes count from TX record. */
+ private int totalNodes;
+
+ /** Count of entries updated under current transaction on current node. */
+ private int entriesUpdated;
+
+ /** Sorted set of cache IDs updated during this transaction. */
+ private TreeSet<Integer> caches = new TreeSet<>();
+
+ /**
+ * @param nodes Primary nodes count from TX record.
+ * @param totalNodes Primary + backup nodes count from TX record.
+ */
+ TxOccurrence(int nodes, int totalNodes) {
+ this.nodes = nodes;
+ this.totalNodes = totalNodes;
+ }
+
+ /**
+ * Handles data entry from data record.
+ * @param entry object updated.
+ */
+ void onDataEntry(DataEntry entry) {
+ entriesUpdated++;
+
+ caches.add(entry.cacheId());
+ }
+ }
+
+ /**
+ * @param <K> key type parameter.
+ */
+ private static class LruMap<K> {
+ /** Max size of map after which eviction may start. */
+ private int maxSize;
+
+ /**
+ * Evict prohibited boundary. If this number of usages is accumulate in eldest entry it will not be removed
+ * anyway.
+ */
+ private int evictProhibited;
+ /**
+ * Evicted count. Number of entries removed during statistic accumulation. Zero value means all records were
+ * processed, created top (popular combination search) is totally correct. Non zero means top may be not
+ * correct.
+ */
+ private int evicted;
+
+ /** Map with data. */
+ private Map<K, Integer> map = new LinkedHashMap<K, Integer>(16, 0.75F, false) {
+ @Override protected boolean removeEldestEntry(Map.Entry<K, Integer> eldest) {
+ if (size() < maxSize)
+ return false;
+
+ final boolean evictNow = eldest.getValue() < evictProhibited;
+
+ if (evictNow)
+ evicted++;
+
+ return evictNow;
+ }
+ };
+
+ /**
+ * @param maxSize Max size of map.
+ * @param evictProhibited usages count, when evict became prohibited.
+ */
+ LruMap(int maxSize, int evictProhibited) {
+ this.maxSize = maxSize;
+ this.evictProhibited = evictProhibited;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java
new file mode 100644
index 0000000..36933d8
--- /dev/null
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/WalStat.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.development.utils;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.transactions.TransactionState;
+import sun.nio.ch.DirectBuffer;
+
+/**
+ * Statistic for overall WAL file
+ */
+public class WalStat {
+ /** Display max: top lines before merging other values */
+ static final int DISPLAY_MAX = 80;
+
+ /** Tx statistics. */
+ private TxWalStat txStat = new TxWalStat();
+
+ /** Segments source: work/archive -> records loaded count & sizes. */
+ private final Map<String, RecordSizeCountStat> segmentsFolder = new TreeMap<>();
+
+ /** Segments index -> records loaded count & sizes. */
+ private final Map<String, RecordSizeCountStat> segmentsIndexes = new TreeMap<>();
+
+ /** Record type -> its count & sizes. */
+ private final Map<String, RecordSizeCountStat> recTypeSizes = new TreeMap<>();
+
+ /** Data Record: entries count */
+ private final Map<Integer, RecordSizeCountStat> dataRecordEntriesCnt = new TreeMap<>();
+
+ /** Data Record: is under TX */
+ private final Map<Boolean, RecordSizeCountStat> dataRecordUnderTx = new TreeMap<>();
+
+ /** Data Entry: operation performed */
+ private final Map<String, RecordSizeCountStat> dataEntryOperation = new TreeMap<>();
+
+ /** Data Entry: cache groups. */
+ private final Map<Integer, RecordSizeCountStat> dataEntryCacheId = new TreeMap<>();
+
+ /** Tx Record: action */
+ private final Map<String, RecordSizeCountStat> txRecordAct = new TreeMap<>();
+
+ /** Tx Record: participating primary nodes */
+ private final Map<Integer, RecordSizeCountStat> txRecordPrimNodesCnt = new TreeMap<>();
+
+ /** Tx Record: participating primary nodes */
+ private final Map<Integer, RecordSizeCountStat> txRecordNodesCnt = new TreeMap<>();
+
+ /** Page snapshot types. */
+ private final Map<String, RecordSizeCountStat> pageSnapshotTypes = new TreeMap<>();
+
+ /** Page snapshot indexes. */
+ private final Map<String, RecordSizeCountStat> pageSnapshotIndexes = new TreeMap<>();
+
+ /** Page snapshot cache groups. */
+ private final Map<Integer, RecordSizeCountStat> pageSnapshotCacheGrp = new TreeMap<>();
+
+ /** Page snapshot: partition ID. */
+ private final Map<Integer, RecordSizeCountStat> pageSnapshotPartId = new TreeMap<>();
+
+ /**
+ * @param key key (parameter) value.
+ * @param record record corresponding to {@code key}.
+ * @param map map to save statistics.
+ * @param <K> key type.
+ */
+ private static <K> void incrementStat(K key, WALRecord record, Map<K, RecordSizeCountStat> map) {
+ incrementStat(key, map, record.size());
+ }
+
+ /**
+ * @param key key (parameter) value.
+ * @param map to save statistics.
+ * @param size record size for record corresponding to {@code key}. Negative value of size means size is unknown for current row.
+ * @param <K> key type.
+ */
+ private static <K> void incrementStat(K key, Map<K, RecordSizeCountStat> map, int size) {
+ final RecordSizeCountStat val = map.get(key);
+ final RecordSizeCountStat recordStat = val == null ? new RecordSizeCountStat() : val;
+ recordStat.occurrence(size);
+ map.put(key, recordStat);
+ }
+
+ /**
+ * Handles WAL record.
+ *
+ * @param record record to handle.
+ * @param walPointer pointer, used to extract segment index.
+ * @param workDir true for work, false for archive folder.
+ */
+ void registerRecord(WALRecord record, WALPointer walPointer, boolean workDir) {
+ WALRecord.RecordType type = record.type();
+
+ if (type == WALRecord.RecordType.PAGE_RECORD)
+ registerPageSnapshot((PageSnapshot)record);
+ else if (type == WALRecord.RecordType.DATA_RECORD)
+ registerDataRecord((DataRecord)record);
+ else if (type == WALRecord.RecordType.TX_RECORD)
+ registerTxRecord((TxRecord)record);
+
+ incrementStat(type.toString(), record, recTypeSizes);
+
+ if (walPointer instanceof FileWALPointer) {
+ final FileWALPointer fPtr = (FileWALPointer)walPointer;
+
+ incrementStat(Long.toString(fPtr.index()), record, segmentsIndexes);
+ incrementStat(workDir ? "work" : "archive", record, segmentsFolder);
+ }
+ }
+
+ /**
+ * @param txRecord TX record to handle.
+ */
+ private void registerTxRecord(TxRecord txRecord) {
+ final TransactionState state = txRecord.state();
+
+ incrementStat(state.toString(), txRecord, txRecordAct);
+
+ int totalNodes = 0;
+ final Map<Short, Collection<Short>> map = txRecord.participatingNodes();
+
+ if (map != null) {
+ incrementStat(map.size(), txRecord, txRecordPrimNodesCnt);
+
+ final HashSet<Object> set = new HashSet<>(150);
+
+ for (Map.Entry<Short, Collection<Short>> next : map.entrySet()) {
+ set.add(next.getKey());
+ set.addAll(next.getValue());
+ }
+
+ totalNodes = set.size();
+
+ incrementStat(totalNodes, txRecord, txRecordNodesCnt);
+ }
+
+ final GridCacheVersion ver = txRecord.nearXidVersion();
+ if (ver != null) {
+ switch (state) {
+ case PREPARING:
+ case PREPARED:
+ txStat.onTxPrepareStart(ver, map != null ? map.size() : 0, totalNodes);
+ break;
+ case COMMITTED:
+ txStat.onTxEnd(ver, true);
+ break;
+ default:
+ txStat.onTxEnd(ver, false);
+ }
+ }
+ }
+
+ /**
+ * @param record page snapshot record to handle.
+ */
+ private void registerPageSnapshot(PageSnapshot record) {
+ FullPageId fullPageId = record.fullPageId();
+ long pageId = fullPageId.pageId();
+
+ incrementStat(getPageType(record), record, pageSnapshotTypes);
+
+ final int idx = PageIdUtils.pageIndex(pageId);
+ final String idxAsStr = idx <= 100 ? Integer.toString(idx) : ">100";
+
+ incrementStat(idxAsStr, record, pageSnapshotIndexes);
+ incrementStat(fullPageId.groupId(), record, pageSnapshotCacheGrp);
+ incrementStat(PageIdUtils.partId(pageId), record, pageSnapshotPartId);
+ }
+
+ /**
+ * @param record data record to handle.
+ */
+ private void registerDataRecord(DataRecord record) {
+ final List<DataEntry> dataEntries = record.writeEntries();
+ if (!dataEntries.isEmpty()) {
+ boolean underTx = false;
+ for (DataEntry next : dataEntries) {
+ final int size = dataEntries.size() > 1 ? -1 : record.size();
+ incrementStat(next.op().toString(), dataEntryOperation, size);
+
+ incrementStat(next.cacheId(), dataEntryCacheId, size);
+
+ txStat.onDataEntry(next);
+
+ underTx |= next.nearXidVersion() != null;
+ }
+
+ incrementStat(underTx, record, dataRecordUnderTx);
+ }
+
+ incrementStat(dataEntries.size(), record, dataRecordEntriesCnt);
+ }
+
+ /**
+ * @param record page snapshot record.
+ * @return string identifier of page (IO) type.
+ */
+ private static String getPageType(PageSnapshot record) {
+ byte[] pageData = record.pageData();
+ ByteBuffer buf = ByteBuffer.allocateDirect(pageData.length);
+
+ try {
+ buf.order(ByteOrder.nativeOrder());
+ buf.put(pageData);
+
+ long addr = GridUnsafe.bufferAddress(buf);
+
+ int type = PageIO.getType(addr);
+ int ver = PageIO.getVersion(addr);
+
+ return PageIO.getPageIO(type, ver).getClass().getSimpleName();
+ }
+ catch (IgniteCheckedException ignored) {
+ }
+ finally {
+ ((DirectBuffer)buf).cleaner().clean();
+ }
+ return "";
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ final StringBuilder sb = new StringBuilder();
+
+ printSizeCountMap(sb, "WAL Segments: Source folder", segmentsFolder);
+ printSizeCountMap(sb, "WAL Segments: File index", segmentsIndexes);
+
+ printSizeCountMap(sb, "Record type", recTypeSizes);
+
+ printSizeCountMap(sb, "Tx Record: Action", txRecordAct);
+ printSizeCountMap(sb, "Tx Record: Primary nodes count", txRecordPrimNodesCnt);
+ printSizeCountMap(sb, "Tx Record: Nodes count", txRecordNodesCnt);
+
+ printSizeCountMap(sb, "Data Record: Entries count", dataRecordEntriesCnt);
+ printSizeCountMap(sb, "Data Record: Under TX", dataRecordUnderTx);
+
+ printSizeCountMap(sb, "Data Entry: Operations", dataEntryOperation);
+ printSizeCountMap(sb, "Data Entry: Cache ID", dataEntryCacheId);
+
+ printSizeCountMap(sb, "Page Snapshot: Page Types", pageSnapshotTypes);
+ printSizeCountMap(sb, "Page Snapshot: Indexes", pageSnapshotIndexes);
+ printSizeCountMap(sb, "Page Snapshot: Cache Groups", pageSnapshotCacheGrp);
+ printSizeCountMap(sb, "Page Snapshot: Partition ID", pageSnapshotPartId);
+
+ sb.append(txStat.toString());
+
+ return sb.toString();
+ }
+
+ /**
+ * @param sb buffer.
+ * @param mapName display name of map.
+ * @param map data.
+ */
+ private void printSizeCountMap(StringBuilder sb, String mapName, Map<?, RecordSizeCountStat> map) {
+ sb.append(mapName).append(": \n");
+ sb.append("key\tsize\tcount\tavg.size");
+ sb.append("\n");
+
+ final List<? extends Map.Entry<?, RecordSizeCountStat>> entries = new ArrayList<>(map.entrySet());
+
+ Collections.sort(entries, new Comparator<Map.Entry<?, RecordSizeCountStat>>() {
+ @Override public int compare(Map.Entry<?, RecordSizeCountStat> o1, Map.Entry<?, RecordSizeCountStat> o2) {
+ return -Integer.compare(o1.getValue().getCount(), o2.getValue().getCount());
+ }
+ });
+
+ RecordSizeCountStat others = new RecordSizeCountStat();
+ int otherRecords = 0;
+ int cnt = 0;
+
+ for (Map.Entry<?, RecordSizeCountStat> next : entries) {
+ if (cnt < DISPLAY_MAX) {
+ sb.append(next.getKey()).append("\t").append(next.getValue()).append("\t");
+ sb.append("\n");
+ }
+ else {
+ otherRecords++;
+ others.add(next.getValue());
+ }
+ cnt++;
+ }
+
+ if (otherRecords > 0) {
+ sb.append("... other ").append(otherRecords).append(" values").append("\t").append(others).append("\t");
+ sb.append("\n");
+ }
+
+ sb.append("\n");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
index c803a36..c4a8c08 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
@@ -458,8 +458,6 @@ public class GridH2Table extends TableBase {
replaced = prevRow0 != null;
}
- assert (replaced && prevRow0 != null) || (!replaced && prevRow0 == null) : "Replaced: " + replaced;
-
if (!replaced)
size.increment();
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryObjectFieldsQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryObjectFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryObjectFieldsQuerySelfTest.java
index 6c34373..62515c3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryObjectFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteBinaryObjectFieldsQuerySelfTest.java
@@ -103,11 +103,16 @@ public class IgniteBinaryObjectFieldsQuerySelfTest extends GridCommonAbstractTes
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
- extClassLoader = getExternalClassLoader();
+ initExtClassLoader();
startGrids(GRID_CNT);
}
+ /** */
+ protected void initExtClassLoader() {
+ extClassLoader = getExternalClassLoader();
+ }
+
/** {@inheritDoc} */
@Override protected void afterTestsStopped() throws Exception {
stopAllGrids();
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
index 83f5015..9af75ed 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
@@ -198,7 +198,7 @@ public class IgniteCacheLockPartitionOnAffinityRunAbstractTest extends GridCache
/**
* @throws Exception If failed.
*/
- private void fillCaches() throws Exception {
+ protected void fillCaches() throws Exception {
grid(0).createCache(Organization.class.getSimpleName());
grid(0).createCache(Person.class.getSimpleName());
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
index 71e737f..a4f398f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.java
@@ -94,7 +94,7 @@ public class IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest extends Igni
* @param mode Atomicity mode.
* @throws Exception If failed.
*/
- private void createCache(String cacheName, CacheAtomicityMode mode) throws Exception {
+ protected void createCache(String cacheName, CacheAtomicityMode mode) throws Exception {
CacheConfiguration ccfg = cacheConfiguration(grid(0).name());
ccfg.setName(cacheName);
@@ -109,17 +109,27 @@ public class IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest extends Igni
@Override protected void beforeTest() throws Exception {
super.beforeTest();
+ createCaches();
+
+ awaitPartitionMapExchange();
+ }
+
+ /** */
+ protected void createCaches() throws Exception {
key.set(0);
createCache(ATOMIC_CACHE, CacheAtomicityMode.ATOMIC);
createCache(TRANSACT_CACHE, CacheAtomicityMode.TRANSACTIONAL);
+ }
- awaitPartitionMapExchange();
+ /** */
+ protected void destroyCaches() throws Exception {
+ grid(0).destroyCache(ATOMIC_CACHE);
+ grid(0).destroyCache(TRANSACT_CACHE);
}
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
- grid(0).destroyCache(ATOMIC_CACHE);
- grid(0).destroyCache(TRANSACT_CACHE);
+ destroyCaches();
super.afterTest();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
index 0c644e3..fc1cea6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
@@ -105,7 +105,6 @@ public class IgniteCacheQueryNodeRestartSelfTest extends GridCacheAbstractSelfTe
public void testRestarts() throws Exception {
int duration = 60 * 1000;
int qryThreadNum = 10;
- final long nodeLifeTime = 2 * 1000;
final int logFreq = 50;
final IgniteCache<Integer, Integer> cache = grid(0).cache(DEFAULT_CACHE_NAME);
@@ -158,27 +157,7 @@ public class IgniteCacheQueryNodeRestartSelfTest extends GridCacheAbstractSelfTe
for (int i = 0; i < GRID_CNT; i++)
grid(i).events().localListen(lsnr, EventType.EVT_CACHE_REBALANCE_STOPPED);
- IgniteInternalFuture<?> fut2 = multithreadedAsync(new Callable<Object>() {
- @SuppressWarnings({"BusyWait"})
- @Override public Object call() throws Exception {
- while (!done.get()) {
- int idx = GRID_CNT;
-
- startGrid(idx);
-
- Thread.sleep(nodeLifeTime);
-
- stopGrid(idx);
-
- int c = restartCnt.incrementAndGet();
-
- if (c % logFreq == 0)
- info("Node restarts: " + c);
- }
-
- return true;
- }
- }, 1, "restart-thread");
+ IgniteInternalFuture<?> fut2 = createRestartAction(done, restartCnt);
Thread.sleep(duration);
@@ -204,6 +183,39 @@ public class IgniteCacheQueryNodeRestartSelfTest extends GridCacheAbstractSelfTe
assert success;
}
+ /**
+ *
+ */
+ protected IgniteInternalFuture createRestartAction(final AtomicBoolean done, final AtomicInteger restartCnt) throws Exception {
+ return multithreadedAsync(new Callable<Object>() {
+ /** */
+ private final long nodeLifeTime = 2 * 1000;
+
+ /** */
+ private final int logFreq = 50;
+
+ @SuppressWarnings({"BusyWait"})
+ @Override public Object call() throws Exception {
+ while (!done.get()) {
+ int idx = GRID_CNT;
+
+ startGrid(idx);
+
+ Thread.sleep(nodeLifeTime);
+
+ stopGrid(idx);
+
+ int c = restartCnt.incrementAndGet();
+
+ if (c % logFreq == 0)
+ info("Node restarts: " + c);
+ }
+
+ return true;
+ }
+ }, 1, "restart-thread");
+ }
+
/** Listener that will wait for specified number of events received. */
private class CollectingEventListener implements IgnitePredicate<Event> {
/** Registered events count. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java
new file mode 100644
index 0000000..c7663ea
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.database.baseline;
+
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest extends IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setInitialSize(200 * 1024 * 1024)
+ .setMaxSize(200 * 1024 * 1024)
+ .setPersistenceEnabled(true)
+ )
+ );
+
+ return cfg;
+ }
+
+ @Override protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throws Exception {
+ CacheConfiguration ccfg = super.cacheConfiguration(igniteInstanceName);
+
+ ccfg.setBackups(2);
+
+ return ccfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ int gridCnt = gridCount();
+
+ startGrids(gridCnt + 1);
+
+ grid(0).active(true);
+
+ stopGrid(gridCnt);
+
+ startGrid(gridCnt + 1);
+
+ fillCaches();
+
+ createCaches();
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ destroyCaches();
+
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java
new file mode 100644
index 0000000..8812829
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.database.baseline;
+
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteBaselineLockPartitionOnAffinityRunTxCacheTest extends IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setInitialSize(200 * 1024 * 1024)
+ .setMaxSize(200 * 1024 * 1024)
+ .setPersistenceEnabled(true)
+ )
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ int gridCnt = gridCount();
+
+ startGrids(gridCnt + 1);
+
+ grid(0).active(true);
+
+ stopGrid(gridCnt);
+
+ startGrid(gridCnt + 1);
+
+ fillCaches();
+
+ createCaches();
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ destroyCaches();
+
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
new file mode 100644
index 0000000..357d8e6
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.database.baseline;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNodeRestartSelfTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteChangingBaselineCacheQueryNodeRestartSelfTest extends IgniteCacheQueryNodeRestartSelfTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(200 * 1024 * 1024)
+ .setPersistenceEnabled(true)
+ )
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(gridCount());
+
+ initStoreStrategy();
+
+ grid(0).active(true);
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteInternalFuture createRestartAction(final AtomicBoolean done, final AtomicInteger restartCnt) throws Exception {
+ return multithreadedAsync(new Callable<Object>() {
+ /** */
+ private final long baselineTopChangeInterval = 30 * 1000;
+
+ /** */
+ private final int logFreq = 50;
+
+ /** flag to indicate that last operation was changing BaselineTopology up (add node) */
+ private boolean lastOpChangeUp;
+
+ @SuppressWarnings({"BusyWait"})
+ @Override public Object call() throws Exception {
+ while (!done.get()) {
+ if (lastOpChangeUp) {
+ //need to do change down: stop node, set new BLT without it
+ stopGrid(gridCount());
+
+ lastOpChangeUp = false;
+ }
+ else {
+ startGrid(gridCount());
+
+ lastOpChangeUp = true;
+ }
+
+ grid(0).cluster().setBaselineTopology(baselineNodes(grid(0).cluster().forServers().nodes()));
+
+ Thread.sleep(baselineTopChangeInterval);
+
+ int c = restartCnt.incrementAndGet();
+
+ if (c % logFreq == 0)
+ info("BaselineTopology changes: " + c);
+ }
+
+ return true;
+ }
+ }, 1, "restart-thread");
+ }
+
+ /** */
+ private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+ Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+ for (ClusterNode clN : clNodes)
+ res.add(clN);
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineBinObjFieldsQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineBinObjFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineBinObjFieldsQuerySelfTest.java
new file mode 100644
index 0000000..e9d1b79
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineBinObjFieldsQuerySelfTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.database.baseline;
+
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.IgniteBinaryObjectFieldsQuerySelfTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteStableBaselineBinObjFieldsQuerySelfTest extends IgniteBinaryObjectFieldsQuerySelfTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(200 * 1024 * 1024)
+ .setPersistenceEnabled(true)
+ )
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ initExtClassLoader();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(GRID_CNT + 1);
+
+ grid(0).active(true);
+
+ stopGrid(GRID_CNT);
+
+ startGrid(GRID_CNT + 1);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ GridTestUtils.deleteDbFiles();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java
new file mode 100644
index 0000000..849bbb6
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteStableBaselineCacheQueryNodeRestartsSelfTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.database.baseline;
+
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNodeRestartSelfTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteStableBaselineCacheQueryNodeRestartsSelfTest extends IgniteCacheQueryNodeRestartSelfTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(200 * 1024 * 1024)
+ .setPersistenceEnabled(true)
+ )
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(gridCount() + 1);
+
+ initStoreStrategy();
+
+ grid(0).active(true);
+
+ stopGrid(gridCount());
+
+ startGrid(gridCount() + 1);
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheAffinityRunTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheAffinityRunTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheAffinityRunTestSuite.java
index ef00fc3..e9c7b79 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheAffinityRunTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheAffinityRunTestSuite.java
@@ -22,6 +22,8 @@ import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAff
import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAffinityRunTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest;
import org.apache.ignite.internal.processors.cache.IgniteCacheLockPartitionOnAffinityRunWithCollisionSpiTest;
+import org.apache.ignite.internal.processors.database.baseline.IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest;
+import org.apache.ignite.internal.processors.database.baseline.IgniteBaselineLockPartitionOnAffinityRunTxCacheTest;
/**
* Compute and Cache tests for affinityRun/Call. These tests is extracted into separate suite
@@ -38,6 +40,8 @@ public class IgniteCacheAffinityRunTestSuite extends TestSuite {
suite.addTestSuite(IgniteCacheLockPartitionOnAffinityRunTest.class);
suite.addTestSuite(IgniteCacheLockPartitionOnAffinityRunWithCollisionSpiTest.class);
suite.addTestSuite(IgniteCacheLockPartitionOnAffinityRunAtomicCacheOpTest.class);
+ suite.addTestSuite(IgniteBaselineLockPartitionOnAffinityRunAtomicCacheTest.class);
+ suite.addTestSuite(IgniteBaselineLockPartitionOnAffinityRunTxCacheTest.class);
suite.addTestSuite(IgniteCacheLockPartitionOnAffinityRunTxCacheOpTest.class);
return suite;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index cf5c9a6..8cfa14e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -136,6 +136,7 @@ import org.apache.ignite.internal.processors.cache.query.IgniteCacheQueryCacheDe
import org.apache.ignite.internal.processors.cache.query.IndexingSpiQuerySelfTest;
import org.apache.ignite.internal.processors.cache.query.IndexingSpiQueryTxSelfTest;
import org.apache.ignite.internal.processors.client.ClientConnectorConfigurationValidationSelfTest;
+import org.apache.ignite.internal.processors.database.baseline.IgniteStableBaselineBinObjFieldsQuerySelfTest;
import org.apache.ignite.internal.processors.query.IgniteSqlDefaultValueTest;
import org.apache.ignite.internal.processors.query.IgniteSqlDistributedJoinSelfTest;
import org.apache.ignite.internal.processors.query.IgniteSqlSkipReducerOnUpdateDmlFlagSelfTest;
@@ -248,6 +249,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
suite.addTestSuite(GridCacheCrossCacheQuerySelfTest.class);
suite.addTestSuite(GridCacheQuerySerializationSelfTest.class);
suite.addTestSuite(IgniteBinaryObjectFieldsQuerySelfTest.class);
+ suite.addTestSuite(IgniteStableBaselineBinObjFieldsQuerySelfTest.class);
suite.addTestSuite(IgniteBinaryWrappedObjectFieldsQuerySelfTest.class);
suite.addTestSuite(IgniteCacheQueryH2IndexingLeakTest.class);
suite.addTestSuite(IgniteCacheQueryNoRebalanceSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index abe06ec..11d98e2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -42,6 +42,8 @@ import org.apache.ignite.internal.processors.cache.index.DynamicIndexPartitioned
import org.apache.ignite.internal.processors.cache.index.DynamicIndexReplicatedAtomicConcurrentSelfTest;
import org.apache.ignite.internal.processors.cache.index.DynamicIndexReplicatedTransactionalConcurrentSelfTest;
import org.apache.ignite.internal.processors.cache.query.ScanQueryOffheapExpiryPolicySelfTest;
+import org.apache.ignite.internal.processors.database.baseline.IgniteChangingBaselineCacheQueryNodeRestartSelfTest;
+import org.apache.ignite.internal.processors.database.baseline.IgniteStableBaselineCacheQueryNodeRestartsSelfTest;
import org.apache.ignite.internal.processors.query.IgniteCacheGroupsCompareQueryTest;
import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlDistributedJoinSelfTest;
import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlSegmentedIndexMultiNodeSelfTest;
@@ -85,6 +87,8 @@ public class IgniteCacheQuerySelfTestSuite2 extends TestSuite {
suite.addTestSuite(IgniteCacheClientQueryReplicatedNodeRestartSelfTest.class);
suite.addTestSuite(IgniteCacheQueryNodeFailTest.class);
suite.addTestSuite(IgniteCacheQueryNodeRestartSelfTest.class);
+ suite.addTestSuite(IgniteChangingBaselineCacheQueryNodeRestartSelfTest.class);
+ suite.addTestSuite(IgniteStableBaselineCacheQueryNodeRestartsSelfTest.class);
suite.addTestSuite(IgniteCacheQueryNodeRestartSelfTest2.class);
suite.addTestSuite(IgniteCacheQueryNodeRestartTxSelfTest.class);
suite.addTestSuite(IgniteCacheSqlQueryMultiThreadedSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
index b940d09..4b4279f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
@@ -38,7 +38,11 @@
/** Members that are missing on .NET side and should be added in future. */
private static readonly string[] MissingMembers =
{
- "enableStatistics" // IGNITE-7276
+ "enableStatistics", // IGNITE-7276
+
+ // IGNITE-7301
+ "active",
+ "setBaselineTopology"
};
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
index 235b177..410d911 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
@@ -78,7 +78,8 @@ namespace Apache.Ignite.Core.Tests.ApiParity
"isCacheSanityCheckEnabled",
"TimeServerPortBase",
"TimeServerPortRange",
- "IncludeProperties"
+ "IncludeProperties",
+ "isAutoActivationEnabled" // IGNITE-7301
};
/// <summary>
[10/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
index 9ed4000..c2cc171 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
@@ -17,12 +17,16 @@
package org.apache.ignite.internal.processors.cache.persistence.partstate;
+import java.util.HashSet;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Set;
import java.util.TreeMap;
import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -32,8 +36,13 @@ import org.jetbrains.annotations.Nullable;
*/
public class PartitionAllocationMap {
/** Maps following pairs: (groupId, partId) -> (lastAllocatedCount, allocatedCount) */
+ @GridToStringInclude
private final NavigableMap<GroupPartitionId, PagesAllocationRange> map = new TreeMap<>();
+ /** Partitions forced to be skipped. */
+ @GridToStringInclude
+ private final Set<GroupPartitionId> skippedParts = new HashSet<>();
+
/**
* Returns the value to which the specified key is mapped,
* or {@code null} if this map contains no mapping for the key.
@@ -46,6 +55,13 @@ public class PartitionAllocationMap {
}
/**
+ * @param fullPageId Full page id.
+ */
+ @Nullable public PagesAllocationRange get(FullPageId fullPageId) {
+ return map.get(createCachePartId(fullPageId));
+ }
+
+ /**
* Extracts partition information from full page ID
*
* @param fullId page related to some cache
@@ -81,6 +97,20 @@ public class PartitionAllocationMap {
}
/**
+ * Forces the index partition for the given group ID to be skipped in collected map.
+ *
+ * @param grpId Group ID to skip.
+ * @return {@code true} if skipped partition was added to the ignore list during this call.
+ */
+ public boolean forceSkipIndexPartition(int grpId) {
+ GroupPartitionId idxId = new GroupPartitionId(grpId, PageIdAllocator.INDEX_PARTITION);
+
+ map.remove(idxId);
+
+ return skippedParts.add(idxId);
+ }
+
+ /**
* Returns next (higher) key for provided cache and partition or null
*
* @param key cache and partition to search
@@ -107,7 +137,11 @@ public class PartitionAllocationMap {
* <tt>key</tt>.
*/
public PagesAllocationRange put(GroupPartitionId key, PagesAllocationRange val) {
- return map.put(key, val);
+ return !skippedParts.contains(key) ? map.put(key, val) : null;
}
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(PartitionAllocationMap.class, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
index 45c0b11..c23adda 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
@@ -82,6 +82,13 @@ public class IgniteCacheSnapshotManager<T extends SnapshotOperation> extends Gri
/**
*
*/
+ public boolean partitionsAreFrozen(CacheGroupContext grp) {
+ return false;
+ }
+
+ /**
+ *
+ */
public void restoreState() throws IgniteCheckedException {
// No-op.
}
@@ -170,4 +177,11 @@ public class IgniteCacheSnapshotManager<T extends SnapshotOperation> extends Gri
@Override public void onDeActivate(GridKernalContext kctx) {
// No-op.
}
+
+ /**
+ * @return {@code True} if TX READ records must be logged in WAL.
+ */
+ public boolean needTxReadLogging() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java
index d88d96e..bcad7df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDiscoveryMessage.java
@@ -29,5 +29,10 @@ public interface SnapshotDiscoveryMessage extends DiscoveryCustomMessage {
*
* @return True if exchange is needed, false in other case.
*/
- boolean needExchange();
+ public boolean needExchange();
+
+ /**
+ *
+ */
+ public boolean needAssignPartitions();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
index cb6898a..6b87d32 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -3706,6 +3706,7 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
* @param pageAddr Page address.
* @param walPlc Full page WAL record policy.
* @param io IO.
+ *
* @param cnt Count.
* @param idx Index to remove.
* @throws IgniteCheckedException If failed.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
new file mode 100644
index 0000000..ca63f27
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
@@ -0,0 +1,1256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.persistence.Storable;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.GridStringBuilder;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Data pages IO.
+ */
+public abstract class AbstractDataPageIO<T extends Storable> extends PageIO {
+
+ /** */
+ private static final int SHOW_ITEM = 0b0001;
+
+ /** */
+ private static final int SHOW_PAYLOAD_LEN = 0b0010;
+
+ /** */
+ private static final int SHOW_LINK = 0b0100;
+
+ /** */
+ private static final int FREE_LIST_PAGE_ID_OFF = COMMON_HEADER_END;
+
+ /** */
+ private static final int FREE_SPACE_OFF = FREE_LIST_PAGE_ID_OFF + 8;
+
+ /** */
+ private static final int DIRECT_CNT_OFF = FREE_SPACE_OFF + 2;
+
+ /** */
+ private static final int INDIRECT_CNT_OFF = DIRECT_CNT_OFF + 1;
+
+ /** */
+ private static final int FIRST_ENTRY_OFF = INDIRECT_CNT_OFF + 1;
+
+ /** */
+ public static final int ITEMS_OFF = FIRST_ENTRY_OFF + 2;
+
+ /** */
+ private static final int ITEM_SIZE = 2;
+
+ /** */
+ private static final int PAYLOAD_LEN_SIZE = 2;
+
+ /** */
+ private static final int LINK_SIZE = 8;
+
+ /** */
+ private static final int FRAGMENTED_FLAG = 0b10000000_00000000;
+
+ /** */
+ public static final int MIN_DATA_PAGE_OVERHEAD = ITEMS_OFF + ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
+
+ /**
+ * @param ver Page format version.
+ */
+ protected AbstractDataPageIO(int type, int ver) {
+ super(type, ver);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ super.initNewPage(pageAddr, pageId, pageSize);
+
+ setEmptyPage(pageAddr, pageSize);
+ setFreeListPageId(pageAddr, 0L);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageSize Page size.
+ */
+ private void setEmptyPage(long pageAddr, int pageSize) {
+ setDirectCount(pageAddr, 0);
+ setIndirectCount(pageAddr, 0);
+ setFirstEntryOffset(pageAddr, pageSize, pageSize);
+ setRealFreeSpace(pageAddr, pageSize - ITEMS_OFF, pageSize);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param freeListPageId Free list page ID.
+ */
+ public void setFreeListPageId(long pageAddr, long freeListPageId) {
+ PageUtils.putLong(pageAddr, FREE_LIST_PAGE_ID_OFF, freeListPageId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Free list page ID.
+ */
+ public long getFreeListPageId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, FREE_LIST_PAGE_ID_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Data offset.
+ * @param show What elements of data page entry to show in the result.
+ * @return Data page entry size.
+ */
+ private int getPageEntrySize(long pageAddr, int dataOff, int show) {
+ int payloadLen = PageUtils.getShort(pageAddr, dataOff) & 0xFFFF;
+
+ if ((payloadLen & FRAGMENTED_FLAG) != 0)
+ payloadLen &= ~FRAGMENTED_FLAG; // We are fragmented and have a link.
+ else
+ show &= ~SHOW_LINK; // We are not fragmented, never have a link.
+
+ return getPageEntrySize(payloadLen, show);
+ }
+
+ /**
+ * @param payloadLen Length of the payload, may be a full data row or a row fragment length.
+ * @param show What elements of data page entry to show in the result.
+ * @return Data page entry size.
+ */
+ private int getPageEntrySize(int payloadLen, int show) {
+ assert payloadLen > 0 : payloadLen;
+
+ int res = payloadLen;
+
+ if ((show & SHOW_LINK) != 0)
+ res += LINK_SIZE;
+
+ if ((show & SHOW_ITEM) != 0)
+ res += ITEM_SIZE;
+
+ if ((show & SHOW_PAYLOAD_LEN) != 0)
+ res += PAYLOAD_LEN_SIZE;
+
+ return res;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Entry data offset.
+ * @param pageSize Page size.
+ */
+ private void setFirstEntryOffset(long pageAddr, int dataOff, int pageSize) {
+ assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff <= pageSize : dataOff;
+
+ PageUtils.putShort(pageAddr, FIRST_ENTRY_OFF, (short)dataOff);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Entry data offset.
+ */
+ private int getFirstEntryOffset(long pageAddr) {
+ return PageUtils.getShort(pageAddr, FIRST_ENTRY_OFF) & 0xFFFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param freeSpace Free space.
+ * @param pageSize Page size.
+ */
+ private void setRealFreeSpace(long pageAddr, int freeSpace, int pageSize) {
+ assert freeSpace == actualFreeSpace(pageAddr, pageSize) : freeSpace + " != " + actualFreeSpace(pageAddr, pageSize);
+
+ PageUtils.putShort(pageAddr, FREE_SPACE_OFF, (short)freeSpace);
+ }
+
+ /**
+ * Free space refers to a "max row size (without any data page specific overhead) which is guaranteed to fit into
+ * this data page".
+ *
+ * @param pageAddr Page address.
+ * @return Free space.
+ */
+ public int getFreeSpace(long pageAddr) {
+ if (getFreeItemSlots(pageAddr) == 0)
+ return 0;
+
+ int freeSpace = getRealFreeSpace(pageAddr);
+
+ // We reserve size here because of getFreeSpace() method semantics (see method javadoc).
+ // It means that we must be able to accommodate a row of size which is equal to getFreeSpace(),
+ // plus we will have data page overhead: header of the page as well as item, payload length and
+ // possibly a link to the next row fragment.
+ freeSpace -= ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
+
+ return freeSpace < 0 ? 0 : freeSpace;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return {@code true} If there is no useful data in this page.
+ */
+ public boolean isEmpty(long pageAddr) {
+ return getDirectCount(pageAddr) == 0;
+ }
+
+ /**
+ * Equivalent for {@link #actualFreeSpace(long, int)} but reads saved value.
+ *
+ * @param pageAddr Page address.
+ * @return Free space.
+ */
+ private int getRealFreeSpace(long pageAddr) {
+ return PageUtils.getShort(pageAddr, FREE_SPACE_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param cnt Direct count.
+ */
+ private void setDirectCount(long pageAddr, int cnt) {
+ assert checkCount(cnt) : cnt;
+
+ PageUtils.putByte(pageAddr, DIRECT_CNT_OFF, (byte)cnt);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Direct count.
+ */
+ private int getDirectCount(long pageAddr) {
+ return PageUtils.getByte(pageAddr, DIRECT_CNT_OFF) & 0xFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param c Closure.
+ * @param <T> Closure return type.
+ * @return Collection of closure results for all items in page.
+ * @throws IgniteCheckedException In case of error in closure body.
+ */
+ public <T> List<T> forAllItems(long pageAddr, CC<T> c) throws IgniteCheckedException {
+ long pageId = getPageId(pageAddr);
+
+ int cnt = getDirectCount(pageAddr);
+
+ List<T> res = new ArrayList<>(cnt);
+
+ for (int i = 0; i < cnt; i++) {
+ long link = PageIdUtils.link(pageId, i);
+
+ res.add(c.apply(link));
+ }
+
+ return res;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param cnt Indirect count.
+ */
+ private void setIndirectCount(long pageAddr, int cnt) {
+ assert checkCount(cnt) : cnt;
+
+ PageUtils.putByte(pageAddr, INDIRECT_CNT_OFF, (byte)cnt);
+ }
+
+ /**
+ * @param idx Index.
+ * @return {@code true} If the index is valid.
+ */
+ protected boolean checkIndex(int idx) {
+ return idx >= 0 && idx < 0xFF;
+ }
+
+ /**
+ * @param cnt Counter value.
+ * @return {@code true} If the counter fits 1 byte.
+ */
+ private boolean checkCount(int cnt) {
+ return cnt >= 0 && cnt <= 0xFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Indirect count.
+ */
+ private int getIndirectCount(long pageAddr) {
+ return PageUtils.getByte(pageAddr, INDIRECT_CNT_OFF) & 0xFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Number of free entry slots.
+ */
+ private int getFreeItemSlots(long pageAddr) {
+ return 0xFF - getDirectCount(pageAddr);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @return Found index of indirect item.
+ */
+ private int findIndirectItemIndex(long pageAddr, int itemId, int directCnt, int indirectCnt) {
+ int low = directCnt;
+ int high = directCnt + indirectCnt - 1;
+
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+
+ int cmp = Integer.compare(itemId(getItem(pageAddr, mid)), itemId);
+
+ if (cmp < 0)
+ low = mid + 1;
+ else if (cmp > 0)
+ high = mid - 1;
+ else
+ return mid; // found
+ }
+
+ throw new IllegalStateException("Item not found: " + itemId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageSize Page size.
+ * @return String representation.
+ */
+ private String printPageLayout(long pageAddr, int pageSize) {
+ SB b = new SB();
+
+ printPageLayout(pageAddr, pageSize, b);
+
+ return b.toString();
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageSize Page size.
+ * @param b B.
+ */
+ protected void printPageLayout(long pageAddr, int pageSize, GridStringBuilder b) {
+ int directCnt = getDirectCount(pageAddr);
+ int indirectCnt = getIndirectCount(pageAddr);
+ int free = getRealFreeSpace(pageAddr);
+
+ boolean valid = directCnt >= indirectCnt;
+
+ b.appendHex(PageIO.getPageId(pageAddr)).a(" [");
+
+ int entriesSize = 0;
+
+ for (int i = 0; i < directCnt; i++) {
+ if (i != 0)
+ b.a(", ");
+
+ short item = getItem(pageAddr, i);
+
+ if (item < ITEMS_OFF || item >= pageSize)
+ valid = false;
+
+ entriesSize += getPageEntrySize(pageAddr, item, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+ b.a(item);
+ }
+
+ b.a("][");
+
+ Collection<Integer> set = new HashSet<>();
+
+ for (int i = directCnt; i < directCnt + indirectCnt; i++) {
+ if (i != directCnt)
+ b.a(", ");
+
+ short item = getItem(pageAddr, i);
+
+ int itemId = itemId(item);
+ int directIdx = directItemIndex(item);
+
+ if (!set.add(directIdx) || !set.add(itemId))
+ valid = false;
+
+ assert indirectItem(itemId, directIdx) == item;
+
+ if (itemId < directCnt || directIdx < 0 || directIdx >= directCnt)
+ valid = false;
+
+ if (i > directCnt && itemId(getItem(pageAddr, i - 1)) >= itemId)
+ valid = false;
+
+
+ b.a(itemId).a('^').a(directIdx);
+ }
+
+ b.a("][free=").a(free);
+
+ int actualFree = pageSize - ITEMS_OFF - (entriesSize + (directCnt + indirectCnt) * ITEM_SIZE);
+
+ if (free != actualFree) {
+ b.a(", actualFree=").a(actualFree);
+
+ valid = false;
+ }
+ else
+ b.a("]");
+
+ assert valid : b.toString();
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+ * @param pageSize Page size.
+ * @return Data entry offset in bytes.
+ */
+ protected int getDataOffset(long pageAddr, int itemId, int pageSize) {
+ assert checkIndex(itemId) : itemId;
+
+ int directCnt = getDirectCount(pageAddr);
+
+ assert directCnt > 0 : "itemId=" + itemId + ", directCnt=" + directCnt + ", page=" + printPageLayout(pageAddr, pageSize);
+
+ if (itemId >= directCnt) { // Need to do indirect lookup.
+ int indirectCnt = getIndirectCount(pageAddr);
+
+ // Must have indirect items here.
+ assert indirectCnt > 0 : "itemId=" + itemId + ", directCnt=" + directCnt + ", indirectCnt=" + indirectCnt +
+ ", page=" + printPageLayout(pageAddr, pageSize);
+
+ int indirectItemIdx = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
+
+ assert indirectItemIdx >= directCnt : indirectItemIdx + " " + directCnt;
+ assert indirectItemIdx < directCnt + indirectCnt : indirectItemIdx + " " + directCnt + " " + indirectCnt;
+
+ itemId = directItemIndex(getItem(pageAddr, indirectItemIdx));
+
+ assert itemId >= 0 && itemId < directCnt : itemId + " " + directCnt + " " + indirectCnt; // Direct item.
+ }
+
+ return directItemToOffset(getItem(pageAddr, itemId));
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Points to the entry start.
+ * @return Link to the next entry fragment or 0 if no fragments left or if entry is not fragmented.
+ */
+ private long getNextFragmentLink(long pageAddr, int dataOff) {
+ assert isFragmented(pageAddr, dataOff);
+
+ return PageUtils.getLong(pageAddr, dataOff + PAYLOAD_LEN_SIZE);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Data offset.
+ * @return {@code true} If the data row is fragmented across multiple pages.
+ */
+ protected boolean isFragmented(long pageAddr, int dataOff) {
+ return (PageUtils.getShort(pageAddr, dataOff) & FRAGMENTED_FLAG) != 0;
+ }
+
+ /**
+ * Sets position to start of actual fragment data and limit to it's end.
+ *
+ * @param pageAddr Page address.
+ * @param itemId Item to position on.
+ * @param pageSize Page size.
+ * @return {@link DataPagePayload} object.
+ */
+ public DataPagePayload readPayload(final long pageAddr, final int itemId, final int pageSize) {
+ int dataOff = getDataOffset(pageAddr, itemId, pageSize);
+
+ boolean fragmented = isFragmented(pageAddr, dataOff);
+ long nextLink = fragmented ? getNextFragmentLink(pageAddr, dataOff) : 0;
+ int payloadSize = getPageEntrySize(pageAddr, dataOff, 0);
+
+ return new DataPagePayload(dataOff + PAYLOAD_LEN_SIZE + (fragmented ? LINK_SIZE : 0),
+ payloadSize,
+ nextLink);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Item index.
+ * @return Item.
+ */
+ private short getItem(long pageAddr, int idx) {
+ return PageUtils.getShort(pageAddr, itemOffset(idx));
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Item index.
+ * @param item Item.
+ */
+ private void setItem(long pageAddr, int idx, short item) {
+ PageUtils.putShort(pageAddr, itemOffset(idx), item);
+ }
+
+ /**
+ * @param idx Index of the item.
+ * @return Offset in buffer.
+ */
+ private int itemOffset(int idx) {
+ assert checkIndex(idx) : idx;
+
+ return ITEMS_OFF + idx * ITEM_SIZE;
+ }
+
+ /**
+ * @param directItem Direct item.
+ * @return Offset of an entry payload inside of the page.
+ */
+ private int directItemToOffset(short directItem) {
+ return directItem & 0xFFFF;
+ }
+
+ /**
+ * @param dataOff Data offset.
+ * @return Direct item.
+ */
+ private short directItemFromOffset(int dataOff) {
+ assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff < Short.MAX_VALUE : dataOff;
+
+ return (short)dataOff;
+ }
+
+ /**
+ * @param indirectItem Indirect item.
+ * @return Index of corresponding direct item.
+ */
+ private int directItemIndex(short indirectItem) {
+ return indirectItem & 0xFF;
+ }
+
+ /**
+ * @param indirectItem Indirect item.
+ * @return Fixed item ID (the index used for referencing an entry from the outside).
+ */
+ private int itemId(short indirectItem) {
+ return (indirectItem & 0xFFFF) >>> 8;
+ }
+
+ /**
+ * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+ * @param directItemIdx Index of corresponding direct item.
+ * @return Indirect item.
+ */
+ private short indirectItem(int itemId, int directItemIdx) {
+ assert checkIndex(itemId) : itemId;
+ assert checkIndex(directItemIdx) : directItemIdx;
+
+ return (short)((itemId << 8) | directItemIdx);
+ }
+
+ /**
+ * Move the last direct item to the free slot and reference it with indirect item on the same place.
+ *
+ * @param pageAddr Page address.
+ * @param freeDirectIdx Free slot.
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @return {@code true} If the last direct item already had corresponding indirect item.
+ */
+ private boolean moveLastItem(long pageAddr, int freeDirectIdx, int directCnt, int indirectCnt) {
+ int lastIndirectId = findIndirectIndexForLastDirect(pageAddr, directCnt, indirectCnt);
+
+ int lastItemId = directCnt - 1;
+
+ assert lastItemId != freeDirectIdx;
+
+ short indirectItem = indirectItem(lastItemId, freeDirectIdx);
+
+ assert itemId(indirectItem) == lastItemId && directItemIndex(indirectItem) == freeDirectIdx;
+
+ setItem(pageAddr, freeDirectIdx, getItem(pageAddr, lastItemId));
+ setItem(pageAddr, lastItemId, indirectItem);
+
+ assert getItem(pageAddr, lastItemId) == indirectItem;
+
+ if (lastIndirectId != -1) { // Fix pointer to direct item.
+ setItem(pageAddr, lastIndirectId, indirectItem(itemId(getItem(pageAddr, lastIndirectId)), freeDirectIdx));
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @return Index of indirect item for the last direct item.
+ */
+ private int findIndirectIndexForLastDirect(long pageAddr, int directCnt, int indirectCnt) {
+ int lastDirectId = directCnt - 1;
+
+ for (int i = directCnt, end = directCnt + indirectCnt; i < end; i++) {
+ short item = getItem(pageAddr, i);
+
+ if (directItemIndex(item) == lastDirectId)
+ return i;
+ }
+
+ return -1;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param itemId Item ID.
+ * @param pageSize Page size.
+ * @param payload Row data.
+ * @param row Row.
+ * @param rowSize Row size.
+ * @return {@code True} if entry is not fragmented.
+ * @throws IgniteCheckedException If failed.
+ */
+ public boolean updateRow(
+ final long pageAddr,
+ int itemId,
+ int pageSize,
+ @Nullable byte[] payload,
+ @Nullable T row,
+ final int rowSize) throws IgniteCheckedException {
+ assert checkIndex(itemId) : itemId;
+ assert row != null ^ payload != null;
+
+ final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
+
+ if (isFragmented(pageAddr, dataOff))
+ return false;
+
+ if (row != null)
+ writeRowData(pageAddr, dataOff, rowSize, row, false);
+ else
+ writeRowData(pageAddr, dataOff, payload);
+
+ return true;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+ * @param pageSize Page size.
+ * @return Next link for fragmented entries or {@code 0} if none.
+ * @throws IgniteCheckedException If failed.
+ */
+ public long removeRow(long pageAddr, int itemId, int pageSize) throws IgniteCheckedException {
+ assert checkIndex(itemId) : itemId;
+
+ final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
+ final long nextLink = isFragmented(pageAddr, dataOff) ? getNextFragmentLink(pageAddr, dataOff) : 0;
+
+ // Record original counts to calculate delta in free space in the end of remove.
+ final int directCnt = getDirectCount(pageAddr);
+ final int indirectCnt = getIndirectCount(pageAddr);
+
+ int curIndirectCnt = indirectCnt;
+
+ assert directCnt > 0 : directCnt; // Direct count always represents overall number of live items.
+
+ // Remove the last item on the page.
+ if (directCnt == 1) {
+ assert (indirectCnt == 0 && itemId == 0) ||
+ (indirectCnt == 1 && itemId == itemId(getItem(pageAddr, 1))) : itemId;
+
+ setEmptyPage(pageAddr, pageSize);
+ }
+ else {
+ // Get the entry size before the actual remove.
+ int rmvEntrySize = getPageEntrySize(pageAddr, dataOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+ int indirectId = 0;
+
+ if (itemId >= directCnt) { // Need to remove indirect item.
+ assert indirectCnt > 0;
+
+ indirectId = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
+
+ assert indirectId >= directCnt;
+
+ itemId = directItemIndex(getItem(pageAddr, indirectId));
+
+ assert itemId < directCnt;
+ }
+
+ boolean dropLast = true;
+
+ if (itemId + 1 < directCnt) // It is not the last direct item.
+ dropLast = moveLastItem(pageAddr, itemId, directCnt, indirectCnt);
+
+ if (indirectId == 0) {// For the last direct item with no indirect item.
+ if (dropLast)
+ moveItems(pageAddr, directCnt, indirectCnt, -1, pageSize);
+ else
+ curIndirectCnt++;
+ }
+ else {
+ if (dropLast)
+ moveItems(pageAddr, directCnt, indirectId - directCnt, -1, pageSize);
+
+ moveItems(pageAddr, indirectId + 1, directCnt + indirectCnt - indirectId - 1, dropLast ? -2 : -1, pageSize);
+
+ if (dropLast)
+ curIndirectCnt--;
+ }
+
+ setIndirectCount(pageAddr, curIndirectCnt);
+ setDirectCount(pageAddr, directCnt - 1);
+
+ assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
+
+ // Increase free space.
+ setRealFreeSpace(pageAddr,
+ getRealFreeSpace(pageAddr) + rmvEntrySize + ITEM_SIZE * (directCnt - getDirectCount(pageAddr) + indirectCnt - getIndirectCount(pageAddr)),
+ pageSize);
+ }
+
+ return nextLink;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @param cnt Count.
+ * @param step Step.
+ * @param pageSize Page size.
+ */
+ private void moveItems(long pageAddr, int idx, int cnt, int step, int pageSize) {
+ assert cnt >= 0 : cnt;
+
+ if (cnt != 0)
+ moveBytes(pageAddr, itemOffset(idx), cnt * ITEM_SIZE, step * ITEM_SIZE, pageSize);
+ }
+
+ /**
+ * @param newEntryFullSize New entry full size (with item, length and link).
+ * @param firstEntryOff First entry data offset.
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @return {@code true} If there is enough space for the entry.
+ */
+ private boolean isEnoughSpace(int newEntryFullSize, int firstEntryOff, int directCnt, int indirectCnt) {
+ return ITEMS_OFF + ITEM_SIZE * (directCnt + indirectCnt) <= firstEntryOff - newEntryFullSize;
+ }
+
+ /**
+ * Adds row to this data page and sets respective link to the given row object.
+ *
+ * @param pageAddr Page address.
+ * @param row Data row.
+ * @param rowSize Row size.
+ * @param pageSize Page size.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void addRow(
+ final long pageId,
+ final long pageAddr,
+ T row,
+ final int rowSize,
+ final int pageSize
+ ) throws IgniteCheckedException {
+ assert rowSize <= getFreeSpace(pageAddr) : "can't call addRow if not enough space for the whole row";
+
+ int fullEntrySize = getPageEntrySize(rowSize, SHOW_PAYLOAD_LEN | SHOW_ITEM);
+
+ int directCnt = getDirectCount(pageAddr);
+ int indirectCnt = getIndirectCount(pageAddr);
+
+ int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
+
+ writeRowData(pageAddr, dataOff, rowSize, row, true);
+
+ int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+
+ setLinkByPageId(row, pageId, itemId);
+ }
+
+ /**
+ * Adds row to this data page and sets respective link to the given row object.
+ *
+ * @param pageAddr Page address.
+ * @param payload Payload.
+ * @param pageSize Page size.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void addRow(
+ long pageAddr,
+ byte[] payload,
+ int pageSize
+ ) throws IgniteCheckedException {
+ assert payload.length <= getFreeSpace(pageAddr) : "can't call addRow if not enough space for the whole row";
+
+ int fullEntrySize = getPageEntrySize(payload.length, SHOW_PAYLOAD_LEN | SHOW_ITEM);
+
+ int directCnt = getDirectCount(pageAddr);
+ int indirectCnt = getIndirectCount(pageAddr);
+
+ int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
+
+ writeRowData(pageAddr, dataOff, payload);
+
+ addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param entryFullSize New entry full size (with item, length and link).
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @param dataOff First entry offset.
+ * @param pageSize Page size.
+ * @return First entry offset after compaction.
+ */
+ private int compactIfNeed(
+ final long pageAddr,
+ final int entryFullSize,
+ final int directCnt,
+ final int indirectCnt,
+ int dataOff,
+ int pageSize
+ ) {
+ if (!isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt)) {
+ dataOff = compactDataEntries(pageAddr, directCnt, pageSize);
+
+ assert isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt);
+ }
+
+ return dataOff;
+ }
+
+ /**
+ * Put item reference on entry.
+ *
+ * @param pageAddr Page address.
+ * @param fullEntrySize Full entry size (with link, payload size and item).
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @param dataOff Data offset.
+ * @param pageSize Page size.
+ * @return Item ID.
+ */
+ private int addItem(final long pageAddr,
+ final int fullEntrySize,
+ final int directCnt,
+ final int indirectCnt,
+ final int dataOff,
+ final int pageSize) {
+ setFirstEntryOffset(pageAddr, dataOff, pageSize);
+
+ int itemId = insertItem(pageAddr, dataOff, directCnt, indirectCnt, pageSize);
+
+ assert checkIndex(itemId) : itemId;
+ assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
+
+ // Update free space. If number of indirect items changed, then we were able to reuse an item slot.
+ setRealFreeSpace(pageAddr,
+ getRealFreeSpace(pageAddr) - fullEntrySize + (getIndirectCount(pageAddr) != indirectCnt ? ITEM_SIZE : 0),
+ pageSize);
+
+ return itemId;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param fullEntrySize Full entry size.
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @param pageSize Page size.
+ * @return Offset in the buffer where the entry must be written.
+ */
+ private int getDataOffsetForWrite(long pageAddr, int fullEntrySize, int directCnt, int indirectCnt, int pageSize) {
+ int dataOff = getFirstEntryOffset(pageAddr);
+
+ // Compact if we do not have enough space for entry.
+ dataOff = compactIfNeed(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+
+ // We will write data right before the first entry.
+ dataOff -= fullEntrySize - ITEM_SIZE;
+
+ return dataOff;
+ }
+
+ /**
+ * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
+ *
+ * @param pageMem Page memory.
+ * @param pageAddr Page address.
+ * @param row Data row.
+ * @param written Number of bytes of row size that was already written.
+ * @param rowSize Row size.
+ * @param pageSize Page size.
+ * @return Written payload size.
+ * @throws IgniteCheckedException If failed.
+ */
+ public int addRowFragment(
+ PageMemory pageMem,
+ long pageAddr,
+ T row,
+ int written,
+ int rowSize,
+ int pageSize
+ ) throws IgniteCheckedException {
+ return addRowFragment(pageMem, pageAddr, written, rowSize, row.link(), row, null, pageSize);
+ }
+
+ /**
+ * Adds this payload as a fragment to this data page.
+ *
+ * @param pageAddr Page address.
+ * @param payload Payload bytes.
+ * @param lastLink Link to the previous written fragment (link to the tail).
+ * @param pageSize Page size.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void addRowFragment(
+ long pageAddr,
+ byte[] payload,
+ long lastLink,
+ int pageSize
+ ) throws IgniteCheckedException {
+ addRowFragment(null, pageAddr, 0, 0, lastLink, null, payload, pageSize);
+ }
+
+ /**
+ * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
+ *
+ * @param pageMem Page memory.
+ * @param pageAddr Page address.
+ * @param written Number of bytes of row size that was already written.
+ * @param rowSize Row size.
+ * @param lastLink Link to the previous written fragment (link to the tail).
+ * @param row Row.
+ * @param payload Payload bytes.
+ * @param pageSize Page size.
+ * @return Written payload size.
+ * @throws IgniteCheckedException If failed.
+ */
+ private int addRowFragment(
+ PageMemory pageMem,
+ long pageAddr,
+ int written,
+ int rowSize,
+ long lastLink,
+ T row,
+ byte[] payload,
+ int pageSize
+ ) throws IgniteCheckedException {
+ assert payload == null ^ row == null;
+
+ int directCnt = getDirectCount(pageAddr);
+ int indirectCnt = getIndirectCount(pageAddr);
+
+ int payloadSize = payload != null ? payload.length :
+ Math.min(rowSize - written, getFreeSpace(pageAddr));
+
+ int fullEntrySize = getPageEntrySize(payloadSize, SHOW_PAYLOAD_LEN | SHOW_LINK | SHOW_ITEM);
+ int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
+
+ if (payload == null) {
+ ByteBuffer buf = pageMem.pageBuffer(pageAddr);
+
+ buf.position(dataOff);
+
+ short p = (short)(payloadSize | FRAGMENTED_FLAG);
+
+ buf.putShort(p);
+ buf.putLong(lastLink);
+
+ int rowOff = rowSize - written - payloadSize;
+
+ writeFragmentData(row, buf, rowOff, payloadSize);
+ }
+ else {
+ PageUtils.putShort(pageAddr, dataOff, (short)(payloadSize | FRAGMENTED_FLAG));
+
+ PageUtils.putLong(pageAddr, dataOff + 2, lastLink);
+
+ PageUtils.putBytes(pageAddr, dataOff + 10, payload);
+ }
+
+ int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+
+ if (row != null)
+ setLink(row, pageAddr, itemId);
+
+ return payloadSize;
+ }
+
+ /**
+ * @param row Row to set link to.
+ * @param pageAddr Page address.
+ * @param itemId Item ID.
+ */
+ private void setLink(T row, long pageAddr, int itemId) {
+ long pageId = getPageId(pageAddr);
+
+ setLinkByPageId(row, pageId, itemId);
+ }
+
+ /**
+ * @param row Row to set link to.
+ * @param pageId Page ID.
+ * @param itemId Item ID.
+ */
+ private void setLinkByPageId(T row, long pageId, int itemId) {
+ row.link(PageIdUtils.link(pageId, itemId));
+ }
+
+ /**
+ * Write row data fragment.
+ *
+ * @param row Row.
+ * @param buf Byte buffer.
+ * @param rowOff Offset in row data bytes.
+ * @param payloadSize Data length that should be written in a fragment.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected abstract void writeFragmentData(
+ final T row,
+ final ByteBuffer buf,
+ final int rowOff,
+ final int payloadSize
+ ) throws IgniteCheckedException;
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Data offset.
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @param pageSize Page size.
+ * @return Item ID (insertion index).
+ */
+ private int insertItem(long pageAddr, int dataOff, int directCnt, int indirectCnt, int pageSize) {
+ if (indirectCnt > 0) {
+ // If the first indirect item is on correct place to become the last direct item, do the transition
+ // and insert the new item into the free slot which was referenced by this first indirect item.
+ short item = getItem(pageAddr, directCnt);
+
+ if (itemId(item) == directCnt) {
+ int directItemIdx = directItemIndex(item);
+
+ setItem(pageAddr, directCnt, getItem(pageAddr, directItemIdx));
+ setItem(pageAddr, directItemIdx, directItemFromOffset(dataOff));
+
+ setDirectCount(pageAddr, directCnt + 1);
+ setIndirectCount(pageAddr, indirectCnt - 1);
+
+ return directItemIdx;
+ }
+ }
+
+ // Move all the indirect items forward to make a free slot and insert new item at the end of direct items.
+ moveItems(pageAddr, directCnt, indirectCnt, +1, pageSize);
+
+ setItem(pageAddr, directCnt, directItemFromOffset(dataOff));
+
+ setDirectCount(pageAddr, directCnt + 1);
+ assert getDirectCount(pageAddr) == directCnt + 1;
+
+ return directCnt; // Previous directCnt will be our itemId.
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param directCnt Direct items count.
+ * @param pageSize Page size.
+ * @return New first entry offset.
+ */
+ private int compactDataEntries(long pageAddr, int directCnt, int pageSize) {
+ assert checkCount(directCnt) : directCnt;
+
+ int[] offs = new int[directCnt];
+
+ for (int i = 0; i < directCnt; i++) {
+ int off = directItemToOffset(getItem(pageAddr, i));
+
+ offs[i] = (off << 8) | i; // This way we'll be able to sort by offset using Arrays.sort(...).
+ }
+
+ Arrays.sort(offs);
+
+ // Move right all of the entries if possible to make the page as compact as possible to its tail.
+ int prevOff = pageSize;
+
+ final int start = directCnt - 1;
+ int curOff = offs[start] >>> 8;
+ int curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+ for (int i = start; i >= 0; i--) {
+ assert curOff < prevOff : curOff;
+
+ int delta = prevOff - (curOff + curEntrySize);
+
+ int off = curOff;
+ int entrySize = curEntrySize;
+
+ if (delta != 0) { // Move right.
+ assert delta > 0 : delta;
+
+ int itemId = offs[i] & 0xFF;
+
+ setItem(pageAddr, itemId, directItemFromOffset(curOff + delta));
+
+ for (int j = i - 1; j >= 0; j--) {
+ int offNext = offs[j] >>> 8;
+ int nextSize = getPageEntrySize(pageAddr, offNext, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+ if (offNext + nextSize == off) {
+ i--;
+
+ off = offNext;
+ entrySize += nextSize;
+
+ itemId = offs[j] & 0xFF;
+ setItem(pageAddr, itemId, directItemFromOffset(offNext + delta));
+ }
+ else {
+ curOff = offNext;
+ curEntrySize = nextSize;
+
+ break;
+ }
+ }
+
+ moveBytes(pageAddr, off, entrySize, delta, pageSize);
+
+ off += delta;
+ }
+ else if (i > 0) {
+ curOff = offs[i - 1] >>> 8;
+ curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
+ }
+
+ prevOff = off;
+ }
+
+ return prevOff;
+ }
+
+ /**
+ * Full-scan free space calculation procedure.
+ *
+ * @param pageAddr Page to scan.
+ * @param pageSize Page size.
+ * @return Actual free space in the buffer.
+ */
+ private int actualFreeSpace(long pageAddr, int pageSize) {
+ int directCnt = getDirectCount(pageAddr);
+
+ int entriesSize = 0;
+
+ for (int i = 0; i < directCnt; i++) {
+ int off = directItemToOffset(getItem(pageAddr, i));
+
+ int entrySize = getPageEntrySize(pageAddr, off, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+ entriesSize += entrySize;
+ }
+
+ return pageSize - ITEMS_OFF - entriesSize - (directCnt + getIndirectCount(pageAddr)) * ITEM_SIZE;
+ }
+
+ /**
+ * @param addr Address.
+ * @param off Offset.
+ * @param cnt Count.
+ * @param step Step.
+ * @param pageSize Page size.
+ */
+ private void moveBytes(long addr, int off, int cnt, int step, int pageSize) {
+ assert step != 0 : step;
+ assert off + step >= 0;
+ assert off + step + cnt <= pageSize : "[off=" + off + ", step=" + step + ", cnt=" + cnt +
+ ", cap=" + pageSize + ']';
+
+ PageHandler.copyMemory(addr, off, addr, off + step, cnt);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Data offset.
+ * @param payloadSize Payload size.
+ * @param row Data row.
+ * @param newRow {@code False} if existing cache entry is updated, in this case skip key data write.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected abstract void writeRowData(
+ long pageAddr,
+ int dataOff,
+ int payloadSize,
+ T row,
+ boolean newRow
+ ) throws IgniteCheckedException;
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Data offset.
+ * @param payload Payload
+ */
+ protected void writeRowData(
+ long pageAddr,
+ int dataOff,
+ byte[] payload
+ ) {
+ PageUtils.putShort(pageAddr, dataOff, (short)payload.length);
+ dataOff += 2;
+
+ PageUtils.putBytes(pageAddr, dataOff, payload);
+ }
+
+ /**
+ * @param row Row.
+ * @return Row size in page.
+ * @throws IgniteCheckedException if failed.
+ */
+ public abstract int getRowSize(T row) throws IgniteCheckedException;
+
+ /**
+ * Defines closure interface for applying computations to data page items.
+ *
+ * @param <T> Closure return type.
+ */
+ public interface CC<T> {
+ /**
+ * Closure body.
+ *
+ * @param link Link to item.
+ * @return Closure return value.
+ * @throws IgniteCheckedException In case of error in closure body.
+ */
+ public T apply(long link) throws IgniteCheckedException;
+ }
+}
[04/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
new file mode 100644
index 0000000..1a9393a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
@@ -0,0 +1,993 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.cluster.DetachedClusterNode;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
+import org.apache.ignite.internal.processors.cluster.BaselineTopologyHistory;
+import org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class IgniteBaselineAffinityTopologyActivationTest extends GridCommonAbstractTest {
+ /** */
+ private String consId;
+
+ /** Entries count to add to cache. */
+ private static final int ENTRIES_COUNT = 100;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ if (consId != null)
+ cfg.setConsistentId(consId);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration().setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true).setMaxSize(10 * 1024 * 1024)
+
+ ).setWalMode(WALMode.LOG_ONLY)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(false);
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /**
+ * Verifies that when old but compatible node
+ * (it is the node that once wasn't presented in branchingHistory but hasn't participated in any branching point)
+ * joins the cluster after restart, cluster gets activated.
+ */
+ public void testAutoActivationWithCompatibleOldNode() throws Exception {
+ startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+ startGridWithConsistentId("C").active(true);
+
+ stopAllGrids(false);
+
+ startGridWithConsistentId("A");
+ startGridWithConsistentId("B").active(true);
+
+ {
+ IgniteEx nodeA = grid("A");
+
+ assertNotNull(nodeA.cluster().currentBaselineTopology());
+ assertEquals(3, nodeA.cluster().currentBaselineTopology().size());
+
+ assertTrue(nodeA.cluster().active());
+ }
+
+ stopAllGrids(false);
+
+ startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+
+ {
+ IgniteEx nodeA = grid("A");
+
+ assertNotNull(nodeA.cluster().currentBaselineTopology());
+ assertEquals(3, nodeA.cluster().currentBaselineTopology().size());
+
+ assertFalse(nodeA.cluster().active());
+ }
+
+ final Ignite nodeC = startGridWithConsistentId("C");
+
+ boolean active = GridTestUtils.waitForCondition(
+ new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return nodeC.active();
+ }
+ },
+ 10_000
+ );
+
+ assertTrue(active);
+ }
+
+ /**
+ * IgniteCluster::setBaselineTopology(long topVer) should throw an exception
+ * when online node from current BaselineTopology is not presented in topology version.
+ */
+ public void testBltChangeTopVerRemoveOnlineNodeFails() throws Exception {
+ Ignite ignite = startGridWithConsistentId("A");
+
+ ignite.active(true);
+
+ long singleNodeTopVer = ignite.cluster().topologyVersion();
+
+ startGridWithConsistentId("OnlineConsID");
+
+ ignite.cluster().setBaselineTopology(baselineNodes(ignite.cluster().forServers().nodes()));
+
+ boolean expectedExceptionThrown = false;
+
+ try {
+ ignite.cluster().setBaselineTopology(singleNodeTopVer);
+ }
+ catch (IgniteException e) {
+ String errMsg = e.getMessage();
+ assertTrue(errMsg.startsWith("Removing online nodes"));
+ assertTrue(errMsg.contains("[OnlineConsID]"));
+
+ expectedExceptionThrown = true;
+ }
+
+ assertTrue(expectedExceptionThrown);
+ }
+
+ /**
+ * Verifies that online nodes cannot be removed from BaselineTopology (this may change in future).
+ */
+ public void testOnlineNodesCannotBeRemovedFromBaselineTopology() throws Exception {
+ Ignite nodeA = startGridWithConsistentId("A");
+ Ignite nodeB = startGridWithConsistentId("B");
+ Ignite nodeC = startGridWithConsistentId("OnlineConsID");
+
+ nodeC.active(true);
+
+ boolean expectedExceptionIsThrown = false;
+
+ try {
+ nodeC.cluster().setBaselineTopology(Arrays.asList((BaselineNode) nodeA.cluster().localNode(),
+ nodeB.cluster().localNode()));
+ } catch (IgniteException e) {
+ String errMsg = e.getMessage();
+ assertTrue(errMsg.startsWith("Removing online nodes"));
+ assertTrue(errMsg.contains("[OnlineConsID]"));
+
+ expectedExceptionIsThrown = true;
+ }
+
+ assertTrue(expectedExceptionIsThrown);
+ }
+
+ /**
+ *
+ */
+ public void testNodeFailsToJoinWithIncompatiblePreviousBaselineTopology() throws Exception {
+ startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+ Ignite nodeC = startGridWithConsistentId("C");
+
+ nodeC.active(true);
+
+ stopAllGrids(false);
+
+ Ignite nodeA = startGridWithConsistentId("A");
+ startGridWithConsistentId("B").active(true);
+
+ nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+ stopAllGrids(false);
+
+ startGridWithConsistentId("C").active(true);
+
+ stopGrid("C", false);
+
+ startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+
+ boolean expectedExceptionThrown = false;
+
+ try {
+ startGridWithConsistentId("C");
+ }
+ catch (IgniteCheckedException e) {
+ expectedExceptionThrown = true;
+
+ if (e.getCause() != null && e.getCause().getCause() != null) {
+ Throwable rootCause = e.getCause().getCause();
+
+ if (!(rootCause instanceof IgniteSpiException) || !rootCause.getMessage().contains("not compatible"))
+ Assert.fail("Unexpected ignite exception was thrown: " + e);
+ }
+ else
+ throw e;
+ }
+
+ assertTrue("Expected exception wasn't thrown.", expectedExceptionThrown);
+ }
+
+ /**
+ * Verifies scenario when parts of grid were activated independently they are not allowed to join
+ * into the same grid again (due to risks of incompatible data modifications).
+ */
+ public void testIncompatibleBltNodeIsProhibitedToJoinCluster() throws Exception {
+ startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+ startGridWithConsistentId("C").active(true);
+
+ stopAllGrids(false);
+
+ startGridWithConsistentId("A");
+ startGridWithConsistentId("B").active(true);
+
+ stopAllGrids(false);
+
+ startGridWithConsistentId("C").active(true);
+
+ stopAllGrids(false);
+
+ startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+
+ boolean expectedExceptionThrown = false;
+
+ try {
+ startGridWithConsistentId("C");
+ }
+ catch (IgniteCheckedException e) {
+ expectedExceptionThrown = true;
+
+ if (e.getCause() != null && e.getCause().getCause() != null) {
+ Throwable rootCause = e.getCause().getCause();
+
+ if (!(rootCause instanceof IgniteSpiException) || !rootCause.getMessage().contains("not compatible"))
+ Assert.fail("Unexpected ignite exception was thrown: " + e);
+ }
+ else
+ throw e;
+ }
+
+ assertTrue("Expected exception wasn't thrown.", expectedExceptionThrown);
+ }
+
+ /**
+ * Test verifies that node with out-of-data but still compatible Baseline Topology is allowed to join the cluster.
+ */
+ public void testNodeWithOldBltIsAllowedToJoinCluster() throws Exception {
+ final long expectedHash1 = (long)"A".hashCode() + "B".hashCode() + "C".hashCode();
+
+ BaselineTopologyVerifier verifier1 = new BaselineTopologyVerifier() {
+ @Override public void verify(BaselineTopology blt) {
+ assertNotNull(blt);
+
+ assertEquals(3, blt.consistentIds().size());
+
+ long activationHash = U.field(blt, "branchingPntHash");
+
+ assertEquals(expectedHash1, activationHash);
+ }
+ };
+
+ final long expectedHash2 = (long)"A".hashCode() + "B".hashCode();
+
+ BaselineTopologyVerifier verifier2 = new BaselineTopologyVerifier() {
+ @Override public void verify(BaselineTopology blt) {
+ assertNotNull(blt);
+
+ assertEquals(3, blt.consistentIds().size());
+
+ long activationHash = U.field(blt, "branchingPntHash");
+
+ assertEquals(expectedHash2, activationHash);
+ }
+ };
+
+ Ignite nodeA = startGridWithConsistentId("A");
+ Ignite nodeB = startGridWithConsistentId("B");
+ Ignite nodeC = startGridWithConsistentId("C");
+
+ nodeC.active(true);
+ verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeA, nodeB, nodeC});
+
+ stopAllGrids(false);
+
+ nodeA = startGridWithConsistentId("A");
+ nodeB = startGridWithConsistentId("B");
+
+ nodeB.active(true);
+
+ verifyBaselineTopologyOnNodes(verifier2, new Ignite[] {nodeA, nodeB});
+
+ stopAllGrids(false);
+
+ nodeA = startGridWithConsistentId("A");
+ nodeB = startGridWithConsistentId("B");
+ nodeC = startGridWithConsistentId("C");
+
+ verifyBaselineTopologyOnNodes(verifier2, new Ignite[] {nodeA, nodeB, nodeC});
+ }
+
+ /**
+ * Verifies that when new node outside of baseline topology joins active cluster with BLT already set
+ * it receives BLT from the cluster and stores it locally.
+ */
+ public void testNewNodeJoinsToActiveCluster() throws Exception {
+ Ignite nodeA = startGridWithConsistentId("A");
+ Ignite nodeB = startGridWithConsistentId("B");
+ Ignite nodeC = startGridWithConsistentId("C");
+
+ nodeC.active(true);
+
+ BaselineTopologyVerifier verifier1 = new BaselineTopologyVerifier() {
+ @Override public void verify(BaselineTopology blt) {
+ assertNotNull(blt);
+
+ assertEquals(3, blt.consistentIds().size());
+ }
+ };
+
+ verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeA, nodeB, nodeC});
+
+ Ignite nodeD = startGridWithConsistentId("D");
+
+ verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeD});
+
+ stopAllGrids(false);
+
+ nodeD = startGridWithConsistentId("D");
+
+ assertFalse(nodeD.active());
+
+ verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeD});
+ }
+
+ /**
+ *
+ */
+ public void testRemoveNodeFromBaselineTopology() throws Exception {
+ final long expectedActivationHash = (long)"A".hashCode() + "C".hashCode();
+
+ BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+ @Override public void verify(BaselineTopology blt) {
+ assertNotNull(blt);
+
+ assertEquals(2, blt.consistentIds().size());
+
+ long activationHash = U.field(blt, "branchingPntHash");
+
+ assertEquals(expectedActivationHash, activationHash);
+ }
+ };
+
+ Ignite nodeA = startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+ Ignite nodeC = startGridWithConsistentId("C");
+
+ nodeC.active(true);
+
+ stopGrid("B", false);
+
+ nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+ boolean activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return grid("A").active();
+ }
+ }, 10_000);
+
+ assertEquals(true, activated);
+
+ verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeC});
+
+ stopAllGrids(false);
+
+ nodeA = startGridWithConsistentId("A");
+ nodeC = startGridWithConsistentId("C");
+
+ activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return grid("A").active();
+ }
+ }, 10_000);
+
+ assertTrue(activated);
+
+ verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeC});
+ }
+
+ /**
+ *
+ */
+ public void testAddNodeToBaselineTopology() throws Exception {
+ final long expectedActivationHash = (long)"A".hashCode() + "B".hashCode() + "C".hashCode() + "D".hashCode();
+
+ BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+ @Override public void verify(BaselineTopology blt) {
+ assertNotNull(blt);
+
+ assertEquals(4, blt.consistentIds().size());
+
+ long activationHash = U.field(blt, "branchingPntHash");
+
+ assertEquals(expectedActivationHash, activationHash);
+ }
+ };
+
+ Ignite nodeA = startGridWithConsistentId("A");
+ Ignite nodeB = startGridWithConsistentId("B");
+ Ignite nodeC = startGridWithConsistentId("C");
+
+ nodeC.active(true);
+
+ IgniteEx nodeD = (IgniteEx) startGridWithConsistentId("D");
+
+ nodeD.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+ verifyBaselineTopologyOnNodes(verifier, new Ignite[]{nodeA, nodeB, nodeC, nodeD});
+ }
+
+ /**
+ * Verifies that baseline topology is removed successfully through baseline changing API.
+ */
+ public void testRemoveBaselineTopology() throws Exception {
+ BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+ @Override public void verify(BaselineTopology blt) {
+ assertNull(blt);
+ }
+ };
+
+ Ignite nodeA = startGridWithConsistentId("A");
+ Ignite nodeB = startGridWithConsistentId("B");
+ Ignite nodeC = startGridWithConsistentId("C");
+
+ nodeA.active(true);
+
+ nodeA.cluster().setBaselineTopology(null);
+
+ verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeB, nodeC});
+ }
+
+ /** */
+ private interface BaselineTopologyVerifier {
+ /** */
+ public void verify(BaselineTopology blt);
+ }
+
+ /** */
+ private interface BaselineTopologyHistoryVerifier {
+ /** */
+ public void verify(BaselineTopologyHistory bltHist);
+ }
+
+ /** */
+ private void verifyBaselineTopologyOnNodes(BaselineTopologyVerifier bltVerifier, Ignite[] igs) {
+ for (Ignite ig : igs) {
+ BaselineTopology blt = getBaselineTopology(ig);
+
+ bltVerifier.verify(blt);
+ }
+ }
+
+ /** */
+ private void verifyBaselineTopologyHistoryOnNodes(BaselineTopologyHistoryVerifier bltHistVerifier, Ignite[] igs) {
+ for (Ignite ig : igs) {
+ BaselineTopologyHistory blt = getBaselineTopologyHistory(ig);
+
+ bltHistVerifier.verify(blt);
+ }
+ }
+
+ /** */
+ private Ignite startGridWithConsistentId(String consId) throws Exception {
+ this.consId = consId;
+
+ return startGrid(consId);
+ }
+
+ /**
+ * Verifies that when new node joins already active cluster and new activation request is issued,
+ * no changes to BaselineTopology branching history happen.
+ */
+ public void testActivationHashIsNotUpdatedOnMultipleActivationRequests() throws Exception {
+ final long expectedActivationHash = (long)"A".hashCode();
+
+ BaselineTopologyVerifier verifier = new BaselineTopologyVerifier() {
+ @Override public void verify(BaselineTopology blt) {
+ long activationHash = U.field(blt, "branchingPntHash");
+
+ assertEquals(expectedActivationHash, activationHash);
+ }
+ };
+
+ Ignite nodeA = startGridWithConsistentId("A");
+
+ nodeA.active(true);
+
+ Ignite nodeB = startGridWithConsistentId("B");
+
+ nodeA.active(true);
+
+ verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeB});
+ }
+
+ /**
+ * Verifies that grid is autoactivated when full BaselineTopology is preset even on one node
+ * and then all other nodes from BaselineTopology are started.
+ */
+ public void testAutoActivationWithBaselineTopologyPreset() throws Exception {
+ Ignite ig = startGridWithConsistentId("A");
+
+ ig.active(true);
+
+ ig.cluster().setBaselineTopology(Arrays.asList(new BaselineNode[] {
+ createBaselineNodeWithConsId("A"), createBaselineNodeWithConsId("B"), createBaselineNodeWithConsId("C")}));
+
+ stopAllGrids();
+
+ final Ignite ig1 = startGridWithConsistentId("A");
+
+ startGridWithConsistentId("B");
+
+ startGridWithConsistentId("C");
+
+ boolean activated = GridTestUtils.waitForCondition(
+ new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return ig1.active();
+ }
+ },
+ 10_000
+ );
+
+ assertTrue(activated);
+ }
+
+ /**
+ * Creates BaselineNode with specific attribute indicating that this node is not client.
+ */
+ private BaselineNode createBaselineNodeWithConsId(String consId) {
+ Map<String, Object> attrs = new HashMap<>();
+
+ attrs.put("org.apache.ignite.cache.client", false);
+
+ return new DetachedClusterNode(consId, attrs);
+ }
+
+ /** */
+ public void testAutoActivationSimple() throws Exception {
+ startGrids(3);
+
+ IgniteEx srv = grid(0);
+
+ srv.active(true);
+
+ createAndFillCache(srv);
+
+ // TODO: final implementation should work with cancel == true.
+ stopAllGrids();
+
+ //note: no call for activation after grid restart
+ startGrids(3);
+
+ final Ignite ig = grid(0);
+
+ boolean clusterActive = GridTestUtils.waitForCondition(
+ new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return ig.active();
+ }
+ },
+ 10_000);
+
+ assertTrue(clusterActive);
+
+ checkDataInCache((IgniteEx) ig);
+ }
+
+ /**
+ *
+ */
+ public void testNoAutoActivationOnJoinNewNodeToInactiveCluster() throws Exception {
+ startGrids(2);
+
+ IgniteEx srv = grid(0);
+
+ srv.active(true);
+
+ awaitPartitionMapExchange();
+
+ assertTrue(srv.active());
+
+ srv.active(false);
+
+ assertFalse(srv.active());
+
+ startGrid(2);
+
+ Thread.sleep(3_000);
+
+ assertFalse(srv.active());
+ }
+
+ /**
+ * Verifies that neither BaselineTopology nor BaselineTopologyHistory are changed when cluster is deactivated.
+ */
+ public void testBaselineTopologyRemainsTheSameOnClusterDeactivation() throws Exception {
+ startGrids(2);
+
+ IgniteEx srv = grid(0);
+
+ srv.active(true);
+
+ awaitPartitionMapExchange();
+
+ assertTrue(srv.active());
+
+ srv.active(false);
+
+ BaselineTopology blt = getBaselineTopology(srv);
+
+ BaselineTopologyHistory bltHist = getBaselineTopologyHistory(srv);
+
+ assertEquals(0, blt.id());
+
+ assertEquals(2, blt.consistentIds().size());
+
+ assertEquals(1, blt.branchingHistory().size());
+
+ assertEquals(0, bltHist.history().size());
+ }
+
+ /**
+ *
+ */
+ public void testBaselineHistorySyncWithNewNode() throws Exception {
+ final long expectedBranchingHash = "A".hashCode() + "B".hashCode() + "C".hashCode();
+
+ BaselineTopologyHistoryVerifier verifier = new BaselineTopologyHistoryVerifier() {
+ @Override public void verify(BaselineTopologyHistory bltHist) {
+ assertNotNull(bltHist);
+
+ assertEquals(1, bltHist.history().size());
+
+ BaselineTopologyHistoryItem histItem = bltHist.history().get(0);
+
+ assertEquals(1, histItem.branchingHistory().size());
+
+ long actualBranchingHash = histItem.branchingHistory().get(0);
+
+ assertEquals(expectedBranchingHash, actualBranchingHash);
+ }
+ };
+
+ Ignite nodeA = startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+ startGridWithConsistentId("C");
+
+ nodeA.active(true);
+
+ stopGrid("C", false);
+
+ nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+ startGridWithConsistentId("D");
+
+ stopAllGrids(false);
+
+ Ignite nodeD = startGridWithConsistentId("D");
+
+ verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {nodeD});
+ }
+
+ /**
+ *
+ */
+ public void testBaselineHistorySyncWithOldNodeWithCompatibleHistory() throws Exception {
+ final long expectedBranchingHash0 = "A".hashCode() + "B".hashCode() + "C".hashCode();
+
+ final long expectedBranchingHash1 = "A".hashCode() + "B".hashCode();
+
+ BaselineTopologyHistoryVerifier verifier = new BaselineTopologyHistoryVerifier() {
+ @Override public void verify(BaselineTopologyHistory bltHist) {
+ assertNotNull(bltHist);
+
+ assertEquals(2, bltHist.history().size());
+
+ BaselineTopologyHistoryItem histItem = bltHist.history().get(0);
+
+ assertEquals(1, histItem.branchingHistory().size());
+
+ long actualBranchingHash0 = histItem.branchingHistory().get(0);
+
+ assertEquals(expectedBranchingHash0, actualBranchingHash0);
+
+ histItem = bltHist.history().get(1);
+
+ assertEquals(1, histItem.branchingHistory().size());
+
+ long actualBranchingHash1 = histItem.branchingHistory().get(0);
+
+ assertEquals(expectedBranchingHash1, actualBranchingHash1);
+ }
+ };
+
+ Ignite nodeA = startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+ startGridWithConsistentId("C");
+
+ nodeA.active(true);
+
+ stopGrid("C", false);
+
+ nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+ stopGrid("B", false);
+
+ nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+ startGridWithConsistentId("B");
+
+ stopAllGrids(false);
+
+ startGridWithConsistentId("A");
+ Ignite nodeB = startGridWithConsistentId("B");
+
+ verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {nodeB});
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testBaselineNotDeletedOnDeactivation() throws Exception {
+ Ignite nodeA = startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+ startGridWithConsistentId("C");
+
+ nodeA.active(true);
+
+ assertNotNull(nodeA.cluster().currentBaselineTopology());
+
+ nodeA.active(false);
+
+ stopAllGrids();
+
+ nodeA = startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+ startGridWithConsistentId("C");
+
+ final Ignite ig = nodeA;
+
+ boolean clusterActive = GridTestUtils.waitForCondition(
+ new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return ig.active();
+ }
+ },
+ 10_000);
+
+ assertNotNull(nodeA.cluster().currentBaselineTopology());
+
+ assertTrue(clusterActive);
+ }
+
+ /**
+ *
+ */
+ public void testNodeWithBltIsProhibitedToJoinNewCluster() throws Exception {
+ BaselineTopologyVerifier nullVerifier = new BaselineTopologyVerifier() {
+ @Override public void verify(BaselineTopology blt) {
+ assertNull(blt);
+ }
+ };
+
+ Ignite nodeC = startGridWithConsistentId("C");
+
+ nodeC.active(true);
+
+ stopGrid("C", false);
+
+ Ignite nodeA = startGridWithConsistentId("A");
+ Ignite nodeB = startGridWithConsistentId("B");
+
+ verifyBaselineTopologyOnNodes(nullVerifier, new Ignite[] {nodeA, nodeB});
+
+ boolean expectedExceptionThrown = false;
+
+ try {
+ startGridWithConsistentId("C");
+ }
+ catch (IgniteCheckedException e) {
+ expectedExceptionThrown = true;
+
+ if (e.getCause() != null && e.getCause().getCause() != null) {
+ Throwable rootCause = e.getCause().getCause();
+
+ if (!(rootCause instanceof IgniteSpiException) || !rootCause.getMessage().contains("Node with set up BaselineTopology"))
+ Assert.fail("Unexpected ignite exception was thrown: " + e);
+ }
+ else
+ throw e;
+ }
+
+ assertTrue("Expected exception wasn't thrown.", expectedExceptionThrown);
+ }
+
+ /**
+ * Restore this test when requirements for BaselineTopology deletion are clarified and this feature
+ * is covered with more tests.
+ */
+ public void _testBaselineTopologyHistoryIsDeletedOnBaselineDelete() throws Exception {
+ BaselineTopologyHistoryVerifier verifier = new BaselineTopologyHistoryVerifier() {
+ @Override public void verify(BaselineTopologyHistory bltHist) {
+ assertNotNull(bltHist);
+
+ assertEquals(0, bltHist.history().size());
+ }
+ };
+
+ Ignite nodeA = startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+ startGridWithConsistentId("C");
+
+ nodeA.active(true);
+
+ stopAllGrids(false);
+
+ nodeA = startGridWithConsistentId("A");
+ startGridWithConsistentId("B");
+
+ nodeA.active(true);
+
+ nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+ stopAllGrids(false);
+
+ nodeA = startGridWithConsistentId("A");
+
+ nodeA.active(true);
+
+ nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
+
+ stopAllGrids(false);
+
+ final Ignite node = startGridWithConsistentId("A");
+
+ boolean activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return node.active();
+ }
+ }, 10_000);
+
+ assertTrue(activated);
+
+ node.cluster().setBaselineTopology(null);
+
+ verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {node});
+
+ stopAllGrids(false);
+
+ nodeA = startGridWithConsistentId("A");
+
+ verifyBaselineTopologyHistoryOnNodes(verifier, new Ignite[] {nodeA});
+ }
+
+ /**
+ * Retrieves baseline topology from ignite node instance.
+ *
+ * @param ig Ig.
+ */
+ private BaselineTopology getBaselineTopology(Ignite ig) {
+ return ((DiscoveryDataClusterState) U.field(
+ (Object) U.field(
+ (Object) U.field(ig, "ctx"),
+ "stateProc"),
+ "globalState"))
+ .baselineTopology();
+ }
+
+ /** */
+ private BaselineTopologyHistory getBaselineTopologyHistory(Ignite ig) {
+ return U.field(
+ (Object) U.field(
+ (Object) U.field(ig, "ctx"),
+ "stateProc"),
+ "bltHist");
+ }
+
+ /** */
+ private void checkDataInCache(IgniteEx srv) {
+ IgniteCache<Object, Object> cache = srv.cache(DEFAULT_CACHE_NAME);
+
+ for (int i = 0; i < ENTRIES_COUNT; i++) {
+ TestValue testVal = (TestValue) cache.get(i);
+
+ assertNotNull(testVal);
+
+ assertEquals(i, testVal.id);
+ }
+ }
+
+ /** */
+ private void createAndFillCache(Ignite srv) {
+ IgniteCache cache = srv.getOrCreateCache(cacheConfiguration());
+
+ for (int i = 0; i < ENTRIES_COUNT; i++)
+ cache.put(i, new TestValue(i, "str" + i));
+ }
+
+ /** */
+ private CacheConfiguration cacheConfiguration() {
+ return new CacheConfiguration()
+ .setName(DEFAULT_CACHE_NAME)
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+ .setBackups(2)
+ .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ }
+
+ /** */
+ private static final class TestValue {
+ /** */
+ private final int id;
+
+ /** */
+ private final String strId;
+
+ /** */
+ private TestValue(int id, String strId) {
+ this.id = id;
+ this.strId = strId;
+ }
+ }
+
+ /** */
+ private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+ Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+ for (ClusterNode clN : clNodes)
+ res.add(clN);
+
+ return res;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
index 50d7e7e..73c269a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
@@ -17,6 +17,11 @@
package org.apache.ignite.internal.processors.cache.persistence;
import java.io.File;
+import java.nio.file.CopyOption;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.util.Collection;
import org.apache.ignite.Ignite;
@@ -35,6 +40,7 @@ import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -56,12 +62,21 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
private static final String DYNAMIC_STR_FIELD_NAME = "strField";
/** */
+ private static final String CUSTOM_WORK_DIR_NAME_PATTERN = "node%s_workDir";
+
+ /** */
private boolean clientMode;
+ /** */
+ private String customWorkSubDir;
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
+ if (customWorkSubDir != null)
+ cfg.setWorkDirectory(Paths.get(U.defaultWorkDirectory(), customWorkSubDir).toString());
+
cfg.setClientMode(clientMode);
cfg.setDataStorageConfiguration(
@@ -99,6 +114,257 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
}
/**
+ * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+ */
+ public void testUpdatedBinaryMetadataIsPreservedOnJoinToOldCoordinator() throws Exception {
+ Ignite ignite0 = startGridInASeparateWorkDir("A");
+ Ignite ignite1 = startGridInASeparateWorkDir("B");
+
+ ignite0.active(true);
+
+ IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
+
+ BinaryObject bo = ignite0
+ .binary()
+ .builder(DYNAMIC_TYPE_NAME)
+ .setField(DYNAMIC_INT_FIELD_NAME, 10)
+ .build();
+
+ cache0.put(0, bo);
+
+ stopGrid("A");
+
+ IgniteCache<Object, Object> cache1 = ignite1.cache(CACHE_NAME);
+
+ bo = ignite1
+ .binary()
+ .builder(DYNAMIC_TYPE_NAME)
+ .setField(DYNAMIC_INT_FIELD_NAME, 20)
+ .setField(DYNAMIC_STR_FIELD_NAME, "str")
+ .build();
+
+ cache1.put(1, bo);
+
+ stopAllGrids();
+
+ ignite0 = startGridInASeparateWorkDir("A");
+ ignite1 = startGridInASeparateWorkDir("B");
+
+ awaitPartitionMapExchange();
+
+ cache0 = ignite0.cache(CACHE_NAME).withKeepBinary();
+
+ BinaryObject bObj0 = (BinaryObject) cache0.get(0);
+
+ assertEquals(10, (int) bObj0.field("intField"));
+
+ cache1 = ignite1.cache(CACHE_NAME).withKeepBinary();
+
+ BinaryObject bObj1 = (BinaryObject) cache1.get(1);
+
+ assertEquals(20, (int) bObj1.field("intField"));
+ assertEquals("str", bObj1.field("strField"));
+ }
+
+ /**
+ * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+ */
+ public void testNewBinaryMetadataIsWrittenOnOldCoordinator() throws Exception {
+ Ignite ignite0 = startGridInASeparateWorkDir("A");
+ Ignite ignite1 = startGridInASeparateWorkDir("B");
+
+ ignite0.active(true);
+
+ IgniteCache<Object, Object> cache0 = ignite0.cache(CACHE_NAME);
+
+ BinaryObject bObj0 = ignite0.binary()
+ .builder("DynamicType0").setField("intField", 10).build();
+
+ cache0.put(0, bObj0);
+
+ stopGrid("A");
+
+ IgniteCache<Object, Object> cache1 = ignite1.cache(CACHE_NAME);
+
+ BinaryObject bObj1 = ignite1.binary()
+ .builder("DynamicType1").setField("strField", "str").build();
+
+ cache1.put(1, bObj1);
+
+ stopAllGrids();
+
+ ignite0 = startGridInASeparateWorkDir("A");
+
+ startGridInASeparateWorkDir("B");
+
+ awaitPartitionMapExchange();
+
+ stopGrid("B");
+
+ cache0 = ignite0.cache(CACHE_NAME).withKeepBinary();
+
+ bObj0 = (BinaryObject) cache0.get(0);
+ bObj1 = (BinaryObject) cache0.get(1);
+
+ assertEquals("DynamicType0", binaryTypeName(bObj0));
+ assertEquals("DynamicType1", binaryTypeName(bObj1));
+
+ assertEquals(10, (int) bObj0.field(DYNAMIC_INT_FIELD_NAME));
+ assertEquals("str", bObj1.field(DYNAMIC_STR_FIELD_NAME));
+ }
+
+ /**
+ * Verifies that newer BinaryMetadata is applied not only on coordinator but on all nodes
+ * with outdated version.
+ *
+ * In the following test nodeB was offline when BinaryMetadata was updated,
+ * after full cluster restart it starts second (so it doesn't take the role of coordinator)
+ * but metadata update is propagated to it anyway.
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+ */
+ public void testNewBinaryMetadataIsPropagatedToAllOutOfDataNodes() throws Exception {
+ Ignite igniteA = startGridInASeparateWorkDir("A");
+ startGridInASeparateWorkDir("B");
+ Ignite igniteC = startGridInASeparateWorkDir("C");
+ startGridInASeparateWorkDir("D");
+
+ igniteA.active(true);
+
+ BinaryObject bObj0 = igniteA.binary()
+ .builder(DYNAMIC_TYPE_NAME).setField(DYNAMIC_INT_FIELD_NAME, 10).build();
+
+ igniteA.cache(CACHE_NAME).put(0, bObj0);
+
+ stopGrid("A");
+ stopGrid("B");
+
+ BinaryObject bObj1 = igniteC.binary()
+ .builder(DYNAMIC_TYPE_NAME)
+ .setField(DYNAMIC_INT_FIELD_NAME, 20)
+ .setField(DYNAMIC_STR_FIELD_NAME, "str").build();
+
+ igniteC.cache(CACHE_NAME).put(1, bObj1);
+
+ //full cluster restart
+ stopAllGrids();
+
+ //node A becomes a coordinator
+ igniteA = startGridInASeparateWorkDir("A");
+ //node B isn't a coordinator, but metadata update is propagated to if
+ startGridInASeparateWorkDir("B");
+ //these two nodes will provide an updated version of metadata when started
+ startGridInASeparateWorkDir("C");
+ startGridInASeparateWorkDir("D");
+
+ awaitPartitionMapExchange();
+
+ //stopping everything to make sure that nodeB will lose its in-memory BinaryMetadata cache
+ // and will have to reload it from FS when restarted later
+ stopAllGrids();
+
+ Ignite igniteB = startGridInASeparateWorkDir("B");
+
+ igniteB.active(true);
+
+ bObj1 = (BinaryObject) igniteB.cache(CACHE_NAME).withKeepBinary().get(1);
+
+ assertEquals(20, (int) bObj1.field(DYNAMIC_INT_FIELD_NAME));
+ assertEquals("str", bObj1.field(DYNAMIC_STR_FIELD_NAME));
+ }
+
+ /** */
+ private Ignite startGridInASeparateWorkDir(String nodeName) throws Exception {
+ customWorkSubDir = String.format(CUSTOM_WORK_DIR_NAME_PATTERN, nodeName);
+ return startGrid(nodeName);
+ }
+
+ /** */
+ private String binaryTypeName(BinaryObject bObj) {
+ return bObj.type().typeName();
+ }
+
+ /**
+ * If joining node has incompatible BinaryMetadata (e.g. when user manually copies binary_meta file),
+ * coordinator detects it and fails the node providing information about conflict.
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7258">IGNITE-7258</a> refer to the following JIRA for more context about the problem verified by the test.
+ */
+ public void testNodeWithIncompatibleMetadataIsProhibitedToJoinTheCluster() throws Exception {
+ final String decimalFieldName = "decField";
+
+ Ignite igniteA = startGridInASeparateWorkDir("A");
+ Ignite igniteB = startGridInASeparateWorkDir("B");
+
+ String bConsId = igniteB.cluster().localNode().consistentId().toString();
+
+ igniteA.active(true);
+
+ IgniteCache<Object, Object> cache = igniteA.cache(CACHE_NAME);
+
+ BinaryObject bObj = igniteA.binary()
+ .builder(DYNAMIC_TYPE_NAME).setField(decimalFieldName, 10).build();
+
+ cache.put(0, bObj);
+
+ stopAllGrids();
+
+ Ignite igniteC = startGridInASeparateWorkDir("C");
+ String cConsId = igniteC.cluster().localNode().consistentId().toString();
+ igniteC.active(true);
+
+ cache = igniteC.cache(CACHE_NAME);
+ bObj = igniteC.binary().builder(DYNAMIC_TYPE_NAME).setField(decimalFieldName, 10L).build();
+
+ cache.put(0, bObj);
+
+ stopAllGrids();
+
+ copyIncompatibleBinaryMetadata(
+ String.format(CUSTOM_WORK_DIR_NAME_PATTERN, "C"),
+ cConsId,
+ String.format(CUSTOM_WORK_DIR_NAME_PATTERN, "B"),
+ bConsId,
+ DYNAMIC_TYPE_NAME.toLowerCase().hashCode() + ".bin");
+
+ startGridInASeparateWorkDir("A");
+
+ boolean exceptedExceptionThrown = false;
+ try {
+ startGridInASeparateWorkDir("B");
+ }
+ catch (Exception e) {
+ if (e.getCause() != null && e.getCause().getCause() != null) {
+ if (e.getCause().getCause().getMessage().contains(
+ String.format("[typeName=%s, fieldName=%s, fieldTypeName1=int, fieldTypeName2=long]",
+ DYNAMIC_TYPE_NAME,
+ decimalFieldName)
+ ))
+ exceptedExceptionThrown = true;
+ }
+ else
+ throw e;
+ }
+
+ assertTrue(exceptedExceptionThrown);
+ }
+
+ /** */
+ private void copyIncompatibleBinaryMetadata(String fromWorkDir,
+ String fromConsId,
+ String toWorkDir,
+ String toConsId,
+ String fileName
+ ) throws Exception {
+ String workDir = U.defaultWorkDirectory();
+
+ Path fromFile = Paths.get(workDir, fromWorkDir, "binary_meta", "node00-" + fromConsId, fileName);
+ Path toFile = Paths.get(workDir, toWorkDir, "binary_meta", "node00-" + toConsId, fileName);
+
+ Files.copy(fromFile, toFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ /**
* Test verifies that binary metadata from regular java classes is saved and restored correctly
* on cluster restart.
*/
@@ -237,7 +503,7 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
awaitPartitionMapExchange();
- examineDynamicMetadata(4, contentExaminer0, contentExaminer1, structureExaminer1);
+ examineDynamicMetadata(2, contentExaminer0, contentExaminer1, structureExaminer1);
}
/**
@@ -350,6 +616,8 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
stopAllGrids();
cleanIgniteWorkDir();
+
+ customWorkSubDir = null;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
index 8e43e93..bf20ee4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -686,7 +686,12 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
private static class CoordinatorNodeFilter implements IgnitePredicate<ClusterNode> {
/** {@inheritDoc} */
@Override public boolean apply(ClusterNode node) {
- return node.order() > 1;
+ try {
+ return node.order() > 1;
+ }
+ catch (UnsupportedOperationException e) {
+ return false;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
index e9828f5..301c0bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
@@ -56,6 +56,8 @@ public class IgnitePersistentStoreDataStructuresTest extends GridCommonAbstractT
cfg.setDataStorageConfiguration(memCfg);
+ cfg.setAutoActivationEnabled(false);
+
return cfg;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java
new file mode 100644
index 0000000..c6aed7d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAllBaselineNodesOnlineFullApiSelfTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteAllBaselineNodesOnlineFullApiSelfTest extends IgniteBaselineAbstractFullApiSelfTest {
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ initStoreStrategy();
+
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(gridCount());
+
+ grid(0).active(true);
+
+ awaitPartitionMapExchange();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
new file mode 100644
index 0000000..8dcfc0b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteBaselineAbstractFullApiSelfTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
+
+/**
+ *
+ */
+public abstract class IgniteBaselineAbstractFullApiSelfTest extends GridCacheAbstractFullApiSelfTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(200 * 1024 * 1024)
+ .setPersistenceEnabled(true)));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 4;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getPartitionMapExchangeTimeout() {
+ return 60_000;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java
new file mode 100644
index 0000000..16fa971
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCachePutAllFailoverTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.CachePutAllFailoverAbstractTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Failover test for cache putAll operations when BaselineTopology is changed down
+ * (existing node gets stopped and removed from BaselineTopology).
+ */
+public class IgniteChangingBaselineDownCachePutAllFailoverTest extends CachePutAllFailoverAbstractTest {
+ /** */
+ private static final int GRIDS_COUNT = 5;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setInitialSize(200 * 1024 * 1024)
+ .setMaxSize(200 * 1024 * 1024)
+ .setCheckpointPageBufferSize(200 * 1024 * 1024)
+ )
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(GRIDS_COUNT);
+
+ grid(0).active(true);
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+ return GridTestUtils.runAsync(new Callable() {
+ @Override public Object call() throws Exception {
+ Thread.currentThread().setName("restart-thread");
+
+ U.sleep(15_000);
+
+ ThreadLocalRandom tlr = ThreadLocalRandom.current();
+
+ int idx = tlr.nextInt(1, GRIDS_COUNT);
+
+ log.info("Stopping node " + idx);
+
+ stopGrid(idx);
+
+ IgniteEx ig0 = grid(0);
+
+ ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+ U.sleep(3_000);
+
+ return null;
+ }
+ });
+ }
+
+ /** */
+ private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+ Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+ for (ClusterNode clN : clNodes)
+ res.add(clN);
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java
new file mode 100644
index 0000000..86b5809
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineDownCacheRemoveFailoverTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractRemoveFailureTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * Failover test for cache remove operations when BaselineTopology is changed down
+ * (existing node gets stopped and removed from BaselineTopology).
+ */
+public class IgniteChangingBaselineDownCacheRemoveFailoverTest extends GridCacheAbstractRemoveFailureTest {
+ /** */
+ private static final int GRIDS_COUNT = 5;
+
+ /** {@inheritDoc} */
+ @Override protected CacheMode cacheMode() {
+ return PARTITIONED;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicityMode atomicityMode() {
+ return TRANSACTIONAL;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected NearCacheConfiguration nearCache() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setInitialSize(200 * 1024 * 1024)
+ .setMaxSize(200 * 1024 * 1024)
+ .setCheckpointPageBufferSize(200 * 1024 * 1024)
+ )
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(GRIDS_COUNT);
+
+ startGrid(GRIDS_COUNT);
+
+ grid(0).active(true);
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean stop, final AtomicReference<CyclicBarrier> cmp) {
+ return GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ Thread.currentThread().setName("restart-thread");
+
+ U.sleep(random(5000, 10_000));
+
+ log.info("Stopping node " + GRIDS_COUNT);
+
+ stopGrid(GRIDS_COUNT);
+
+ IgniteEx ig0 = grid(0);
+
+ ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+ while (!stop.get()) {
+ CyclicBarrier barrier = cmp.get();
+
+ if (barrier != null) {
+ log.info("Wait data check.");
+
+ barrier.await(60_000, TimeUnit.MILLISECONDS);
+
+ log.info("Finished wait data check.");
+ }
+ }
+
+ return null;
+ }
+ });
+ }
+
+ /** */
+ private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+ Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+ for (ClusterNode clN : clNodes)
+ res.add(clN);
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java
new file mode 100644
index 0000000..44c8402
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCachePutAllFailoverTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.CachePutAllFailoverAbstractTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Failover test for cache putAll operations when BaselineTopology is changed up
+ * (new node is added to BaselineTopology).
+ */
+public class IgniteChangingBaselineUpCachePutAllFailoverTest extends CachePutAllFailoverAbstractTest {
+ /** */
+ private static final int GRIDS_COUNT = 5;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setInitialSize(200 * 1024 * 1024)
+ .setMaxSize(200 * 1024 * 1024)
+ .setCheckpointPageBufferSize(200 * 1024 * 1024)
+ )
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(GRIDS_COUNT);
+
+ grid(0).active(true);
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+ return GridTestUtils.runAsync(new Callable() {
+ @Override public Object call() throws Exception {
+ Thread.currentThread().setName("restart-thread");
+
+ U.sleep(15_000);
+
+ log.info("Starting node");
+
+ startGrid(GRIDS_COUNT);
+
+ IgniteEx ig0 = grid(0);
+
+ ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+ U.sleep(3_000);
+
+ return null;
+ }
+ });
+ }
+
+ /** */
+ private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+ Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+ for (ClusterNode clN : clNodes)
+ res.add(clN);
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java
new file mode 100644
index 0000000..8391e0c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteChangingBaselineUpCacheRemoveFailoverTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractRemoveFailureTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * Failover test for cache remove operations when BaselineTopology is changed up
+ * (new node is added to BaselineTopology).
+ */
+public class IgniteChangingBaselineUpCacheRemoveFailoverTest extends GridCacheAbstractRemoveFailureTest {
+ /** */
+ private static final int GRIDS_COUNT = 5;
+
+ /** {@inheritDoc} */
+ @Override protected CacheMode cacheMode() {
+ return PARTITIONED;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicityMode atomicityMode() {
+ return TRANSACTIONAL;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected NearCacheConfiguration nearCache() {
+ return null;
+ }
+
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setInitialSize(200 * 1024 * 1024)
+ .setMaxSize(200 * 1024 * 1024)
+ .setCheckpointPageBufferSize(200 * 1024 * 1024)
+ )
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(GRIDS_COUNT);
+
+ grid(0).active(true);
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean stop, final AtomicReference<CyclicBarrier> cmp) {
+ return GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ Thread.currentThread().setName("restart-thread");
+
+ U.sleep(random(5000, 10_000));
+
+ log.info("Starting node " + GRIDS_COUNT);
+
+ startGrid(GRIDS_COUNT);
+
+ IgniteEx ig0 = grid(0);
+
+ ig0.cluster().setBaselineTopology(baselineNodes(ig0.cluster().forServers().nodes()));
+
+ while (!stop.get()) {
+ CyclicBarrier barrier = cmp.get();
+
+ if (barrier != null) {
+ log.info("Wait data check.");
+
+ barrier.await(60_000, TimeUnit.MILLISECONDS);
+
+ log.info("Finished wait data check.");
+ }
+ }
+
+ return null;
+ }
+ });
+ }
+
+ /** */
+ private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+ Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+ for (ClusterNode clN : clNodes)
+ res.add(clN);
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java
new file mode 100644
index 0000000..3fd4974
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOfflineBaselineNodeFullApiSelfTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteOfflineBaselineNodeFullApiSelfTest extends IgniteBaselineAbstractFullApiSelfTest {
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ initStoreStrategy();
+
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(gridCount());
+
+ startGrid("offlineBaselineNode");
+
+ grid(0).active(true);
+
+ stopGrid("offlineBaselineNode");
+
+ awaitPartitionMapExchange();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java
new file mode 100644
index 0000000..07c475b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteOnlineNodeOutOfBaselineFullApiSelfTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteOnlineNodeOutOfBaselineFullApiSelfTest extends IgniteBaselineAbstractFullApiSelfTest {
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ initStoreStrategy();
+
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(gridCount());
+
+ grid(0).active(true);
+
+ startGrid("onlineOutOfBaselineNode");
+
+ awaitPartitionMapExchange();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java
new file mode 100644
index 0000000..13f3489
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCachePutAllFailoverTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.CachePutAllFailoverAbstractTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class IgniteStableBaselineCachePutAllFailoverTest extends CachePutAllFailoverAbstractTest {
+ /** */
+ private static final int GRIDS_COUNT = 3;
+
+ /** */
+ private static final int OUT_OF_BASELINE_GRID_ID = GRIDS_COUNT;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setInitialSize(200 * 1024 * 1024)
+ .setMaxSize(200 * 1024 * 1024)
+ .setCheckpointPageBufferSize(200 * 1024 * 1024)
+ )
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+ return GridTestUtils.runAsync(new Callable() {
+ @Override public Object call() throws Exception {
+ Thread.currentThread().setName("restart-thread");
+
+ while (!finished.get() && System.currentTimeMillis() < endTime) {
+ ThreadLocalRandom tlr = ThreadLocalRandom.current();
+
+ int idx = tlr.nextInt(1, GRIDS_COUNT + 1);
+
+ log.info("Stopping node " + idx);
+
+ stopGrid(idx);
+
+ U.sleep(tlr.nextInt(500, 700));
+
+ log.info("Restarting node " + idx);
+
+ startGrid(idx);
+ }
+
+ return null;
+ }
+ });
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(GRIDS_COUNT);
+
+ grid(0).active(true);
+
+ startGrid(OUT_OF_BASELINE_GRID_ID);
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+}
[14/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index ed4c520..2a374a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import javax.cache.CacheException;
@@ -157,7 +158,8 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
if (ctx.clientNode())
ctx.event().addLocalEventListener(clientDisconLsnr, EVT_CLIENT_NODE_DISCONNECTED);
- metadataFileStore = new BinaryMetadataFileStore(metadataLocCache, ctx, log, binaryMetadataFileStoreDir);
+ if (!ctx.clientNode())
+ metadataFileStore = new BinaryMetadataFileStore(metadataLocCache, ctx, log, binaryMetadataFileStoreDir);
transport = new BinaryMetadataTransport(metadataLocCache, metadataFileStore, ctx, log);
@@ -241,7 +243,8 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
}
}
- metadataFileStore.restoreMetadata();
+ if (!ctx.clientNode())
+ metadataFileStore.restoreMetadata();
}
}
@@ -865,15 +868,20 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
}
/** {@inheritDoc} */
- @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode rmtNode) {
- IgniteNodeValidationResult res = super.validateNode(rmtNode);
-
- if (res != null)
- return res;
+ @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode rmtNode, DiscoveryDataBag.JoiningNodeDiscoveryData discoData) {
+ IgniteNodeValidationResult res;
if (getBoolean(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK) || !(marsh instanceof BinaryMarshaller))
return null;
+ if ((res = validateBinaryConfiguration(rmtNode)) != null)
+ return res;
+
+ return validateBinaryMetadata(rmtNode.id(), (Map<Integer, BinaryMetadataHolder>) discoData.joiningNodeData());
+ }
+
+ /** */
+ private IgniteNodeValidationResult validateBinaryConfiguration(ClusterNode rmtNode) {
Object rmtBinaryCfg = rmtNode.attribute(IgniteNodeAttributes.ATTR_BINARY_CONFIGURATION);
ClusterNode locNode = ctx.discovery().localNode();
@@ -892,6 +900,38 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
return null;
}
+ /** */
+ private IgniteNodeValidationResult validateBinaryMetadata(UUID rmtNodeId, Map<Integer, BinaryMetadataHolder> newNodeMeta) {
+ if (newNodeMeta == null)
+ return null;
+
+ for (Map.Entry<Integer, BinaryMetadataHolder> metaEntry : newNodeMeta.entrySet()) {
+ if (!metadataLocCache.containsKey(metaEntry.getKey()))
+ continue;
+
+ BinaryMetadata locMeta = metadataLocCache.get(metaEntry.getKey()).metadata();
+ BinaryMetadata rmtMeta = metaEntry.getValue().metadata();
+
+ if (locMeta == null || rmtMeta == null)
+ continue;
+
+ try {
+ BinaryUtils.mergeMetadata(locMeta, rmtMeta);
+ }
+ catch (Exception e) {
+ String locMsg = "Exception was thrown when merging binary metadata from node %s: %s";
+
+ String rmtMsg = "Exception was thrown on coordinator when merging binary metadata from this node: %s";
+
+ return new IgniteNodeValidationResult(rmtNodeId,
+ String.format(locMsg, rmtNodeId.toString(), e.getMessage()),
+ String.format(rmtMsg, e.getMessage()));
+ }
+ }
+
+ return null;
+ }
+
/** {@inheritDoc} */
@Nullable @Override public DiscoveryDataExchangeType discoveryDataType() {
return BINARY_PROC;
@@ -910,6 +950,69 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
}
/** {@inheritDoc} */
+ @Override public void collectJoiningNodeData(DiscoveryDataBag dataBag) {
+ Map<Integer, BinaryMetadataHolder> res = U.newHashMap(metadataLocCache.size());
+
+ for (Map.Entry<Integer,BinaryMetadataHolder> e : metadataLocCache.entrySet())
+ res.put(e.getKey(), e.getValue());
+
+ dataBag.addJoiningNodeData(BINARY_PROC.ordinal(), (Serializable) res);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onJoiningNodeDataReceived(DiscoveryDataBag.JoiningNodeDiscoveryData data) {
+ Map<Integer,BinaryMetadataHolder> newNodeMeta = (Map<Integer, BinaryMetadataHolder>) data.joiningNodeData();
+
+ if (newNodeMeta == null)
+ return;
+
+ UUID joiningNode = data.joiningNodeId();
+
+ for (Map.Entry<Integer, BinaryMetadataHolder> metaEntry : newNodeMeta.entrySet()) {
+ if (metadataLocCache.containsKey(metaEntry.getKey())) {
+ BinaryMetadataHolder localMetaHolder = metadataLocCache.get(metaEntry.getKey());
+
+ BinaryMetadata newMeta = metaEntry.getValue().metadata();
+ BinaryMetadata localMeta = localMetaHolder.metadata();
+
+ BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(localMeta, newMeta);
+
+ if (mergedMeta != localMeta) {
+ //put mergedMeta to local cache and store to disk
+ U.log(log,
+ String.format("Newer version of existing BinaryMetadata[typeId=%d, typeName=%s] " +
+ "is received from node %s; updating it locally",
+ mergedMeta.typeId(),
+ mergedMeta.typeName(),
+ joiningNode));
+
+ metadataLocCache.put(metaEntry.getKey(),
+ new BinaryMetadataHolder(mergedMeta,
+ localMetaHolder.pendingVersion(),
+ localMetaHolder.acceptedVersion()));
+
+ metadataFileStore.writeMetadata(mergedMeta);
+ }
+ }
+ else {
+ BinaryMetadataHolder newMetaHolder = metaEntry.getValue();
+ BinaryMetadata newMeta = newMetaHolder.metadata();
+
+ U.log(log,
+ String.format("New BinaryMetadata[typeId=%d, typeName=%s] " +
+ "is received from node %s; adding it locally",
+ newMeta.typeId(),
+ newMeta.typeName(),
+ joiningNode));
+
+ metadataLocCache.put(metaEntry.getKey(), newMetaHolder);
+
+ metadataFileStore.writeMetadata(newMeta);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public void onGridDataReceived(GridDiscoveryData data) {
Map<Integer, BinaryMetadataHolder> receivedData = (Map<Integer, BinaryMetadataHolder>) data.commonData();
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index f84fb99..5733039 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -45,6 +45,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.GridCacheReturn;
import org.apache.ignite.internal.processors.cache.GridCacheReturnCompletableWrapper;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
@@ -480,7 +481,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
cctx.database().checkpointReadLock();
try {
- Collection<IgniteTxEntry> entries = near() ? allEntries() : writeEntries();
+ Collection<IgniteTxEntry> entries = near() || cctx.snapshot().needTxReadLogging() ? allEntries() : writeEntries();
List<DataEntry> dataEntries = null;
@@ -563,7 +564,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
GridCacheVersion dhtVer = cached.isNear() ? writeVersion() : null;
if (!near() && cacheCtx.group().persistenceEnabled() &&
- op != NOOP && op != RELOAD && op != READ) {
+ op != NOOP && op != RELOAD && (op != READ || cctx.snapshot().needTxReadLogging())) {
if (dataEntries == null)
dataEntries = new ArrayList<>(entries.size());
@@ -605,7 +606,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
else {
assert val != null : txEntry;
- cached.innerSet(this,
+ GridCacheUpdateTxResult updRes = cached.innerSet(this,
eventNodeId(),
nodeId,
val,
@@ -627,6 +628,9 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
dhtVer,
txEntry.updateCounter());
+ if (updRes.loggedPointer() != null)
+ ptr = updRes.loggedPointer();
+
// Keep near entry up to date.
if (nearCached != null) {
CacheObject val0 = cached.valueBytes();
@@ -641,7 +645,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
}
}
else if (op == DELETE) {
- cached.innerRemove(this,
+ GridCacheUpdateTxResult updRes = cached.innerRemove(this,
eventNodeId(),
nodeId,
false,
@@ -659,6 +663,9 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
dhtVer,
txEntry.updateCounter());
+ if (updRes.loggedPointer() != null)
+ ptr = updRes.loggedPointer();
+
// Keep near entry up to date.
if (nearCached != null)
nearCached.updateOrEvict(xidVer, null, 0, 0, nodeId, topVer);
@@ -751,7 +758,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
if (!near() && !F.isEmpty(dataEntries) && cctx.wal() != null)
cctx.wal().log(new DataRecord(dataEntries));
- if (ptr != null)
+ if (ptr != null && !cctx.tm().logTxRecords())
cctx.wal().fsync(ptr);
}
catch (StorageException e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
index dcc08d4..f793328 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAssignmentFetchFuture.java
@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.GridNodeOrderComparator;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
import org.apache.ignite.internal.IgniteNeedReconnectException;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.managers.discovery.DiscoCache;
@@ -105,7 +105,7 @@ public class GridDhtAssignmentFetchFuture extends GridFutureAdapter<GridDhtAffin
tmp.add(node);
}
- Collections.sort(tmp, GridNodeOrderComparator.INSTANCE);
+ Collections.sort(tmp, NodeOrderComparator.getInstance());
this.availableNodes = tmp;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 28f9c76..953b47d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -647,6 +647,8 @@ public abstract class GridDhtCacheAdapter<K, V> extends GridDistributedCacheAdap
if (part.reserve()) {
GridCacheEntryEx entry = null;
+ ctx.shared().database().checkpointReadLock();
+
try {
long ttl = CU.ttlForLoad(plc);
@@ -678,6 +680,8 @@ public abstract class GridDhtCacheAdapter<K, V> extends GridDistributedCacheAdap
entry.context().evicts().touch(entry, topVer);
part.release();
+
+ ctx.shared().database().checkpointReadUnlock();
}
}
else if (log.isDebugEnabled())
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 62e2b12..cd5a951 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -213,6 +213,9 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
// TODO ignite-db
throw new IgniteException(e);
}
+
+ // Todo log moving state
+ casState(state.get(), MOVING);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index e09d611..9ccaecf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache.distributed.dht;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -38,6 +39,7 @@ import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.EventType;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
@@ -49,6 +51,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.util.F0;
import org.apache.ignite.internal.util.GridAtomicLong;
import org.apache.ignite.internal.util.GridPartitionStateMap;
@@ -240,7 +244,8 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
try {
AffinityTopologyVersion exchTopVer = exchFut.initialVersion();
- assert exchTopVer.compareTo(readyTopVer) > 0 : "Invalid topology version [topVer=" + readyTopVer +
+ assert exchTopVer.compareTo(readyTopVer) > 0 : "Invalid topology version [grp=" + grp.cacheOrGroupName() +
+ ", topVer=" + readyTopVer +
", exchTopVer=" + exchTopVer +
", fut=" + exchFut + ']';
@@ -354,16 +359,27 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
assert exchId.isJoined() || added;
for (int p = 0; p < num; p++) {
- if (localNode(p, aff)) {
+ IgnitePageStoreManager storeMgr = ctx.pageStore();
+
+ if (localNode(p, aff)
+ || (storeMgr instanceof FilePageStoreManager
+ && grp.persistenceEnabled()
+ && Files.exists(((FilePageStoreManager)storeMgr).getPath(grp.sharedGroup(), grp.cacheOrGroupName(), p)))) {
GridDhtLocalPartition locPart = createPartition(p);
- boolean owned = locPart.own();
+ if (grp.persistenceEnabled()) {
+ GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)grp.shared().database();
- assert owned : "Failed to own partition for oldest node [grp=" + grp.cacheOrGroupName() +
- ", part=" + locPart + ']';
+ locPart.restoreState(db.readPartitionState(grp, locPart.id()));
+ }
+ else {
+ boolean owned = locPart.own();
- if (log.isDebugEnabled()) {
- log.debug("Owned partition for oldest node [grp=" + grp.cacheOrGroupName() +
+ assert owned : "Failed to own partition for oldest node [grp=" + grp.cacheOrGroupName() +
+ ", part=" + locPart + ']';
+
+ if (log.isDebugEnabled())
+ log.debug("Owned partition for oldest node [grp=" + grp.cacheOrGroupName() +
", part=" + locPart + ']');
}
@@ -782,7 +798,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
try {
GridDhtLocalPartition part = locParts.get(p);
- if (part != null)
+ if (part != null && part.state() != EVICTED)
return part;
part = new GridDhtLocalPartition(ctx, grp, p);
@@ -1109,10 +1125,12 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
* @param states Additional partition states.
* @return List of nodes for the partition.
*/
- private List<ClusterNode> nodes(int p,
+ private List<ClusterNode> nodes(
+ int p,
AffinityTopologyVersion topVer,
GridDhtPartitionState state,
- GridDhtPartitionState... states) {
+ GridDhtPartitionState... states
+ ) {
Collection<UUID> allIds = F.nodeIds(discoCache.cacheGroupAffinityNodes(grp.groupId()));
lock.readLock().lock();
@@ -1534,13 +1552,26 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (part == null)
continue;
- long updCntr = cntrMap.updateCounter(part.id());
+ boolean reserve = part.reserve();
+
+ try {
+ GridDhtPartitionState state = part.state();
+
+ if (!reserve || state == EVICTED || state == RENTING)
+ continue;
+
+ long updCntr = cntrMap.updateCounter(part.id());
- if (updCntr > part.updateCounter())
- part.updateCounter(updCntr);
- else if (part.updateCounter() > 0) {
- cntrMap.initialUpdateCounter(part.id(), part.initialUpdateCounter());
- cntrMap.updateCounter(part.id(), part.updateCounter());
+ if (updCntr > part.updateCounter())
+ part.updateCounter(updCntr);
+ else if (part.updateCounter() > 0) {
+ cntrMap.initialUpdateCounter(part.id(), part.initialUpdateCounter());
+ cntrMap.updateCounter(part.id(), part.updateCounter());
+ }
+ }
+ finally {
+ if (reserve)
+ part.release();
}
}
}
@@ -1868,17 +1899,6 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (locPart != null) {
boolean marked = plc == PartitionLossPolicy.IGNORE ? locPart.own() : locPart.markLost();
- if (!marked && locPart.state() == RENTING)
- try {
- //TODO https://issues.apache.org/jira/browse/IGNITE-6433
- locPart.tryEvict();
- locPart.rent(false).get();
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to wait for RENTING partition eviction after partition LOST event",
- e);
- }
-
if (marked)
updateLocal(locPart.id(), locPart.state(), updSeq, resTopVer);
@@ -2067,6 +2087,9 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
* @return Checks if any of the local partitions need to be evicted.
*/
private boolean checkEvictions(long updateSeq, AffinityAssignment aff) {
+ if (!ctx.kernalContext().state().evictionsAllowed())
+ return false;
+
boolean changed = false;
UUID locId = ctx.localNodeId();
@@ -2105,7 +2128,7 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
int ownerCnt = nodeIds.size();
int affCnt = affNodes.size();
- if (ownerCnt > affCnt) {
+ if (ownerCnt > affCnt) { //TODO !!! we could loss all owners in such case. Should be fixed by GG-13223
// Sort by node orders in ascending order.
Collections.sort(nodes, CU.nodeComparator(true));
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
index d04870a..3c3150a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
@@ -80,7 +80,7 @@ public abstract class GridDhtTopologyFutureAdapter extends GridFutureAdapter<Aff
if (err != null)
return err;
- if (!cctx.shared().kernalContext().state().publicApiActiveState())
+ if (!cctx.shared().kernalContext().state().publicApiActiveState(true))
return new CacheInvalidStateException(
"Failed to perform cache operation (cluster is not activated): " + cctx.name());
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
index e4a7141..c7282a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
@@ -474,13 +474,7 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
IgniteTxEntry existing = entry(e.txKey());
if (existing != null) {
- // Must keep NOOP operation if received READ because it means that the lock was sent to a backup node.
- if (e.op() == READ) {
- if (existing.op() != NOOP)
- existing.op(e.op());
- }
- else
- existing.op(e.op()); // Absolutely must set operation, as default is DELETE.
+ existing.op(e.op()); // Absolutely must set operation, as default is DELETE.
existing.value(e.value(), e.hasWriteValue(), e.hasReadValue());
existing.entryProcessors(e.entryProcessors());
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index a1f3984..bde15c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -347,6 +347,8 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
+ cctx.database().checkpointReadLock();
+
try {
if ((txEntry.op() == CREATE || txEntry.op() == UPDATE) &&
txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) {
@@ -509,6 +511,9 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
catch (GridCacheEntryRemovedException e) {
assert false : "Got entry removed exception while holding transactional lock on entry [e=" + e + ", cached=" + cached + ']';
}
+ finally {
+ cctx.database().checkpointReadUnlock();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 87d615e..179a8ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -395,10 +395,17 @@ public class GridPartitionedGetFuture<K, V> extends CacheDistributedGetFutureAda
}
boolean fastLocGet = (!forcePrimary || affNodes.get(0).isLocal()) &&
- cctx.allowFastLocalRead(part, affNodes, topVer);
+ cctx.reserveForFastLocalGet(part, topVer);
- if (fastLocGet && localGet(topVer, key, part, locVals))
- return false;
+ if (fastLocGet) {
+ try {
+ if (localGet(topVer, key, part, locVals))
+ return false;
+ }
+ finally {
+ cctx.releaseForFastLocalGet(part, topVer);
+ }
+ }
ClusterNode node = affinityNode(affNodes);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index adb634a..4e34bcb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -335,10 +335,17 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
}
boolean fastLocGet = (!forcePrimary || affNodes.get(0).isLocal()) &&
- cctx.allowFastLocalRead(part, affNodes, topVer);
+ cctx.reserveForFastLocalGet(part, topVer);
- if (fastLocGet && localGet(topVer, part))
- return null;
+ if (fastLocGet) {
+ try {
+ if (localGet(topVer, part))
+ return null;
+ }
+ finally {
+ cctx.releaseForFastLocalGet(part, topVer);
+ }
+ }
ClusterNode affNode = affinityNode(affNodes);
@@ -659,6 +666,8 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
* @param ver Version.
*/
private void setResult(@Nullable CacheObject val, @Nullable GridCacheVersion ver) {
+ cctx.shared().database().checkpointReadLock();
+
try {
assert !skipVals;
@@ -680,6 +689,9 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
catch (Exception e) {
onDone(e);
}
+ finally {
+ cctx.shared().database().checkpointReadUnlock();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
index 4b9aef0..d6956a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
@@ -87,6 +87,7 @@ public class GridNearAtomicFullUpdateRequest extends GridNearAtomicAbstractUpdat
/** Conflict expire times. */
private GridLongList conflictExpireTimes;
+
/** Optional arguments for entry processor. */
@GridDirectTransient
private Object[] invokeArgs;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
index 0648b4e..3536908 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
@@ -18,12 +18,14 @@
package org.apache.ignite.internal.processors.cache.distributed.dht.colocated;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
@@ -76,6 +78,12 @@ public class GridDhtDetachedCacheEntry extends GridDistributedCacheEntry {
}
/** {@inheritDoc} */
+ @Override protected WALPointer logTxUpdate(IgniteInternalTx tx, CacheObject val, long expireTime, long updCntr)
+ throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override protected void removeValue() throws IgniteCheckedException {
// No-op for detached entries, index is updated on primary or backup nodes.
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
index 83c0231..c0de7cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
@@ -181,4 +181,21 @@ public class CachePartitionPartialCountersMap implements Serializable {
return map0;
}
+
+ /** {@inheritDoc} */
+ public String toString() {
+ StringBuilder sb = new StringBuilder("CachePartitionPartialCountersMap {");
+
+ for (int i = 0; i < partIds.length; i++) {
+ sb.append(partIds[i]).append("=(").append(initialUpdCntrs[i]).append(",")
+ .append(updCntrs[i]).append(")");
+
+ if (i != partIds.length - 1)
+ sb.append(", ");
+ }
+
+ sb.append("}");
+
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index d6f15c9..0c2de93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -272,7 +272,7 @@ public class GridDhtPartitionDemander {
long delay = grp.config().getRebalanceDelay();
- if (delay == 0 || force) {
+ if ((delay == 0 || force) && assigns != null) {
final RebalanceFuture oldFut = rebalanceFut;
final RebalanceFuture fut = new RebalanceFuture(grp, assigns, log, cnt);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index a8b195d..cf2c925 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -55,6 +55,7 @@ import org.apache.ignite.internal.events.DiscoveryCustomEvent;
import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.managers.discovery.DiscoCache;
import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -69,6 +70,7 @@ import org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
import org.apache.ignite.internal.processors.cache.StateChangeRequest;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
@@ -76,10 +78,13 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -107,6 +112,8 @@ import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverJoinEvent;
+import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverLeftEvent;
import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap.PARTIAL_COUNTERS_MAP_SINCE;
/**
@@ -245,7 +252,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
/** */
@GridToStringExclude
- private volatile IgniteDhtPartitionsToReloadMap partsToReload = new IgniteDhtPartitionsToReloadMap();
+ private final IgniteDhtPartitionsToReloadMap partsToReload = new IgniteDhtPartitionsToReloadMap();
/** */
private final AtomicBoolean done = new AtomicBoolean();
@@ -458,6 +465,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
return exchActions != null && exchActions.deactivate();
}
+ /** */
+ public boolean changedBaseline() {
+ return exchActions != null && exchActions.changedBaseline();
+ }
+
/**
* @return First event discovery event.
*
@@ -706,8 +718,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
private void initCachesOnLocalJoin() throws IgniteCheckedException {
cctx.activate();
- List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches =
- cctx.cache().cachesToStartOnLocalJoin();
+ LocalJoinCachesContext locJoinCtx = exchActions == null ? null : exchActions.localJoinContext();
+
+ List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches = locJoinCtx == null ? null :
+ locJoinCtx.caches();
if (!cctx.kernalContext().clientNode()) {
List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
@@ -724,7 +738,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
cctx.database().readCheckpointAndRestoreMemory(startDescs);
}
- cctx.cache().startCachesOnLocalJoin(caches, initialVersion());
+ cctx.cache().startCachesOnLocalJoin(locJoinCtx, initialVersion());
}
/**
@@ -797,74 +811,99 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
assert req != null : exchActions;
- if (req.activate()) {
- if (log.isInfoEnabled()) {
- log.info("Start activation process [nodeId=" + cctx.localNodeId() +
- ", client=" + cctx.kernalContext().clientNode() +
- ", topVer=" + initialVersion() + "]");
- }
+ DiscoveryDataClusterState state = cctx.kernalContext().state().clusterState();
- try {
- cctx.activate();
+ if (state.transitionError() != null)
+ changeGlobalStateE = state.transitionError();
- if (!cctx.kernalContext().clientNode()) {
- List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
+ if (req.activeChanged()) {
+ if (req.activate()) {
+ if (log.isInfoEnabled()) {
+ log.info("Start activation process [nodeId=" + cctx.localNodeId() +
+ ", client=" + cctx.kernalContext().clientNode() +
+ ", topVer=" + initialVersion() + "]");
+ }
- for (ExchangeActions.CacheActionData startReq : exchActions.cacheStartRequests()) {
- DynamicCacheDescriptor desc = startReq.descriptor();
+ try {
+ cctx.activate();
- if (CU.isPersistentCache(desc.cacheConfiguration(), cctx.gridConfig().getDataStorageConfiguration()))
- startDescs.add(desc);
+ if (!cctx.kernalContext().clientNode()) {
+ List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
+
+ for (ExchangeActions.CacheActionData startReq : exchActions.cacheStartRequests()) {
+ DynamicCacheDescriptor desc = startReq.descriptor();
+
+ if (CU.isPersistentCache(desc.cacheConfiguration(),
+ cctx.gridConfig().getDataStorageConfiguration()))
+ startDescs.add(desc);
+ }
+
+ cctx.database().readCheckpointAndRestoreMemory(startDescs);
}
- cctx.database().readCheckpointAndRestoreMemory(startDescs);
+ cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
+
+ if (log.isInfoEnabled()) {
+ log.info("Successfully activated caches [nodeId=" + cctx.localNodeId() +
+ ", client=" + cctx.kernalContext().clientNode() +
+ ", topVer=" + initialVersion() + "]");
+ }
}
+ catch (Exception e) {
+ U.error(log, "Failed to activate node components [nodeId=" + cctx.localNodeId() +
+ ", client=" + cctx.kernalContext().clientNode() +
+ ", topVer=" + initialVersion() + "]", e);
- cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
+ changeGlobalStateE = e;
+ if (crd) {
+ synchronized (mux) {
+ changeGlobalStateExceptions.put(cctx.localNodeId(), e);
+ }
+ }
+ }
+ }
+ else {
if (log.isInfoEnabled()) {
- log.info("Successfully activated caches [nodeId=" + cctx.localNodeId() +
+ log.info("Start deactivation process [nodeId=" + cctx.localNodeId() +
", client=" + cctx.kernalContext().clientNode() +
", topVer=" + initialVersion() + "]");
}
- }
- catch (Exception e) {
- U.error(log, "Failed to activate node components [nodeId=" + cctx.localNodeId() +
- ", client=" + cctx.kernalContext().clientNode() +
- ", topVer=" + initialVersion() + "]", e);
- changeGlobalStateE = e;
+ try {
+ cctx.kernalContext().dataStructures().onDeActivate(cctx.kernalContext());
+
+ cctx.kernalContext().service().onDeActivate(cctx.kernalContext());
+
+ cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
- if (crd) {
- synchronized (mux) {
- changeGlobalStateExceptions.put(cctx.localNodeId(), e);
+ if (log.isInfoEnabled()) {
+ log.info("Successfully deactivated data structures, services and caches [" +
+ "nodeId=" + cctx.localNodeId() +
+ ", client=" + cctx.kernalContext().clientNode() +
+ ", topVer=" + initialVersion() + "]");
}
}
+ catch (Exception e) {
+ U.error(log, "Failed to deactivate node components [nodeId=" + cctx.localNodeId() +
+ ", client=" + cctx.kernalContext().clientNode() +
+ ", topVer=" + initialVersion() + "]", e);
+
+ changeGlobalStateE = e;
+ }
}
}
- else {
- if (log.isInfoEnabled()) {
- log.info("Start deactivation process [nodeId=" + cctx.localNodeId() +
- ", client=" + cctx.kernalContext().clientNode() +
- ", topVer=" + initialVersion() + "]");
- }
-
+ else if (req.activate()) {
+ // TODO: BLT changes on inactive cluster can't be handled easily because persistent storage hasn't been initialized yet.
try {
- cctx.kernalContext().dataStructures().onDeActivate(cctx.kernalContext());
+ cctx.affinity().onBaselineTopologyChanged(this, crd);
- cctx.kernalContext().service().onDeActivate(cctx.kernalContext());
-
- cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
-
- if (log.isInfoEnabled()) {
- log.info("Successfully deactivated data structures, services and caches [" +
- "nodeId=" + cctx.localNodeId() +
- ", client=" + cctx.kernalContext().clientNode() +
- ", topVer=" + initialVersion() + "]");
- }
+ if (CU.isPersistenceEnabled(cctx.kernalContext().config()) && !cctx.kernalContext().clientNode())
+ cctx.kernalContext().state().onBaselineTopologyChanged(req.baselineTopology(),
+ req.prevBaselineTopologyHistoryItem());
}
catch (Exception e) {
- U.error(log, "Failed to deactivate node components [nodeId=" + cctx.localNodeId() +
+ U.error(log, "Failed to change baseline topology [nodeId=" + cctx.localNodeId() +
", client=" + cctx.kernalContext().clientNode() +
", topVer=" + initialVersion() + "]", e);
@@ -1021,6 +1060,8 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
+ cctx.database().beforeExchange(this);
+
if (!exchCtx.mergeExchanges()) {
for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
if (grp.isLocal() || cacheGroupStopping(grp.groupId()))
@@ -1032,8 +1073,6 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
- cctx.database().beforeExchange(this);
-
if (crd.isLocal()) {
if (remaining.isEmpty())
onAllReceived(null);
@@ -1521,7 +1560,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
ExchangeDiscoveryEvents evts = exchCtx.events();
for (DiscoveryEvent evt : exchCtx.events().events()) {
- if (evts.serverLeftEvent(evt)) {
+ if (serverLeftEvent(evt)) {
for (CacheGroupContext grp : cctx.cache().cacheGroups())
grp.affinityFunction().removeNode(evt.eventNode().id());
}
@@ -1533,9 +1572,20 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (firstDiscoEvt instanceof DiscoveryCustomEvent)
((DiscoveryCustomEvent)firstDiscoEvt).customMessage(null);
- if (err == null)
+ if (err == null) {
cctx.exchange().lastFinishedFuture(this);
+ if (exchCtx != null && (exchCtx.events().hasServerLeft() || exchCtx.events().hasServerJoin())) {
+ ExchangeDiscoveryEvents evts = exchCtx.events();
+
+ for (DiscoveryEvent evt : exchCtx.events().events()) {
+ if (serverLeftEvent(evt) || serverJoinEvent(evt))
+ logExchange(evt);
+ }
+ }
+
+ }
+
return true;
}
@@ -1543,6 +1593,37 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
/**
+ * Log exchange event.
+ *
+ * @param evt Discovery event.
+ */
+ private void logExchange(DiscoveryEvent evt) {
+ if (cctx.kernalContext().state().publicApiActiveState(false) && cctx.wal() != null) {
+ if (((FileWriteAheadLogManager)cctx.wal()).serializerVersion() > 1)
+ try {
+ ExchangeRecord.Type type = null;
+
+ if (evt.type() == EVT_NODE_JOINED)
+ type = ExchangeRecord.Type.JOIN;
+ else if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
+ type = ExchangeRecord.Type.LEFT;
+
+ BaselineTopology blt = cctx.kernalContext().state().clusterState().baselineTopology();
+
+ if (type != null && blt != null) {
+ Short constId = blt.consistentIdMapping().get(evt.eventNode().consistentId());
+
+ if (constId != null)
+ cctx.wal().log(new ExchangeRecord(constId, type));
+ }
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Fail during log exchange record.", e);
+ }
+ }
+ }
+
+ /**
* Cleans up resources to avoid excessive memory usage.
*/
public void cleanUp() {
@@ -2047,7 +2128,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
if (state != GridDhtPartitionState.OWNING && state != GridDhtPartitionState.MOVING)
continue;
- long cntr = state == GridDhtPartitionState.MOVING ? part.initialUpdateCounter() : part.updateCounter();
+ final long cntr = state == GridDhtPartitionState.MOVING ? part.initialUpdateCounter() : part.updateCounter();
Long minCntr = minCntrs.get(part.id());
@@ -2301,25 +2382,27 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
- for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
- if (!grpCtx.isLocal())
- grpCtx.topology().applyUpdateCounters();
- }
-
if (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
assert firstDiscoEvt instanceof DiscoveryCustomEvent;
- if (activateCluster())
+ if (activateCluster() || changedBaseline())
assignPartitionsStates();
- if (((DiscoveryCustomEvent)firstDiscoEvt).customMessage() instanceof DynamicCacheChangeBatch) {
+ DiscoveryCustomMessage discoveryCustomMessage = ((DiscoveryCustomEvent) firstDiscoEvt).customMessage();
+
+ if (discoveryCustomMessage instanceof DynamicCacheChangeBatch) {
if (exchActions != null) {
+ assignPartitionsStates();
+
Set<String> caches = exchActions.cachesToResetLostPartitions();
if (!F.isEmpty(caches))
resetLostPartitions(caches);
}
}
+ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
+ && ((SnapshotDiscoveryMessage)discoveryCustomMessage).needAssignPartitions())
+ assignPartitionsStates();
}
else {
if (exchCtx.events().hasServerJoin())
@@ -2329,6 +2412,11 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
detectLostPartitions(resTopVer);
}
+ for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
+ if (!grpCtx.isLocal())
+ grpCtx.topology().applyUpdateCounters();
+ }
+
updateLastVersion(cctx.versions().last());
cctx.versions().onExchange(lastVer.get().order());
@@ -2415,12 +2503,31 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
cctx.kernalContext().state().onStateChangeError(changeGlobalStateExceptions, req);
}
+ else {
+ boolean hasMoving = !partsToReload.isEmpty();
+
+ Set<Integer> waitGrps = cctx.affinity().waitGroups();
+
+ if (!hasMoving) {
+ for (CacheGroupContext grpCtx : cctx.cache().cacheGroups()) {
+ if (waitGrps.contains(grpCtx.groupId()) && grpCtx.topology().hasMovingPartitions()) {
+ hasMoving = true;
+
+ break;
+ }
+
+ }
+ }
+
+ cctx.kernalContext().state().onExchangeFinishedOnCoordinator(this, hasMoving);
+ }
boolean active = !stateChangeErr && req.activate();
ChangeGlobalStateFinishMessage stateFinishMsg = new ChangeGlobalStateFinishMessage(
req.requestId(),
- active);
+ active,
+ !stateChangeErr);
cctx.discovery().sendCustomEvent(stateFinishMsg);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
index 2a72e95..37ca7e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
@@ -85,4 +85,11 @@ public class IgniteDhtPartitionsToReloadMap implements Serializable {
parts.add(partId);
}
+
+ /**
+ * @return {@code True} if empty.
+ */
+ public boolean isEmpty() {
+ return map == null || map.isEmpty();
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/StopCachesOnClientReconnectExchangeTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/StopCachesOnClientReconnectExchangeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/StopCachesOnClientReconnectExchangeTask.java
new file mode 100644
index 0000000..e89bbd1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/StopCachesOnClientReconnectExchangeTask.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
+
+import java.util.Collection;
+import org.apache.ignite.internal.processors.cache.CachePartitionExchangeWorkerTask;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class StopCachesOnClientReconnectExchangeTask extends GridFutureAdapter<Void>
+ implements CachePartitionExchangeWorkerTask {
+ /** */
+ @GridToStringInclude
+ private final Collection<GridCacheAdapter> stoppedCaches;
+
+ /**
+ * @param stoppedCaches Collection of stopped caches.
+ */
+ public StopCachesOnClientReconnectExchangeTask(Collection<GridCacheAdapter> stoppedCaches) {
+ this.stoppedCaches = stoppedCaches;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean skipForExchangeMerge() {
+ return false;
+ }
+
+ /**
+ * @return Collection of stopped caches.
+ */
+ public Collection<GridCacheAdapter> stoppedCaches() {
+ return stoppedCaches;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(StopCachesOnClientReconnectExchangeTask.class, this, super.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index c4523ac..322e63c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.distributed.near;
import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheLockCandidates;
import org.apache.ignite.internal.processors.cache.CacheObject;
@@ -473,6 +474,12 @@ public class GridNearCacheEntry extends GridDistributedCacheEntry {
}
/** {@inheritDoc} */
+ @Override protected WALPointer logTxUpdate(IgniteInternalTx tx, CacheObject val, long expireTime, long updCntr)
+ throws IgniteCheckedException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public CacheDataRow unswap(CacheDataRow row, boolean checkExpire) {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index b175018..29dd12f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -478,10 +478,17 @@ public final class GridNearGetFuture<K, V> extends CacheDistributedGetFutureAdap
}
if (v == null) {
- boolean fastLocGet = allowLocRead && cctx.allowFastLocalRead(part, affNodes, topVer);
+ boolean fastLocGet = allowLocRead && cctx.reserveForFastLocalGet(part, topVer);
- if (fastLocGet && localDhtGet(key, part, topVer, isNear))
- break;
+ if (fastLocGet) {
+ try {
+ if (localDhtGet(key, part, topVer, isNear))
+ break;
+ }
+ finally {
+ cctx.releaseForFastLocalGet(part, topVer);
+ }
+ }
ClusterNode affNode = affinityNode(affNodes);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 6cf076b..8d8c0b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -30,6 +30,8 @@ import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.cluster.ClusterTopologyException;
+import org.apache.ignite.internal.IgniteDiagnosticAware;
+import org.apache.ignite.internal.IgniteDiagnosticPrepareContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
@@ -45,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.transactions.TxDeadlock;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
@@ -71,7 +74,8 @@ import static org.apache.ignite.transactions.TransactionState.PREPARING;
/**
*
*/
-public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepareFutureAdapter {
+public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepareFutureAdapter implements
+ IgniteDiagnosticAware {
/** */
@GridToStringExclude
private KeyLockFuture keyLockFut;
@@ -783,6 +787,61 @@ public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepa
}
/** {@inheritDoc} */
+ @Override public void addDiagnosticRequest(IgniteDiagnosticPrepareContext ctx) {
+ if (!isDone()) {
+ for (IgniteInternalFuture<GridNearTxPrepareResponse> fut : futures()) {
+ if (!fut.isDone()) {
+ if (fut instanceof MiniFuture) {
+ MiniFuture miniFut = (MiniFuture)fut;
+
+ UUID nodeId = miniFut.node().id();
+ GridCacheVersion dhtVer = miniFut.m.dhtVersion();
+ GridCacheVersion nearVer = tx.nearXidVersion();
+
+ if (dhtVer != null) {
+ ctx.remoteTxInfo(
+ nodeId,
+ dhtVer,
+ nearVer,
+ "GridNearOptimisticTxPrepareFuture waiting for remote node response [" +
+ "nodeId=" + nodeId +
+ ", topVer=" + tx.topologyVersion() +
+ ", dhtVer=" + dhtVer +
+ ", nearVer=" + nearVer +
+ ", futId=" + futId +
+ ", miniId=" + miniFut.futId +
+ ", tx=" + tx + ']');
+ }
+ else {
+ ctx.basicInfo(
+ cctx.localNodeId(),
+ "GridNearOptimisticTxPrepareFuture waiting for remote node response [" +
+ "nodeId=" + nodeId +
+ ", topVer=" + tx.topologyVersion() +
+ ", dhtVer=" + dhtVer +
+ ", nearVer=" + nearVer +
+ ", futId=" + futId +
+ ", miniId=" + miniFut.futId +
+ ", tx=" + tx + ']');
+ }
+ }
+ else if (fut instanceof KeyLockFuture) {
+ KeyLockFuture keyFut = (KeyLockFuture)fut;
+
+ ctx.basicInfo(
+ cctx.localNodeId(),
+ "GridNearOptimisticTxPrepareFuture waiting for local keys lock [" +
+ "node=" + cctx.localNodeId() +
+ ", topVer=" + tx.topologyVersion() +
+ ", allKeysAdded=" + keyFut.allKeysAdded +
+ ", keys=" + keyFut.lockKeys + ']');
+ }
+ }
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
Collection<String> futs = F.viewReadOnly(futures(), new C1<IgniteInternalFuture<?>, String>() {
@Override public String apply(IgniteInternalFuture<?> f) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
index f09b6c8..780674e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
@@ -177,7 +177,7 @@ public abstract class GridNearOptimisticTxPrepareFutureAdapter extends GridNearT
protected Collection<IgniteTxKey> lockKeys = new GridConcurrentHashSet<>();
/** */
- private volatile boolean allKeysAdded;
+ protected volatile boolean allKeysAdded;
/**
* @param key Key to track for locking.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareFutureAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareFutureAdapter.java
index ea002f6..1be84c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareFutureAdapter.java
@@ -27,8 +27,8 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.processors.cache.GridCacheCompoundFuture;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
-import org.apache.ignite.internal.processors.cache.GridCacheVersionedFuture;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.GridCacheVersionedFuture;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxMapping;
import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtDetachedCacheEntry;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
index 57aeaef..44f0a3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
@@ -24,7 +24,7 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
/**
* Cache data row.
*/
-public interface CacheDataRow extends CacheSearchRow {
+public interface CacheDataRow extends CacheSearchRow, Storable {
/**
* @return Cache value.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
index ef148ba..06fe9fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
@@ -72,7 +72,7 @@ public class DataRegionMetricsImpl implements DataRegionMetrics {
/**
* @param memPlcCfg DataRegionConfiguration.
- */
+ */
public DataRegionMetricsImpl(DataRegionConfiguration memPlcCfg) {
this(memPlcCfg, null);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
index 9161d69..1d4b812 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
@@ -124,7 +124,8 @@ public abstract class DataStructure implements PageLockListener {
*/
protected final long acquirePage(long pageId) throws IgniteCheckedException {
assert PageIdUtils.flag(pageId) == FLAG_IDX && PageIdUtils.partId(pageId) == INDEX_PARTITION ||
- PageIdUtils.flag(pageId) == FLAG_DATA && PageIdUtils.partId(pageId) <= MAX_PARTITION_ID : U.hexLong(pageId);
+ PageIdUtils.flag(pageId) == FLAG_DATA && PageIdUtils.partId(pageId) <= MAX_PARTITION_ID :
+ U.hexLong(pageId) + " flag=" + PageIdUtils.flag(pageId) + " part=" + PageIdUtils.partId(pageId);
return pageMem.acquirePage(grpId, pageId);
}
[09/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
index 173ed66..8a04749 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
@@ -18,74 +18,23 @@
package org.apache.ignite.internal.processors.cache.persistence.tree.io;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.GridStringBuilder;
-import org.apache.ignite.internal.util.typedef.internal.SB;
-import org.jetbrains.annotations.Nullable;
/**
* Data pages IO.
*/
-public class DataPageIO extends PageIO {
+public class DataPageIO extends AbstractDataPageIO<CacheDataRow> {
/** */
public static final IOVersions<DataPageIO> VERSIONS = new IOVersions<>(
new DataPageIO(1)
);
- /** */
- private static final int SHOW_ITEM = 0b0001;
-
- /** */
- private static final int SHOW_PAYLOAD_LEN = 0b0010;
-
- /** */
- private static final int SHOW_LINK = 0b0100;
-
- /** */
- private static final int FREE_LIST_PAGE_ID_OFF = COMMON_HEADER_END;
-
- /** */
- private static final int FREE_SPACE_OFF = FREE_LIST_PAGE_ID_OFF + 8;
-
- /** */
- private static final int DIRECT_CNT_OFF = FREE_SPACE_OFF + 2;
-
- /** */
- private static final int INDIRECT_CNT_OFF = DIRECT_CNT_OFF + 1;
-
- /** */
- private static final int FIRST_ENTRY_OFF = INDIRECT_CNT_OFF + 1;
-
- /** Offset of items (internal page pointers) within data page */
- public static final int ITEMS_OFF = FIRST_ENTRY_OFF + 2;
-
- /** */
- private static final int ITEM_SIZE = 2;
-
- /** */
- private static final int PAYLOAD_LEN_SIZE = 2;
-
- /** */
- private static final int LINK_SIZE = 8;
-
- /** */
- private static final int FRAGMENTED_FLAG = 0b10000000_00000000;
-
- /** */
- public static final int MIN_DATA_PAGE_OVERHEAD = ITEMS_OFF + ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
-
/**
* @param ver Page format version.
*/
@@ -94,959 +43,8 @@ public class DataPageIO extends PageIO {
}
/** {@inheritDoc} */
- @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
- super.initNewPage(pageAddr, pageId, pageSize);
-
- setEmptyPage(pageAddr, pageSize);
- setFreeListPageId(pageAddr, 0L);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageSize Page size.
- */
- private void setEmptyPage(long pageAddr, int pageSize) {
- setDirectCount(pageAddr, 0);
- setIndirectCount(pageAddr, 0);
- setFirstEntryOffset(pageAddr, pageSize, pageSize);
- setRealFreeSpace(pageAddr, pageSize - ITEMS_OFF, pageSize);
- }
-
- /**
- * @param pageAddr Page address.
- * @param freeListPageId Free list page ID.
- */
- public void setFreeListPageId(long pageAddr, long freeListPageId) {
- PageUtils.putLong(pageAddr, FREE_LIST_PAGE_ID_OFF, freeListPageId);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Free list page ID.
- */
- public long getFreeListPageId(long pageAddr) {
- return PageUtils.getLong(pageAddr, FREE_LIST_PAGE_ID_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Data offset.
- * @param show What elements of data page entry to show in the result.
- * @return Data page entry size.
- */
- private int getPageEntrySize(long pageAddr, int dataOff, int show) {
- int payloadLen = PageUtils.getShort(pageAddr, dataOff) & 0xFFFF;
-
- if ((payloadLen & FRAGMENTED_FLAG) != 0)
- payloadLen &= ~FRAGMENTED_FLAG; // We are fragmented and have a link.
- else
- show &= ~SHOW_LINK; // We are not fragmented, never have a link.
-
- return getPageEntrySize(payloadLen, show);
- }
-
- /**
- * @param payloadLen Length of the payload, may be a full data row or a row fragment length.
- * @param show What elements of data page entry to show in the result.
- * @return Data page entry size.
- */
- private int getPageEntrySize(int payloadLen, int show) {
- assert payloadLen > 0 : payloadLen;
-
- int res = payloadLen;
-
- if ((show & SHOW_LINK) != 0)
- res += LINK_SIZE;
-
- if ((show & SHOW_ITEM) != 0)
- res += ITEM_SIZE;
-
- if ((show & SHOW_PAYLOAD_LEN) != 0)
- res += PAYLOAD_LEN_SIZE;
-
- return res;
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Entry data offset.
- * @param pageSize Page size.
- */
- private void setFirstEntryOffset(long pageAddr, int dataOff, int pageSize) {
- assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff <= pageSize : dataOff;
-
- PageUtils.putShort(pageAddr, FIRST_ENTRY_OFF, (short)dataOff);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Entry data offset.
- */
- private int getFirstEntryOffset(long pageAddr) {
- return PageUtils.getShort(pageAddr, FIRST_ENTRY_OFF) & 0xFFFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @param freeSpace Free space.
- * @param pageSize Page size.
- */
- private void setRealFreeSpace(long pageAddr, int freeSpace, int pageSize) {
- assert freeSpace == actualFreeSpace(pageAddr, pageSize) : freeSpace + " != " + actualFreeSpace(pageAddr, pageSize);
-
- PageUtils.putShort(pageAddr, FREE_SPACE_OFF, (short)freeSpace);
- }
-
- /**
- * Free space refers to a "max row size (without any data page specific overhead) which is
- * guaranteed to fit into this data page".
- *
- * @param pageAddr Page address.
- * @return Free space.
- */
- public int getFreeSpace(long pageAddr) {
- if (getFreeItemSlots(pageAddr) == 0)
- return 0;
-
- int freeSpace = getRealFreeSpace(pageAddr);
-
- // We reserve size here because of getFreeSpace() method semantics (see method javadoc).
- // It means that we must be able to accommodate a row of size which is equal to getFreeSpace(),
- // plus we will have data page overhead: header of the page as well as item, payload length and
- // possibly a link to the next row fragment.
- freeSpace -= ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
-
- return freeSpace < 0 ? 0 : freeSpace;
- }
-
- /**
- * @param pageAddr Page address.
- * @return {@code true} If there is no useful data in this page.
- */
- public boolean isEmpty(long pageAddr) {
- return getDirectCount(pageAddr) == 0;
- }
-
- /**
- * Equivalent for {@link #actualFreeSpace(long, int)} but reads saved value.
- *
- * @param pageAddr Page address.
- * @return Free space.
- */
- private int getRealFreeSpace(long pageAddr) {
- return PageUtils.getShort(pageAddr, FREE_SPACE_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param cnt Direct count.
- */
- private void setDirectCount(long pageAddr, int cnt) {
- assert checkCount(cnt): cnt;
-
- PageUtils.putByte(pageAddr, DIRECT_CNT_OFF, (byte)cnt);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Direct count.
- */
- private int getDirectCount(long pageAddr) {
- return PageUtils.getByte(pageAddr, DIRECT_CNT_OFF) & 0xFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @param c Closure.
- * @param <T> Closure return type.
- * @return Collection of closure results for all items in page.
- * @throws IgniteCheckedException In case of error in closure body.
- */
- public <T> List<T> forAllItems(long pageAddr, CC<T> c) throws IgniteCheckedException {
- long pageId = getPageId(pageAddr);
-
- int cnt = getDirectCount(pageAddr);
-
- List<T> res = new ArrayList<>(cnt);
-
- for (int i = 0; i < cnt; i++) {
- long link = PageIdUtils.link(pageId, i);
-
- res.add(c.apply(link));
- }
-
- return res;
- }
-
- /**
- * @param pageAddr Page address.
- * @param cnt Indirect count.
- */
- private void setIndirectCount(long pageAddr, int cnt) {
- assert checkCount(cnt): cnt;
-
- PageUtils.putByte(pageAddr, INDIRECT_CNT_OFF, (byte)cnt);
- }
-
- /**
- * @param idx Index.
- * @return {@code true} If the index is valid.
- */
- private boolean checkIndex(int idx) {
- return idx >= 0 && idx < 0xFF;
- }
-
- /**
- * @param cnt Counter value.
- * @return {@code true} If the counter fits 1 byte.
- */
- private boolean checkCount(int cnt) {
- return cnt >= 0 && cnt <= 0xFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Indirect count.
- */
- private int getIndirectCount(long pageAddr) {
- return PageUtils.getByte(pageAddr, INDIRECT_CNT_OFF) & 0xFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Number of free entry slots.
- */
- private int getFreeItemSlots(long pageAddr) {
- return 0xFF - getDirectCount(pageAddr);
- }
-
- /**
- * @param pageAddr Page address.
- * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @return Found index of indirect item.
- */
- private int findIndirectItemIndex(long pageAddr, int itemId, int directCnt, int indirectCnt) {
- int low = directCnt;
- int high = directCnt + indirectCnt - 1;
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
-
- int cmp = Integer.compare(itemId(getItem(pageAddr, mid)), itemId);
-
- if (cmp < 0)
- low = mid + 1;
- else if (cmp > 0)
- high = mid - 1;
- else
- return mid; // found
- }
-
- throw new IllegalStateException("Item not found: " + itemId);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageSize Page size.
- * @return String representation.
- */
- private String printPageLayout(long pageAddr, int pageSize) {
- SB b = new SB();
-
- printPageLayout(pageAddr, pageSize, b);
-
- return b.toString();
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageSize Page size.
- * @param b B.
- */
- private void printPageLayout(long pageAddr, int pageSize, GridStringBuilder b) {
- int directCnt = getDirectCount(pageAddr);
- int indirectCnt = getIndirectCount(pageAddr);
- int free = getRealFreeSpace(pageAddr);
-
- boolean valid = directCnt >= indirectCnt;
-
- b.appendHex(PageIO.getPageId(pageAddr)).a(" [");
-
- int entriesSize = 0;
-
- for (int i = 0; i < directCnt; i++) {
- if (i != 0)
- b.a(", ");
-
- short item = getItem(pageAddr, i);
-
- if (item < ITEMS_OFF || item >= pageSize)
- valid = false;
-
- entriesSize += getPageEntrySize(pageAddr, item, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
- b.a(item);
- }
-
- b.a("][");
-
- Collection<Integer> set = new HashSet<>();
-
- for (int i = directCnt; i < directCnt + indirectCnt; i++) {
- if (i != directCnt)
- b.a(", ");
-
- short item = getItem(pageAddr, i);
-
- int itemId = itemId(item);
- int directIdx = directItemIndex(item);
-
- if (!set.add(directIdx) || !set.add(itemId))
- valid = false;
-
- assert indirectItem(itemId, directIdx) == item;
-
- if (itemId < directCnt || directIdx < 0 || directIdx >= directCnt)
- valid = false;
-
- if (i > directCnt && itemId(getItem(pageAddr, i - 1)) >= itemId)
- valid = false;
-
-
- b.a(itemId).a('^').a(directIdx);
- }
-
- b.a("][free=").a(free);
-
- int actualFree = pageSize - ITEMS_OFF - (entriesSize + (directCnt + indirectCnt) * ITEM_SIZE);
-
- if (free != actualFree) {
- b.a(", actualFree=").a(actualFree);
-
- valid = false;
- }
- else
- b.a("]");
-
- assert valid : b.toString();
- }
-
- /**
- * @param pageAddr Page address.
- * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
- * @param pageSize Page size.
- * @return Data entry offset in bytes.
- */
- private int getDataOffset(long pageAddr, int itemId, int pageSize) {
- assert checkIndex(itemId): itemId;
-
- int directCnt = getDirectCount(pageAddr);
-
- assert directCnt > 0: "itemId=" + itemId + ", directCnt=" + directCnt + ", page=" + printPageLayout(pageAddr, pageSize);
-
- if (itemId >= directCnt) { // Need to do indirect lookup.
- int indirectCnt = getIndirectCount(pageAddr);
-
- // Must have indirect items here.
- assert indirectCnt > 0: "itemId=" + itemId + ", directCnt=" + directCnt + ", indirectCnt=" + indirectCnt +
- ", page=" + printPageLayout(pageAddr, pageSize);
-
- int indirectItemIdx = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
-
- assert indirectItemIdx >= directCnt : indirectItemIdx + " " + directCnt;
- assert indirectItemIdx < directCnt + indirectCnt: indirectItemIdx + " " + directCnt + " " + indirectCnt;
-
- itemId = directItemIndex(getItem(pageAddr, indirectItemIdx));
-
- assert itemId >= 0 && itemId < directCnt: itemId + " " + directCnt + " " + indirectCnt; // Direct item.
- }
-
- return directItemToOffset(getItem(pageAddr, itemId));
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Points to the entry start.
- * @return Link to the next entry fragment or 0 if no fragments left or if entry is not fragmented.
- */
- private long getNextFragmentLink(long pageAddr, int dataOff) {
- assert isFragmented(pageAddr, dataOff);
-
- return PageUtils.getLong(pageAddr, dataOff + PAYLOAD_LEN_SIZE);
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Data offset.
- * @return {@code true} If the data row is fragmented across multiple pages.
- */
- private boolean isFragmented(long pageAddr, int dataOff) {
- return (PageUtils.getShort(pageAddr, dataOff) & FRAGMENTED_FLAG) != 0;
- }
-
- /**
- * Sets position to start of actual fragment data and limit to it's end.
- *
- * @param pageAddr Page address.
- * @param itemId Item to position on.
- * @param pageSize Page size.
- * @return {@link DataPagePayload} object.
- */
- public DataPagePayload readPayload(final long pageAddr, final int itemId, final int pageSize) {
- int dataOff = getDataOffset(pageAddr, itemId, pageSize);
-
- boolean fragmented = isFragmented(pageAddr, dataOff);
- long nextLink = fragmented ? getNextFragmentLink(pageAddr, dataOff) : 0;
- int payloadSize = getPageEntrySize(pageAddr, dataOff, 0);
-
- return new DataPagePayload(dataOff + PAYLOAD_LEN_SIZE + (fragmented ? LINK_SIZE : 0),
- payloadSize,
- nextLink);
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Item index.
- * @return Item.
- */
- private short getItem(long pageAddr, int idx) {
- return PageUtils.getShort(pageAddr, itemOffset(idx));
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Item index.
- * @param item Item.
- */
- private void setItem(long pageAddr, int idx, short item) {
- PageUtils.putShort(pageAddr, itemOffset(idx), item);
- }
-
- /**
- * @param idx Index of the item.
- * @return Offset in buffer.
- */
- private int itemOffset(int idx) {
- assert checkIndex(idx): idx;
-
- return ITEMS_OFF + idx * ITEM_SIZE;
- }
-
- /**
- * @param directItem Direct item.
- * @return Offset of an entry payload inside of the page.
- */
- private int directItemToOffset(short directItem) {
- return directItem & 0xFFFF;
- }
-
- /**
- * @param dataOff Data offset.
- * @return Direct item.
- */
- private short directItemFromOffset(int dataOff) {
- assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff < Short.MAX_VALUE: dataOff;
-
- return (short)dataOff;
- }
-
- /**
- * @param indirectItem Indirect item.
- * @return Index of corresponding direct item.
- */
- private int directItemIndex(short indirectItem) {
- return indirectItem & 0xFF;
- }
-
- /**
- * @param indirectItem Indirect item.
- * @return Fixed item ID (the index used for referencing an entry from the outside).
- */
- private int itemId(short indirectItem) {
- return (indirectItem & 0xFFFF) >>> 8;
- }
-
- /**
- * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
- * @param directItemIdx Index of corresponding direct item.
- * @return Indirect item.
- */
- private short indirectItem(int itemId, int directItemIdx) {
- assert checkIndex(itemId): itemId;
- assert checkIndex(directItemIdx): directItemIdx;
-
- return (short)((itemId << 8) | directItemIdx);
- }
-
- /**
- * Move the last direct item to the free slot and reference it with indirect item on the same place.
- *
- * @param pageAddr Page address.
- * @param freeDirectIdx Free slot.
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @return {@code true} If the last direct item already had corresponding indirect item.
- */
- private boolean moveLastItem(long pageAddr, int freeDirectIdx, int directCnt, int indirectCnt) {
- int lastIndirectId = findIndirectIndexForLastDirect(pageAddr, directCnt, indirectCnt);
-
- int lastItemId = directCnt - 1;
-
- assert lastItemId != freeDirectIdx;
-
- short indirectItem = indirectItem(lastItemId, freeDirectIdx);
-
- assert itemId(indirectItem) == lastItemId && directItemIndex(indirectItem) == freeDirectIdx;
-
- setItem(pageAddr, freeDirectIdx, getItem(pageAddr, lastItemId));
- setItem(pageAddr, lastItemId, indirectItem);
-
- assert getItem(pageAddr, lastItemId) == indirectItem;
-
- if (lastIndirectId != -1) { // Fix pointer to direct item.
- setItem(pageAddr, lastIndirectId, indirectItem(itemId(getItem(pageAddr, lastIndirectId)), freeDirectIdx));
-
- return true;
- }
-
- return false;
- }
-
- /**
- * @param pageAddr Page address.
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @return Index of indirect item for the last direct item.
- */
- private int findIndirectIndexForLastDirect(long pageAddr, int directCnt, int indirectCnt) {
- int lastDirectId = directCnt - 1;
-
- for (int i = directCnt, end = directCnt + indirectCnt; i < end; i++) {
- short item = getItem(pageAddr, i);
-
- if (directItemIndex(item) == lastDirectId)
- return i;
- }
-
- return -1;
- }
-
- /**
- * @param pageAddr Page address.
- * @param itemId Item ID.
- * @param pageSize Page size.
- * @param payload Row data.
- * @param row Row.
- * @param rowSize Row size.
- * @return {@code True} if entry is not fragmented.
- * @throws IgniteCheckedException If failed.
- */
- public boolean updateRow(
- final long pageAddr,
- int itemId,
- int pageSize,
- @Nullable byte[] payload,
- @Nullable CacheDataRow row,
- final int rowSize) throws IgniteCheckedException {
- assert checkIndex(itemId) : itemId;
- assert row != null ^ payload != null;
-
- final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
-
- if (isFragmented(pageAddr, dataOff))
- return false;
-
- if (row != null)
- writeRowData(pageAddr, dataOff, rowSize, row, false);
- else
- writeRowData(pageAddr, dataOff, payload);
-
- return true;
- }
-
- /**
- * @param pageAddr Page address.
- * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
- * @param pageSize Page size.
- * @return Next link for fragmented entries or {@code 0} if none.
- * @throws IgniteCheckedException If failed.
- */
- public long removeRow(long pageAddr, int itemId, int pageSize) throws IgniteCheckedException {
- assert checkIndex(itemId) : itemId;
-
- final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
- final long nextLink = isFragmented(pageAddr, dataOff) ? getNextFragmentLink(pageAddr, dataOff) : 0;
-
- // Record original counts to calculate delta in free space in the end of remove.
- final int directCnt = getDirectCount(pageAddr);
- final int indirectCnt = getIndirectCount(pageAddr);
-
- int curIndirectCnt = indirectCnt;
-
- assert directCnt > 0 : directCnt; // Direct count always represents overall number of live items.
-
- // Remove the last item on the page.
- if (directCnt == 1) {
- assert (indirectCnt == 0 && itemId == 0) ||
- (indirectCnt == 1 && itemId == itemId(getItem(pageAddr, 1))) : itemId;
-
- setEmptyPage(pageAddr, pageSize);
- }
- else {
- // Get the entry size before the actual remove.
- int rmvEntrySize = getPageEntrySize(pageAddr, dataOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
- int indirectId = 0;
-
- if (itemId >= directCnt) { // Need to remove indirect item.
- assert indirectCnt > 0;
-
- indirectId = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
-
- assert indirectId >= directCnt;
-
- itemId = directItemIndex(getItem(pageAddr, indirectId));
-
- assert itemId < directCnt;
- }
-
- boolean dropLast = true;
-
- if (itemId + 1 < directCnt) // It is not the last direct item.
- dropLast = moveLastItem(pageAddr, itemId, directCnt, indirectCnt);
-
- if (indirectId == 0) {// For the last direct item with no indirect item.
- if (dropLast)
- moveItems(pageAddr, directCnt, indirectCnt, -1, pageSize);
- else
- curIndirectCnt++;
- }
- else {
- if (dropLast)
- moveItems(pageAddr, directCnt, indirectId - directCnt, -1, pageSize);
-
- moveItems(pageAddr, indirectId + 1, directCnt + indirectCnt - indirectId - 1, dropLast ? -2 : -1, pageSize);
-
- if (dropLast)
- curIndirectCnt--;
- }
-
- setIndirectCount(pageAddr, curIndirectCnt);
- setDirectCount(pageAddr, directCnt - 1);
-
- assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
-
- // Increase free space.
- setRealFreeSpace(pageAddr,
- getRealFreeSpace(pageAddr) + rmvEntrySize + ITEM_SIZE * (directCnt - getDirectCount(pageAddr) + indirectCnt - getIndirectCount(pageAddr)),
- pageSize);
- }
-
- return nextLink;
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Index.
- * @param cnt Count.
- * @param step Step.
- * @param pageSize Page size.
- */
- private void moveItems(long pageAddr, int idx, int cnt, int step, int pageSize) {
- assert cnt >= 0: cnt;
-
- if (cnt != 0)
- moveBytes(pageAddr, itemOffset(idx), cnt * ITEM_SIZE, step * ITEM_SIZE, pageSize);
- }
-
- /**
- * @param newEntryFullSize New entry full size (with item, length and link).
- * @param firstEntryOff First entry data offset.
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @return {@code true} If there is enough space for the entry.
- */
- private boolean isEnoughSpace(int newEntryFullSize, int firstEntryOff, int directCnt, int indirectCnt) {
- return ITEMS_OFF + ITEM_SIZE * (directCnt + indirectCnt) <= firstEntryOff - newEntryFullSize;
- }
-
- /**
- * Adds row to this data page and sets respective link to the given row object.
- *
- * @param pageId page ID.
- * @param pageAddr Page address.
- * @param row Cache data row.
- * @param rowSize Row size.
- * @param pageSize Page size.
- * @throws IgniteCheckedException If failed.
- */
- public void addRow(
- final long pageId,
- final long pageAddr,
- CacheDataRow row,
- final int rowSize,
- final int pageSize
- ) throws IgniteCheckedException {
- assert rowSize <= getFreeSpace(pageAddr): "can't call addRow if not enough space for the whole row";
-
- int fullEntrySize = getPageEntrySize(rowSize, SHOW_PAYLOAD_LEN | SHOW_ITEM);
-
- int directCnt = getDirectCount(pageAddr);
- int indirectCnt = getIndirectCount(pageAddr);
-
- int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
-
- writeRowData(pageAddr, dataOff, rowSize, row, true);
-
- int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
-
- setLinkByPageId(row, pageId, itemId);
- }
-
- /**
- * Adds row to this data page and sets respective link to the given row object.
- *
- * @param pageAddr Page address.
- * @param payload Payload.
- * @param pageSize Page size.
- * @throws IgniteCheckedException If failed.
- */
- public void addRow(
- long pageAddr,
- byte[] payload,
- int pageSize
- ) throws IgniteCheckedException {
- assert payload.length <= getFreeSpace(pageAddr): "can't call addRow if not enough space for the whole row";
-
- int fullEntrySize = getPageEntrySize(payload.length, SHOW_PAYLOAD_LEN | SHOW_ITEM);
-
- int directCnt = getDirectCount(pageAddr);
- int indirectCnt = getIndirectCount(pageAddr);
-
- int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
-
- writeRowData(pageAddr, dataOff, payload);
-
- addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
- }
-
- /**
- * @param pageAddr Page address.
- * @param entryFullSize New entry full size (with item, length and link).
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @param dataOff First entry offset.
- * @param pageSize Page size.
- * @return First entry offset after compaction.
- */
- private int compactIfNeed(
- final long pageAddr,
- final int entryFullSize,
- final int directCnt,
- final int indirectCnt,
- int dataOff,
- int pageSize
- ) {
- if (!isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt)) {
- dataOff = compactDataEntries(pageAddr, directCnt, pageSize);
-
- assert isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt);
- }
-
- return dataOff;
- }
-
- /**
- * Put item reference on entry.
- *
- * @param pageAddr Page address.
- * @param fullEntrySize Full entry size (with link, payload size and item).
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @param dataOff Data offset.
- * @param pageSize Page size.
- * @return Item ID.
- */
- private int addItem(final long pageAddr,
- final int fullEntrySize,
- final int directCnt,
- final int indirectCnt,
- final int dataOff,
- final int pageSize)
- {
- setFirstEntryOffset(pageAddr, dataOff, pageSize);
-
- int itemId = insertItem(pageAddr, dataOff, directCnt, indirectCnt, pageSize);
-
- assert checkIndex(itemId): itemId;
- assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
-
- // Update free space. If number of indirect items changed, then we were able to reuse an item slot.
- setRealFreeSpace(pageAddr,
- getRealFreeSpace(pageAddr) - fullEntrySize + (getIndirectCount(pageAddr) != indirectCnt ? ITEM_SIZE : 0),
- pageSize);
-
- return itemId;
- }
-
- /**
- * @param pageAddr Page address.
- * @param fullEntrySize Full entry size.
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @param pageSize Page size.
- * @return Offset in the buffer where the entry must be written.
- */
- private int getDataOffsetForWrite(long pageAddr, int fullEntrySize, int directCnt, int indirectCnt, int pageSize) {
- int dataOff = getFirstEntryOffset(pageAddr);
-
- // Compact if we do not have enough space for entry.
- dataOff = compactIfNeed(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
-
- // We will write data right before the first entry.
- dataOff -= fullEntrySize - ITEM_SIZE;
-
- return dataOff;
- }
-
- /**
- * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
- *
- * @param pageMem Page memory.
- * @param pageAddr Page address.
- * @param row Cache data row.
- * @param written Number of bytes of row size that was already written.
- * @param rowSize Row size.
- * @param pageSize Page size.
- * @return Written payload size.
- * @throws IgniteCheckedException If failed.
- */
- public int addRowFragment(
- PageMemory pageMem,
- long pageAddr,
- CacheDataRow row,
- int written,
- int rowSize,
- int pageSize
- ) throws IgniteCheckedException {
- return addRowFragment(pageMem, pageAddr, written, rowSize, row.link(), row, null, pageSize);
- }
-
- /**
- * Adds this payload as a fragment to this data page.
- *
- * @param pageAddr Page address.
- * @param payload Payload bytes.
- * @param lastLink Link to the previous written fragment (link to the tail).
- * @param pageSize Page size.
- * @throws IgniteCheckedException If failed.
- */
- public void addRowFragment(
- long pageAddr,
- byte[] payload,
- long lastLink,
- int pageSize
- ) throws IgniteCheckedException {
- addRowFragment(null, pageAddr, 0, 0, lastLink, null, payload, pageSize);
- }
-
- /**
- * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
- *
- * @param pageMem Page memory.
- * @param pageAddr Page address.
- * @param written Number of bytes of row size that was already written.
- * @param rowSize Row size.
- * @param lastLink Link to the previous written fragment (link to the tail).
- * @param row Row.
- * @param payload Payload bytes.
- * @param pageSize Page size.
- * @return Written payload size.
- * @throws IgniteCheckedException If failed.
- */
- private int addRowFragment(
- PageMemory pageMem,
- long pageAddr,
- int written,
- int rowSize,
- long lastLink,
- CacheDataRow row,
- byte[] payload,
- int pageSize
- ) throws IgniteCheckedException {
- assert payload == null ^ row == null;
-
- int directCnt = getDirectCount(pageAddr);
- int indirectCnt = getIndirectCount(pageAddr);
-
- int payloadSize = payload != null ? payload.length :
- Math.min(rowSize - written, getFreeSpace(pageAddr));
-
- int fullEntrySize = getPageEntrySize(payloadSize, SHOW_PAYLOAD_LEN | SHOW_LINK | SHOW_ITEM);
- int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
-
- if (payload == null) {
- ByteBuffer buf = pageMem.pageBuffer(pageAddr);
-
- buf.position(dataOff);
-
- short p = (short)(payloadSize | FRAGMENTED_FLAG);
-
- buf.putShort(p);
- buf.putLong(lastLink);
-
- int rowOff = rowSize - written - payloadSize;
-
- writeFragmentData(row, buf, rowOff, payloadSize);
- }
- else {
- PageUtils.putShort(pageAddr, dataOff, (short)(payloadSize | FRAGMENTED_FLAG));
-
- PageUtils.putLong(pageAddr, dataOff + 2, lastLink);
-
- PageUtils.putBytes(pageAddr, dataOff + 10, payload);
- }
-
- int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
-
- if (row != null)
- setLink(row, pageAddr, itemId);
-
- return payloadSize;
- }
-
- /**
- * @param row Row to set link to.
- * @param pageAddr Page address.
- * @param itemId Item ID.
- */
- private void setLink(CacheDataRow row, long pageAddr, int itemId) {
- long pageId = getPageId(pageAddr);
-
- setLinkByPageId(row, pageId, itemId);
- }
-
- /**
- * @param row Row to set link to.
- * @param pageId Page ID.
- * @param itemId Item ID.
- */
- private void setLinkByPageId(CacheDataRow row, long pageId, int itemId) {
- row.link(PageIdUtils.link(pageId, itemId));
- }
-
- /**
- * Write row data fragment.
- *
- * @param row Row.
- * @param buf Byte buffer.
- * @param rowOff Offset in row data bytes.
- * @param payloadSize Data length that should be written in a fragment.
- * @throws IgniteCheckedException If failed.
- */
- private void writeFragmentData(
+ @Override
+ protected void writeFragmentData(
final CacheDataRow row,
final ByteBuffer buf,
final int rowOff,
@@ -1243,169 +241,9 @@ public class DataPageIO extends PageIO {
CACHE_ID
}
- /**
- * @param pageAddr Page address.
- * @param dataOff Data offset.
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @param pageSize Page size.
- * @return Item ID (insertion index).
- */
- private int insertItem(long pageAddr, int dataOff, int directCnt, int indirectCnt, int pageSize) {
- if (indirectCnt > 0) {
- // If the first indirect item is on correct place to become the last direct item, do the transition
- // and insert the new item into the free slot which was referenced by this first indirect item.
- short item = getItem(pageAddr, directCnt);
-
- if (itemId(item) == directCnt) {
- int directItemIdx = directItemIndex(item);
-
- setItem(pageAddr, directCnt, getItem(pageAddr, directItemIdx));
- setItem(pageAddr, directItemIdx, directItemFromOffset(dataOff));
-
- setDirectCount(pageAddr, directCnt + 1);
- setIndirectCount(pageAddr, indirectCnt - 1);
-
- return directItemIdx;
- }
- }
-
- // Move all the indirect items forward to make a free slot and insert new item at the end of direct items.
- moveItems(pageAddr, directCnt, indirectCnt, +1, pageSize);
-
- setItem(pageAddr, directCnt, directItemFromOffset(dataOff));
-
- setDirectCount(pageAddr, directCnt + 1);
- assert getDirectCount(pageAddr) == directCnt + 1;
-
- return directCnt; // Previous directCnt will be our itemId.
- }
-
- /**
- * @param pageAddr Page address.
- * @param directCnt Direct items count.
- * @param pageSize Page size.
- * @return New first entry offset.
- */
- private int compactDataEntries(long pageAddr, int directCnt, int pageSize) {
- assert checkCount(directCnt): directCnt;
-
- int[] offs = new int[directCnt];
-
- for (int i = 0; i < directCnt; i++) {
- int off = directItemToOffset(getItem(pageAddr, i));
-
- offs[i] = (off << 8) | i; // This way we'll be able to sort by offset using Arrays.sort(...).
- }
-
- Arrays.sort(offs);
-
- // Move right all of the entries if possible to make the page as compact as possible to its tail.
- int prevOff = pageSize;
-
- final int start = directCnt - 1;
- int curOff = offs[start] >>> 8;
- int curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
- for (int i = start; i >= 0; i--) {
- assert curOff < prevOff : curOff;
-
- int delta = prevOff - (curOff + curEntrySize);
-
- int off = curOff;
- int entrySize = curEntrySize;
-
- if (delta != 0) { // Move right.
- assert delta > 0: delta;
-
- int itemId = offs[i] & 0xFF;
-
- setItem(pageAddr, itemId, directItemFromOffset(curOff + delta));
-
- for (int j = i - 1; j >= 0; j--) {
- int offNext = offs[j] >>> 8;
- int nextSize = getPageEntrySize(pageAddr, offNext, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
- if (offNext + nextSize == off) {
- i--;
-
- off = offNext;
- entrySize += nextSize;
-
- itemId = offs[j] & 0xFF;
- setItem(pageAddr, itemId, directItemFromOffset(offNext + delta));
- }
- else {
- curOff = offNext;
- curEntrySize = nextSize;
-
- break;
- }
- }
-
- moveBytes(pageAddr, off, entrySize, delta, pageSize);
-
- off += delta;
- }
- else if (i > 0) {
- curOff = offs[i - 1] >>> 8;
- curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
- }
-
- prevOff = off;
- }
-
- return prevOff;
- }
-
- /**
- * Full-scan free space calculation procedure.
- *
- * @param pageAddr Page to scan.
- * @param pageSize Page size.
- * @return Actual free space in the buffer.
- */
- private int actualFreeSpace(long pageAddr, int pageSize) {
- int directCnt = getDirectCount(pageAddr);
-
- int entriesSize = 0;
-
- for (int i = 0; i < directCnt; i++) {
- int off = directItemToOffset(getItem(pageAddr, i));
-
- int entrySize = getPageEntrySize(pageAddr, off, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
- entriesSize += entrySize;
- }
-
- return pageSize - ITEMS_OFF - entriesSize - (directCnt + getIndirectCount(pageAddr)) * ITEM_SIZE;
- }
-
- /**
- * @param addr Address.
- * @param off Offset.
- * @param cnt Count.
- * @param step Step.
- * @param pageSize Page size.
- */
- private void moveBytes(long addr, int off, int cnt, int step, int pageSize) {
- assert step != 0: step;
- assert off + step >= 0;
- assert off + step + cnt <= pageSize : "[off=" + off + ", step=" + step + ", cnt=" + cnt +
- ", cap=" + pageSize + ']';
-
- PageHandler.copyMemory(addr, off, addr, off + step, cnt);
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Data offset.
- * @param payloadSize Payload size.
- * @param row Data row.
- * @param newRow {@code False} if existing cache entry is updated, in this case skip key data write.
- * @throws IgniteCheckedException If failed.
- */
- private void writeRowData(
+ /** {@inheritDoc} */
+ @Override
+ protected void writeRowData(
long pageAddr,
int dataOff,
int payloadSize,
@@ -1439,12 +277,9 @@ public class DataPageIO extends PageIO {
PageUtils.putLong(addr, 0, row.expireTime());
}
- /**
- * @param pageAddr Page address.
- * @param dataOff Data offset.
- * @param payload Payload
- */
- private void writeRowData(
+ /** {@inheritDoc} */
+ @Override
+ protected void writeRowData(
long pageAddr,
int dataOff,
byte[] payload
@@ -1456,6 +291,11 @@ public class DataPageIO extends PageIO {
}
/** {@inheritDoc} */
+ @Override public int getRowSize(CacheDataRow row) throws IgniteCheckedException {
+ return getRowSize(row, row.cacheId() != 0);
+ }
+
+ /** {@inheritDoc} */
@Override protected void printPage(long addr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException {
sb.a("DataPageIO [\n");
printPageLayout(addr, pageSize, sb);
@@ -1463,18 +303,18 @@ public class DataPageIO extends PageIO {
}
/**
- * Defines closure interface for applying computations to data page items.
- *
- * @param <T> Closure return type.
+ * @param row Row.
+ * @param withCacheId If {@code true} adds cache ID size.
+ * @return Entry size on page.
+ * @throws IgniteCheckedException If failed.
*/
- public interface CC<T> {
- /**
- * Closure body.
- *
- * @param link Link to item.
- * @return Closure return value.
- * @throws IgniteCheckedException In case of error in closure body.
- */
- public T apply(long link) throws IgniteCheckedException;
+ public static int getRowSize(CacheDataRow row, boolean withCacheId) throws IgniteCheckedException {
+ KeyCacheObject key = row.key();
+ CacheObject val = row.value();
+
+ int keyLen = key.valueBytesLength(null);
+ int valLen = val.valueBytesLength(null);
+
+ return keyLen + valLen + CacheVersionIO.size(row.version(), false) + 8 + (withCacheId ? 4 : 0);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
index 60b1aaf..a5236c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
@@ -25,9 +25,10 @@ import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.persistence.MetadataStorage;
+import org.apache.ignite.internal.processors.cache.persistence.IndexStorageImpl;
import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageTree;
import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageLockListener;
import org.apache.ignite.internal.processors.cache.tree.CacheIdAwareDataInnerIO;
@@ -181,6 +182,16 @@ public abstract class PageIO {
/** */
public static final short T_PART_CNTRS = 20;
+ /** */
+ public static final short T_DATA_METASTORAGE = 21;
+
+ /** */
+ public static final short T_DATA_REF_METASTORAGE_INNER = 22;
+
+ /** */
+ public static final short T_DATA_REF_METASTORAGE_LEAF = 23;
+
+
/** Index for payload == 1. */
public static final short T_H2_EX_REF_LEAF_START = 10000;
@@ -460,6 +471,9 @@ public abstract class PageIO {
case T_PAGE_UPDATE_TRACKING:
return (Q)TrackingPageIO.VERSIONS.forVersion(ver);
+ case T_DATA_METASTORAGE:
+ return (Q)SimpleDataPageIO.VERSIONS.forVersion(ver);
+
default:
return (Q)getBPlusIO(type, ver);
}
@@ -518,10 +532,10 @@ public abstract class PageIO {
return (Q)CacheIdAwareDataLeafIO.VERSIONS.forVersion(ver);
case T_METASTORE_INNER:
- return (Q)MetadataStorage.MetaStoreInnerIO.VERSIONS.forVersion(ver);
+ return (Q)IndexStorageImpl.MetaStoreInnerIO.VERSIONS.forVersion(ver);
case T_METASTORE_LEAF:
- return (Q)MetadataStorage.MetaStoreLeafIO.VERSIONS.forVersion(ver);
+ return (Q)IndexStorageImpl.MetaStoreLeafIO.VERSIONS.forVersion(ver);
case T_PENDING_REF_INNER:
return (Q)PendingEntryInnerIO.VERSIONS.forVersion(ver);
@@ -535,6 +549,12 @@ public abstract class PageIO {
case T_CACHE_ID_AWARE_PENDING_REF_LEAF:
return (Q)CacheIdAwarePendingEntryLeafIO.VERSIONS.forVersion(ver);
+ case T_DATA_REF_METASTORAGE_INNER:
+ return (Q)MetastorageTree.MetastorageInnerIO.VERSIONS.forVersion(ver);
+
+ case T_DATA_REF_METASTORAGE_LEAF:
+ return (Q)MetastorageTree.MetastoreLeafIO.VERSIONS.forVersion(ver);
+
default:
// For tests.
if (innerTestIO != null && innerTestIO.getType() == type && innerTestIO.getVersion() == ver)
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
index 4803b44..68e6e2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
@@ -23,9 +23,11 @@ import java.util.Map;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.util.GridStringBuilder;
+import org.apache.ignite.internal.util.GridUnsafe;
/**
- *
+ * Page IO for Partition Counters, IO for pages containing cache ID mapping to its size. Used only for caches in shared
+ * cache groups.
*/
public class PagePartitionCountersIO extends PageIO {
/** */
@@ -40,8 +42,14 @@ public class PagePartitionCountersIO extends PageIO {
/** */
private static final int ITEMS_OFF = NEXT_COUNTERS_PAGE_OFF + 8;
- /** */
- private static final int ITEM_SIZE = 12;
+ /** Serialized size in bytes of cache ID (int) */
+ private static final int CACHE_ID_SIZE = 4;
+
+ /** Serialized size in bytes of cache ID (int) */
+ private static final int CACHE_SIZE_SIZE = 8;
+
+ /** One serialized entry size: Item size = 4 bytes (cache ID) + 8 bytes (cache size) = 12 bytes */
+ public static final int ITEM_SIZE = CACHE_ID_SIZE + CACHE_SIZE_SIZE;
/** */
private static final byte LAST_FLAG = 0b1;
@@ -58,6 +66,22 @@ public class PagePartitionCountersIO extends PageIO {
super(T_PART_CNTRS, ver);
}
+ /**
+ * @param cacheSizes Cache sizes: cache Id in shared group mapped to its size. Not null.
+ * @return Serialized cache sizes or 0-byte length array if map was empty.
+ */
+ public byte[] serializeCacheSizes(Map<Integer, Long> cacheSizes) {
+ byte[] data = new byte[cacheSizes.size() * ITEM_SIZE];
+ long off = GridUnsafe.BYTE_ARR_OFF;
+
+ for (Map.Entry<Integer, Long> entry : cacheSizes.entrySet()) {
+ GridUnsafe.putInt(data, off, entry.getKey()); off += CACHE_ID_SIZE;
+ GridUnsafe.putLong(data, off, entry.getValue()); off += CACHE_SIZE_SIZE;
+ }
+
+ return data;
+ }
+
/** {@inheritDoc} */
@Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
super.initNewPage(pageAddr, pageId, pageSize);
@@ -125,12 +149,12 @@ public class PagePartitionCountersIO extends PageIO {
for (int i = 0; i < cnt; i++) {
int cacheId = PageUtils.getInt(pageAddr, off);
- off += 4;
+ off += CACHE_ID_SIZE;
assert cacheId != 0;
long cacheSize = PageUtils.getLong(pageAddr, off);
- off += 8;
+ off += CACHE_SIZE_SIZE;
assert cacheSize >= 0 : cacheSize;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
index 9a50e5c..3d79884 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
@@ -24,7 +24,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
import org.apache.ignite.internal.util.GridStringBuilder;
/**
- *
+ * IO for partition metadata pages.
*/
public class PagePartitionMetaIO extends PageMetaIO {
/** */
@@ -136,7 +136,7 @@ public class PagePartitionMetaIO extends PageMetaIO {
}
/**
- * @param pageAddr Page address
+ * @param pageAddr Partition metadata page address.
* @param state State.
*/
public boolean setPartitionState(long pageAddr, byte state) {
@@ -149,7 +149,8 @@ public class PagePartitionMetaIO extends PageMetaIO {
}
/**
- * @param pageAddr Page address.
+ * Returns partition counters page identifier, page with caches in cache group sizes.
+ * @param pageAddr Partition metadata page address.
* @return Next meta partial page ID or {@code 0} if it does not exist.
*/
public long getCountersPageId(long pageAddr) {
@@ -157,11 +158,13 @@ public class PagePartitionMetaIO extends PageMetaIO {
}
/**
- * @param pageAddr Page address.
- * @param metaPageId Next partial meta page ID.
+ * Sets new reference to partition counters page (logical cache sizes).
+ *
+ * @param pageAddr Partition metadata page address.
+ * @param cntrsPageId New cache sizes page ID.
*/
- public void setCountersPageId(long pageAddr, long metaPageId) {
- PageUtils.putLong(pageAddr, NEXT_PART_META_PAGE_OFF, metaPageId);
+ public void setCountersPageId(long pageAddr, long cntrsPageId) {
+ PageUtils.putLong(pageAddr, NEXT_PART_META_PAGE_OFF, cntrsPageId);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/SimpleDataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/SimpleDataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/SimpleDataPageIO.java
new file mode 100644
index 0000000..f02e3a0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/SimpleDataPageIO.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageDataRow;
+import org.apache.ignite.internal.util.GridStringBuilder;
+
+/**
+ * Data pages IO for Metastorage.
+ */
+public class SimpleDataPageIO extends AbstractDataPageIO<MetastorageDataRow> {
+ /** */
+ public static final IOVersions<SimpleDataPageIO> VERSIONS = new IOVersions<>(
+ new SimpleDataPageIO(1)
+ );
+
+ /**
+ * @param ver Page format version.
+ */
+ public SimpleDataPageIO(int ver) {
+ super(T_DATA_METASTORAGE, ver);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected void writeFragmentData(
+ final MetastorageDataRow row,
+ final ByteBuffer buf,
+ final int rowOff,
+ final int payloadSize
+ ) throws IgniteCheckedException {
+ int written = writeSizeFragment(row, buf, rowOff, payloadSize);
+
+ if (payloadSize == written)
+ return;
+
+ int start = rowOff > 4 ? rowOff - 4 : 0;
+
+ final int len = Math.min(row.value().length - start, payloadSize - written);
+
+ if (len > 0) {
+ buf.put(row.value(), start, len);
+ written += len;
+ }
+
+ assert written == payloadSize;
+ }
+
+ /** */
+ private int writeSizeFragment(final MetastorageDataRow row, final ByteBuffer buf, final int rowOff,
+ final int payloadSize) {
+ final int size = 4;
+
+ if (rowOff >= size)
+ return 0;
+
+ if (rowOff == 0 && payloadSize >= size) {
+ buf.putInt(row.value().length);
+
+ return size;
+ }
+
+ ByteBuffer buf2 = ByteBuffer.allocate(size);
+ buf2.order(buf.order());
+
+ buf2.putInt(row.value().length);
+ int len = Math.min(size - rowOff, payloadSize);
+ buf.put(buf2.array(), rowOff, len);
+
+ return len;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected void writeRowData(
+ long pageAddr,
+ int dataOff,
+ int payloadSize,
+ MetastorageDataRow row,
+ boolean newRow
+ ) throws IgniteCheckedException {
+ long addr = pageAddr + dataOff;
+
+ if (newRow)
+ PageUtils.putShort(addr, 0, (short)payloadSize);
+
+ PageUtils.putInt(addr, 2, row.value().length);
+ PageUtils.putBytes(addr, 6, row.value());
+ }
+
+ public static byte[] readPayload(long link) {
+ int size = PageUtils.getInt(link, 0);
+
+ return PageUtils.getBytes(link, 4, size);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getRowSize(MetastorageDataRow row) throws IgniteCheckedException {
+ return 4 + row.value().length;
+ }
+
+
+ /** {@inheritDoc} */
+ @Override protected void printPage(long addr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException {
+ sb.a("SimpleDataPageIO [\n");
+ printPageLayout(addr, pageSize, sb);
+ sb.a("\n]");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index c2c07f6..195d181 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory;
import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
import org.apache.ignite.internal.util.typedef.P2;
@@ -219,22 +220,22 @@ public abstract class AbstractWalRecordsIterator
if (hnd == null)
return null;
- FileWALPointer ptr = new FileWALPointer(hnd.idx, (int)hnd.in.position(),0);
+ FileWALPointer actualFilePtr = new FileWALPointer(hnd.idx, (int)hnd.in.position(), 0);
try {
- WALRecord rec = hnd.ser.readRecord(hnd.in, ptr);
+ WALRecord rec = hnd.ser.readRecord(hnd.in, actualFilePtr);
- ptr.length(rec.size());
+ actualFilePtr.length(rec.size());
// cast using diamond operator here can break compile for 7
- return new IgniteBiTuple<>((WALPointer)ptr, postProcessRecord(rec));
+ return new IgniteBiTuple<>((WALPointer)actualFilePtr, postProcessRecord(rec));
}
catch (IOException | IgniteCheckedException e) {
if (e instanceof WalSegmentTailReachedException)
throw (WalSegmentTailReachedException)e;
if (!(e instanceof SegmentEofException))
- handleRecordException(e, ptr);
+ handleRecordException(e, actualFilePtr);
return null;
}
@@ -260,7 +261,7 @@ public abstract class AbstractWalRecordsIterator
@NotNull final Exception e,
@Nullable final FileWALPointer ptr) {
if (log.isInfoEnabled())
- log.info("Stopping WAL iteration due to an exception: " + e.getMessage());
+ log.info("Stopping WAL iteration due to an exception: " + e.getMessage() + ", ptr=" + ptr);
}
/**
@@ -275,7 +276,7 @@ public abstract class AbstractWalRecordsIterator
@Nullable final FileWALPointer start)
throws IgniteCheckedException, FileNotFoundException {
try {
- FileIO fileIO = ioFactory.create(desc.file);
+ FileIO fileIO = desc.isCompressed() ? new UnzipFileIO(desc.file) : ioFactory.create(desc.file);
try {
IgniteBiTuple<Integer, Boolean> tup = FileWriteAheadLogManager.readSerializerVersionAndCompactedFlag(fileIO);
@@ -284,12 +285,13 @@ public abstract class AbstractWalRecordsIterator
boolean isCompacted = tup.get2();
+ if (isCompacted)
+ serializerFactory.skipPositionCheck(true);
+
FileInput in = new FileInput(fileIO, buf);
if (start != null && desc.idx == start.index()) {
if (isCompacted) {
- serializerFactory.skipPositionCheck(true);
-
if (start.fileOffset() != 0)
serializerFactory.recordDeserializeFilter(new StartSeekingFilter(start));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 8c27af0..d29774c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -35,7 +35,9 @@ import java.nio.MappedByteBuffer;
import java.nio.channels.ClosedByInterruptException;
import java.nio.file.Files;
import java.sql.Time;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -95,11 +97,13 @@ import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.CIX1;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.lang.IgniteUuid;
import org.jetbrains.annotations.NotNull;
@@ -120,6 +124,7 @@ import static org.apache.ignite.internal.util.IgniteUtils.findNonPublicMethod;
/**
* File WAL manager.
*/
+@SuppressWarnings("IfMayBeConditional")
public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter implements IgniteWriteAheadLogManager {
/** {@link MappedByteBuffer#force0(java.io.FileDescriptor, long, long)}. */
private static final Method force0 = findNonPublicMethod(
@@ -258,12 +263,15 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
private final int serializerVer =
IgniteSystemProperties.getInteger(IGNITE_WAL_SERIALIZER_VERSION, LATEST_SERIALIZER_VERSION);
- /** Latest segment cleared by {@link #truncate(WALPointer)}. */
+ /** Latest segment cleared by {@link #truncate(WALPointer, WALPointer)}. */
private volatile long lastTruncatedArchiveIdx = -1L;
/** Factory to provide I/O interfaces for read/write operations with files */
private final FileIOFactory ioFactory;
+ /** Next segment archived monitor. */
+ private final Object nextSegmentArchivedMonitor = new Object();
+
/** Updater for {@link #currHnd}, used for verify there are no concurrent update for current log segment handle */
private static final AtomicReferenceFieldUpdater<FileWriteAheadLogManager, FileWriteHandle> CURR_HND_UPD =
AtomicReferenceFieldUpdater.newUpdater(FileWriteAheadLogManager.class, FileWriteHandle.class, "currHnd");
@@ -374,6 +382,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
compressor = new FileCompressor();
decompressor = new FileDecompressor();
+
+ if (!cctx.kernalContext().clientNode())
+ decompressor.start();
}
if (mode != WALMode.NONE) {
@@ -387,6 +398,54 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
/**
+ *
+ */
+ public Collection<File> getAndReserveWalFiles(FileWALPointer low, FileWALPointer high) throws IgniteCheckedException {
+ FileArchiver archiver0 = archiver;
+
+ final long awaitIdx = high.index() - 1;
+
+ awaitSegmentArchived(archiver0, awaitIdx);
+
+ if (!reserve(low))
+ throw new IgniteCheckedException("WAL archive segment has been deleted [idx=" + low.index() + "]");
+
+ List<File> res = new ArrayList<>();
+
+ for (long i = low.index(); i < high.index(); i++) {
+ String segmentName = FileDescriptor.fileName(i);
+
+ File file = new File(walArchiveDir, segmentName);
+
+ if (file.exists())
+ res.add(file);
+ else if ((file = new File(walArchiveDir, segmentName + ".zip")).exists())
+ res.add(file);
+ else
+ throw new IgniteCheckedException("WAL archive segment has been deleted [idx=" + i + "]");
+ }
+
+ return res;
+ }
+
+ /**
+ * @param archiver0 Archiver.
+ * @param awaitIdx Method will wait archivation of that index.
+ */
+ private void awaitSegmentArchived(FileArchiver archiver0, long awaitIdx) throws IgniteInterruptedCheckedException {
+ synchronized (nextSegmentArchivedMonitor) {
+ while (archiver0.lastArchivedAbsoluteIndex() < awaitIdx) {
+ try {
+ nextSegmentArchivedMonitor.wait(2000);
+ }
+ catch (InterruptedException e) {
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ }
+ }
+ }
+
+ /**
* @throws IgniteCheckedException if WAL store path is configured and archive path isn't (or vice versa)
*/
private void checkWalConfiguration() throws IgniteCheckedException {
@@ -453,9 +512,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (compressor != null)
compressor.start();
-
- if (decompressor != null)
- decompressor.start();
}
}
@@ -616,7 +672,20 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
// Need to calculate record size first.
rec.size(serializer.size(rec));
- for (; ; currWrHandle = rollOver(currWrHandle)) {
+ while (true) {
+ if (rec.rollOver()){
+ assert cctx.database().checkpointLockIsHeldByThread();
+
+ long idx = currWrHandle.idx;
+
+ currWrHandle.buf.close();
+
+ currWrHandle = rollOver(currWrHandle);
+
+ if (log != null && log.isDebugEnabled())
+ log.debug("Rollover segment [" + idx + " to " + currWrHandle.idx + "], recordType=" + rec.type());
+ }
+
WALPointer ptr = currWrHandle.addRecord(rec);
if (ptr != null) {
@@ -629,6 +698,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
return ptr;
}
+ else
+ currWrHandle = rollOver(currWrHandle);
checkEnvironment();
@@ -754,14 +825,15 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
/** {@inheritDoc} */
- @Override public int truncate(WALPointer ptr) {
- if (ptr == null)
+ @Override public int truncate(WALPointer low, WALPointer high) {
+ if (high == null)
return 0;
- assert ptr instanceof FileWALPointer : ptr;
+ assert high instanceof FileWALPointer : high;
// File pointer bound: older entries will be deleted from archive
- FileWALPointer fPtr = (FileWALPointer)ptr;
+ FileWALPointer lowPtr = (FileWALPointer)low;
+ FileWALPointer highPtr = (FileWALPointer)high;
FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
@@ -770,12 +842,17 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
FileArchiver archiver0 = archiver;
for (FileDescriptor desc : descs) {
+ if (lowPtr != null && desc.idx < lowPtr.index())
+ continue;
+
// Do not delete reserved or locked segment and any segment after it.
if (archiver0 != null && archiver0.reserved(desc.idx))
return deleted;
+ long lastArchived = archiver0 != null ? archiver0.lastArchivedAbsoluteIndex() : lastArchivedIndex();
+
// We need to leave at least one archived segment to correctly determine the archive index.
- if (desc.idx + 1 < fPtr.index()) {
+ if (desc.idx < highPtr.index() && desc.idx < lastArchived) {
if (!desc.file.delete())
U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
desc.file.getAbsolutePath());
@@ -1085,7 +1162,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
break;
}
- catch (ClosedByInterruptException e) {
+ catch (ClosedByInterruptException ignore) {
interrupted = true;
Thread.interrupted();
@@ -1151,7 +1228,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
createFile(first);
}
else
- checkFiles(0, false, null);
+ checkFiles(0, false, null, null);
}
/**
@@ -1285,9 +1362,15 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** current thread stopping advice */
private volatile boolean stopped;
- /** */
+ /**
+ * Maps absolute segment index to reservation counter. If counter > 0 then we wouldn't delete all segments
+ * which >= reserved segment index.
+ */
private NavigableMap<Long, Integer> reserved = new TreeMap<>();
+ /** Formatted index. */
+ private int formatted;
+
/**
* Maps absolute segment index to locks counter. Lock on segment protects from archiving segment and may come
* from {@link RecordsIterator} during WAL replay. Map itself is guarded by <code>this</code>.
@@ -1391,8 +1474,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
while (curAbsWalIdx == -1 && !stopped)
wait();
- if (curAbsWalIdx != 0 && lastAbsArchivedIdx == -1)
- changeLastArchivedIndexAndWakeupCompressor(curAbsWalIdx - 1);
+ // If the archive directory is empty, we can be sure that there were no WAL segments archived.
+ // This is ensured by the check in truncate() which will leave at least one file there
+ // once it was archived.
}
while (!Thread.currentThread().isInterrupted() && !stopped) {
@@ -1425,7 +1509,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
synchronized (this) {
// Then increase counter to allow rollover on clean working file
- changeLastArchivedIndexAndWakeupCompressor(toArchive);
+ changeLastArchivedIndexAndNotifyWaiters(toArchive);
notifyAll();
}
@@ -1451,11 +1535,15 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/**
* @param idx Index.
*/
- private void changeLastArchivedIndexAndWakeupCompressor(long idx) {
+ private void changeLastArchivedIndexAndNotifyWaiters(long idx) {
lastAbsArchivedIdx = idx;
if (compressor != null)
compressor.onNextSegmentArchived();
+
+ synchronized (nextSegmentArchivedMonitor) {
+ nextSegmentArchivedMonitor.notifyAll();
+ }
}
/**
@@ -1482,11 +1570,20 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
try {
wait();
}
- catch (InterruptedException e) {
+ catch (InterruptedException ignore) {
interrupted.set(true);
}
}
+ // Wait for formatter so that we do not open an empty file in DEFAULT mode.
+ while (curAbsWalIdx % dsCfg.getWalSegments() > formatted)
+ try {
+ wait();
+ }
+ catch (InterruptedException ignore) {
+ interrupted.set(true);
+ }
+
return curAbsWalIdx;
}
}
@@ -1606,11 +1703,24 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
* FileWriteAheadLogManager#checkOrPrepareFiles()}
*/
private void allocateRemainingFiles() throws IgniteCheckedException {
- checkFiles(1, true, new IgnitePredicate<Integer>() {
- @Override public boolean apply(Integer integer) {
- return !checkStop();
+ checkFiles(
+ 1,
+ true,
+ new IgnitePredicate<Integer>() {
+ @Override public boolean apply(Integer integer) {
+ return !checkStop();
+ }
+ },
+ new CI1<Integer>() {
+ @Override public void apply(Integer idx) {
+ synchronized (FileArchiver.this) {
+ formatted = idx;
+
+ FileArchiver.this.notifyAll();
+ }
+ }
}
- });
+ );
}
}
@@ -1754,7 +1864,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (handle != null)
handle.invalidateEnvironment(e);
}
- catch (InterruptedException e) {
+ catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
}
@@ -1802,6 +1912,12 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
while (iter.hasNextX())
iter.nextX();
}
+
+ RecordSerializer ser = new RecordSerializerFactoryImpl(cctx).createSerializer(segmentSerializerVer);
+
+ ByteBuffer heapBuf = prepareSwitchSegmentRecordBuffer(nextSegment, ser);
+
+ zos.write(heapBuf.array());
}
finally {
release(new FileWALPointer(nextSegment, 0, 0));
@@ -1809,6 +1925,25 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
/**
+ * @param nextSegment Segment index.
+ * @param ser Record Serializer.
+ */
+ @NotNull private ByteBuffer prepareSwitchSegmentRecordBuffer(long nextSegment, RecordSerializer ser)
+ throws IgniteCheckedException {
+ SwitchSegmentRecord switchRecord = new SwitchSegmentRecord();
+
+ int switchRecordSize = ser.size(switchRecord);
+ switchRecord.size(switchRecordSize);
+
+ switchRecord.position(new FileWALPointer(nextSegment, 0, switchRecordSize));
+
+ ByteBuffer heapBuf = ByteBuffer.allocate(switchRecordSize);
+
+ ser.writeRecord(switchRecord, heapBuf);
+ return heapBuf;
+ }
+
+ /**
* @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
*/
private void shutdown() throws IgniteInterruptedCheckedException {
@@ -1833,7 +1968,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
private Map<Long, GridFutureAdapter<Void>> decompressionFutures = new HashMap<>();
/** Segments queue. */
- private PriorityBlockingQueue<Long> segmentsQueue = new PriorityBlockingQueue<>();
+ private final PriorityBlockingQueue<Long> segmentsQueue = new PriorityBlockingQueue<>();
/** Byte array for draining data. */
private byte[] arr = new byte[BUF_SIZE];
@@ -1873,7 +2008,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
decompressionFutures.remove(segmentToDecompress).onDone();
}
}
- catch (InterruptedException e){
+ catch (InterruptedException ignore) {
Thread.currentThread().interrupt();
}
catch (IOException e) {
@@ -1934,7 +2069,12 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
* @param p Predicate Exit condition.
* @throws IgniteCheckedException if validation or create file fail.
*/
- private void checkFiles(int startWith, boolean create, IgnitePredicate<Integer> p) throws IgniteCheckedException {
+ private void checkFiles(
+ int startWith,
+ boolean create,
+ @Nullable IgnitePredicate<Integer> p,
+ @Nullable IgniteInClosure<Integer> completionCallback
+ ) throws IgniteCheckedException {
for (int i = startWith; i < dsCfg.getWalSegments() && (p == null || p.apply(i)); i++) {
File checkFile = new File(walWorkDir, FileDescriptor.fileName(i));
@@ -1944,10 +2084,16 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
"the same name already exists): " + checkFile.getAbsolutePath());
else if (checkFile.length() != dsCfg.getWalSegmentSize() && mode == WALMode.DEFAULT)
throw new IgniteCheckedException("Failed to initialize WAL log segment " +
- "(WAL segment size change is not supported):" + checkFile.getAbsolutePath());
+ "(WAL segment size change is not supported in 'DEFAULT' WAL mode) " +
+ "[filePath=" + checkFile.getAbsolutePath() +
+ ", fileSize=" + checkFile.length() +
+ ", configSize=" + dsCfg.getWalSegments() + ']');
}
else if (create)
createFile(checkFile);
+
+ if (completionCallback != null)
+ completionCallback.apply(i);
}
}
@@ -2130,6 +2276,13 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
public String getAbsolutePath() {
return file.getAbsolutePath();
}
+
+ /**
+ * @return True if segment is ZIP compressed.
+ */
+ public boolean isCompressed() {
+ return file.getName().endsWith(".zip");
+ }
}
/**
@@ -2444,7 +2597,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (!needFsync(ptr))
return;
- if (fsyncDelay > 0 && !this.stop.get()) {
+ if (fsyncDelay > 0 && !stop.get()) {
// Delay fsync to collect as many updates as possible: trade latency for throughput.
U.await(fsync, fsyncDelay, TimeUnit.NANOSECONDS);
@@ -2455,7 +2608,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
flushOrWait(ptr);
- if (this.stop.get())
+ if (stop.get())
return;
long lastFsyncPos0 = lastFsyncPos;
@@ -2537,9 +2690,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
*/
private boolean close(boolean rollOver) throws IgniteCheckedException, StorageException {
if (stop.compareAndSet(false, true)) {
- try {
- lock.lock();
+ lock.lock();
+ try {
flushOrWait(null);
try {
@@ -2583,7 +2736,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
try {
fileIO.close();
}
- catch (IOException e) {
+ catch (IOException ignore) {
// No-op.
}
}
@@ -2634,9 +2787,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
/**
- * @throws IgniteCheckedException If failed.
+ *
*/
- private void awaitNext() throws IgniteCheckedException {
+ private void awaitNext() {
lock.lock();
try {
@@ -2686,7 +2839,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
* @return Safely reads current position of the file channel as String. Will return "null" if channel is null.
*/
private String safePosition() {
- FileIO io = this.fileIO;
+ FileIO io = fileIO;
if (io == null)
return "null";
@@ -2854,7 +3007,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** {@inheritDoc} */
@Override protected ReadFileHandle advanceSegment(
- @Nullable final ReadFileHandle curWalSegment) throws IgniteCheckedException {
+ @Nullable final ReadFileHandle curWalSegment
+ ) throws IgniteCheckedException {
if (curWalSegment != null) {
curWalSegment.close();
@@ -2905,7 +3059,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
else
nextHandle.workDir = !readArchive;
-
curRec = null;
return nextHandle;
@@ -2947,7 +3100,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/**
* WAL writer worker.
*/
- class WALWriter extends Thread {
+ @SuppressWarnings("ForLoopReplaceableByForEach")
+ private class WALWriter extends Thread {
/** Unconditional flush. */
private static final long UNCONDITIONAL_FLUSH = -1L;
[13/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 6e4b726..3599594 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -77,6 +77,8 @@ import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
@@ -91,6 +93,7 @@ import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
@@ -108,6 +111,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
@@ -140,6 +145,7 @@ import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteOutClosure;
+import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.mxbean.DataStorageMetricsMXBean;
import org.apache.ignite.thread.IgniteThread;
import org.apache.ignite.thread.IgniteThreadPoolExecutor;
@@ -147,8 +153,10 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import static java.nio.file.StandardOpenOption.READ;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID;
/**
*
@@ -158,6 +166,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** */
public static final String IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC = "IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC";
+ /** MemoryPolicyConfiguration name reserved for meta store. */
+ private static final String METASTORE_DATA_REGION_NAME = "metastoreMemPlc";
+
/** */
private static final long GB = 1024L * 1024 * 1024;
@@ -190,6 +201,12 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** Checkpoint file name pattern. */
private static final Pattern CP_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-(.*)-(START|END)\\.bin");
+ /** Node started file patter. */
+ private static final Pattern NODE_STARTED_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-node-started\\.bin");
+
+ /** Node started file suffix. */
+ private static final String NODE_STARTED_FILE_NAME_SUFFIX = "-node-started.bin";
+
/** */
private static final FileFilter CP_FILE_FILTER = new FileFilter() {
@Override public boolean accept(File f) {
@@ -198,6 +215,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
};
/** */
+ private static final FileFilter NODE_STARTED_FILE_FILTER = new FileFilter() {
+ @Override public boolean accept(File f) {
+ return f.getName().endsWith(NODE_STARTED_FILE_NAME_SUFFIX);
+ }
+ };
+
+ /** */
private static final Comparator<GridDhtLocalPartition> ASC_PART_COMPARATOR = new Comparator<GridDhtLocalPartition>() {
@Override public int compare(GridDhtLocalPartition a, GridDhtLocalPartition b) {
return Integer.compare(a.id(), b.id());
@@ -296,6 +320,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
private final long lockWaitTime;
/** */
+ private final int maxCpHistMemSize;
+
+ /** */
private Map<Integer, Map<Integer, T2<Long, WALPointer>>> reservedForExchange;
/** */
@@ -316,6 +343,12 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** Number of pages in current checkpoint. */
private volatile int currCheckpointPagesCnt;
+ /** */
+ private MetaStorage metaStorage;
+
+ /** */
+ private List<MetastorageLifecycleListener> metastorageLifecycleLsnrs;
+
/**
* @param ctx Kernal context.
*/
@@ -335,6 +368,23 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
persistenceCfg.getMetricsRateTimeInterval(),
persistenceCfg.getMetricsSubIntervalCount()
);
+
+ metastorageLifecycleLsnrs = ctx.internalSubscriptionProcessor().getMetastorageSubscribers();
+
+ maxCpHistMemSize = Math.min(persistenceCfg.getWalHistorySize(),
+ IgniteSystemProperties.getInteger(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, 100));
+ }
+
+ /** */
+ private void notifyMetastorageReadyForRead() throws IgniteCheckedException {
+ for (MetastorageLifecycleListener lsnr : metastorageLifecycleLsnrs)
+ lsnr.onReadyForRead(metaStorage);
+ }
+
+ /** */
+ private void notifyMetastorageReadyForReadWrite() throws IgniteCheckedException {
+ for (MetastorageLifecycleListener lsnr : metastorageLifecycleLsnrs)
+ lsnr.onReadyForReadWrite(metaStorage);
}
/**
@@ -360,6 +410,31 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/** {@inheritDoc} */
+ @Override protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteCheckedException {
+ super.initDataRegions(memCfg);
+
+ addDataRegion(
+ memCfg,
+ createDataRegionConfiguration(memCfg),
+ false
+ );
+ }
+
+ /**
+ * @param storageCfg Data storage configuration.
+ * @return Data region configuration.
+ */
+ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfiguration storageCfg) {
+ DataRegionConfiguration cfg = new DataRegionConfiguration();
+
+ cfg.setName(METASTORE_DATA_REGION_NAME);
+ cfg.setInitialSize(storageCfg.getSystemRegionInitialSize());
+ cfg.setMaxSize(storageCfg.getSystemRegionMaxSize());
+ cfg.setPersistenceEnabled(true);
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
@Override protected void start0() throws IgniteCheckedException {
super.start0();
@@ -397,6 +472,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), kernalCtx, log);
persStoreMetrics.wal(cctx.wal());
+
+ // Here we can get data from metastorage
+ readMetastore();
}
}
@@ -415,6 +493,59 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
);
}
+ /** */
+ private void readMetastore() throws IgniteCheckedException {
+ try {
+ DataStorageConfiguration memCfg = cctx.kernalContext().config().getDataStorageConfiguration();
+
+ DataRegionConfiguration plcCfg = createDataRegionConfiguration(memCfg);
+
+ File allocPath = buildAllocPath(plcCfg);
+
+ DirectMemoryProvider memProvider = allocPath == null ?
+ new UnsafeMemoryProvider(log) :
+ new MappedFileMemoryProvider(
+ log,
+ allocPath);
+
+ DataRegionMetricsImpl memMetrics = new DataRegionMetricsImpl(plcCfg);
+
+ PageMemoryEx storePageMem = (PageMemoryEx)createPageMemory(memProvider, memCfg, plcCfg, memMetrics, false);
+
+ DataRegion regCfg = new DataRegion(storePageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, storePageMem));
+
+ CheckpointStatus status = readCheckpointStatus();
+
+ cctx.pageStore().initializeForMetastorage();
+
+ storePageMem.start();
+
+ checkpointReadLock();
+
+ try {
+ restoreMemory(status, true, storePageMem);
+
+ metaStorage = new MetaStorage(cctx.wal(), regCfg, memMetrics, true);
+
+ metaStorage.init(this);
+
+ applyLastUpdates(status, true);
+
+ notifyMetastorageReadyForRead();
+ }
+ finally {
+ checkpointReadUnlock();
+ }
+
+ metaStorage = null;
+
+ storePageMem.stop();
+ }
+ catch (StorageException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
/**
* Get checkpoint buffer size for the given configuration.
*
@@ -552,7 +683,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** {@inheritDoc} */
@Override public void readCheckpointAndRestoreMemory(
- List<DynamicCacheDescriptor> cachesToStart) throws IgniteCheckedException {
+ List<DynamicCacheDescriptor> cachesToStart
+ ) throws IgniteCheckedException {
+ assert !cctx.localNode().isClient();
+
checkpointReadLock();
try {
@@ -565,13 +699,29 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
CheckpointStatus status = readCheckpointStatus();
+ cctx.pageStore().initializeForMetastorage();
+
+ metaStorage = new MetaStorage(cctx.wal(), dataRegionMap.get(METASTORE_DATA_REGION_NAME),
+ (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME));
+
+ WALPointer restore = restoreMemory(status);
+
// First, bring memory to the last consistent checkpoint state if needed.
// This method should return a pointer to the last valid record in the WAL.
- WALPointer restore = restoreMemory(status);
cctx.wal().resumeLogging(restore);
- cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+ WALPointer ptr = cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+
+ if (ptr != null) {
+ cctx.wal().fsync(ptr);
+
+ nodeStart(ptr);
+ }
+
+ metaStorage.init(this);
+
+ notifyMetastorageReadyForReadWrite();
}
catch (StorageException e) {
throw new IgniteCheckedException(e);
@@ -581,6 +731,93 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
}
+ /**
+ * @param ptr Memory recovery wal pointer.
+ */
+ private void nodeStart(WALPointer ptr) throws IgniteCheckedException {
+ FileWALPointer p = (FileWALPointer)ptr;
+
+ String fileName = U.currentTimeMillis() + "-node-started.bin";
+
+ ByteBuffer buf = ByteBuffer.allocate(20);
+ buf.order(ByteOrder.nativeOrder());
+
+ try (FileChannel ch = FileChannel.open(
+ Paths.get(cpDir.getAbsolutePath(), fileName),
+ StandardOpenOption.CREATE_NEW, StandardOpenOption.APPEND)
+ ) {
+ buf.putLong(p.index());
+
+ buf.putInt(p.fileOffset());
+
+ buf.putInt(p.length());
+
+ buf.flip();
+
+ ch.write(buf);
+
+ buf.clear();
+
+ ch.force(true);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /**
+ *
+ */
+ public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedException {
+ List<T2<Long, WALPointer>> res = new ArrayList<>();
+
+ File[] files = cpDir.listFiles(NODE_STARTED_FILE_FILTER);
+
+ Arrays.sort(files, new Comparator<File>() {
+ @Override public int compare(File o1, File o2) {
+ String n1 = o1.getName();
+ String n2 = o2.getName();
+
+ Long ts1 = Long.valueOf(n1.substring(0, n1.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
+ Long ts2 = Long.valueOf(n2.substring(0, n2.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
+
+ if (ts1 == ts2)
+ return 0;
+ else if (ts1 < ts2)
+ return -1;
+ else
+ return 1;
+ }
+ });
+
+ ByteBuffer buf = ByteBuffer.allocate(20);
+ buf.order(ByteOrder.nativeOrder());
+
+ for (File f : files){
+ String name = f.getName();
+
+ Long ts = Long.valueOf(name.substring(0, name.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
+
+ try (FileChannel ch = FileChannel.open(f.toPath(), READ)) {
+ ch.read(buf);
+
+ buf.flip();
+
+ FileWALPointer ptr = new FileWALPointer(
+ buf.getLong(), buf.getInt(), buf.getInt());
+
+ res.add(new T2<Long, WALPointer>(ts, ptr));
+
+ buf.clear();
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to read node started marker file: " + f.getAbsolutePath(), e);
+ }
+ }
+
+ return res;
+ }
+
/** {@inheritDoc} */
@Override public void lock() throws IgniteCheckedException {
if (fileLockHolder != null) {
@@ -655,10 +892,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
DirectMemoryProvider memProvider,
DataStorageConfiguration memCfg,
DataRegionConfiguration plcCfg,
- DataRegionMetricsImpl memMetrics
+ DataRegionMetricsImpl memMetrics,
+ final boolean trackable
) {
if (!plcCfg.isPersistenceEnabled())
- return super.createPageMemory(memProvider, memCfg, plcCfg, memMetrics);
+ return super.createPageMemory(memProvider, memCfg, plcCfg, memMetrics, trackable);
memMetrics.persistenceEnabled(true);
@@ -680,6 +918,22 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
if (IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED, false))
writeThrottlingEnabled = true;
+ GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
+
+ if (trackable)
+ changeTracker = new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(
+ Long page,
+ FullPageId fullId,
+ PageMemoryEx pageMem
+ ) throws IgniteCheckedException {
+ if (trackable)
+ snapshotMgr.onChangeTrackerPage(page, fullId, pageMem);
+ }
+ };
+ else
+ changeTracker = null;
+
PageMemoryImpl pageMem = new PageMemoryImpl(
memProvider,
calculateFragmentSizes(
@@ -702,15 +956,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
snapshotMgr.flushDirtyPageHandler(fullId, pageBuf, tag);
}
},
- new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
- @Override public void applyx(
- Long page,
- FullPageId fullId,
- PageMemoryEx pageMem
- ) throws IgniteCheckedException {
- snapshotMgr.onChangeTrackerPage(page, fullId, pageMem);
- }
- },
+ changeTracker,
this,
memMetrics,
writeThrottlingEnabled
@@ -1071,7 +1317,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
checkpointReadLock();
try {
- applyLastUpdates(status);
+ applyLastUpdates(status, false);
}
finally {
checkpointReadUnlock();
@@ -1082,6 +1328,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log);
new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
+
+ CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started");
+
+ if (chp != null)
+ chp.cpBeginFut.get();
}
catch (StorageException e) {
throw new IgniteCheckedException(e);
@@ -1440,6 +1691,17 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
* @param status Checkpoint status.
*/
private WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException {
+ return restoreMemory(status, false, (PageMemoryEx)metaStorage.pageMemory());
+ }
+
+ /**
+ * @param status Checkpoint status.
+ * @param storeOnly If {@code True} restores Metastorage only.
+ */
+ private WALPointer restoreMemory(CheckpointStatus status, boolean storeOnly,
+ PageMemoryEx storePageMem) throws IgniteCheckedException {
+ assert !storeOnly || storePageMem != null;
+
if (log.isInfoEnabled())
log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked="
+ status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
@@ -1491,9 +1753,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
// Here we do not require tag check because we may be applying memory changes after
// several repetitive restarts and the same pages may have changed several times.
int grpId = pageRec.fullPageId().groupId();
+
+ if (storeOnly && grpId != METASTORAGE_CACHE_ID)
+ continue;
+
long pageId = pageRec.fullPageId().pageId();
- PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId);
+ PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
long page = pageMem.acquirePage(grpId, pageId, true);
@@ -1521,9 +1787,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
PartitionDestroyRecord destroyRec = (PartitionDestroyRecord)rec;
final int gId = destroyRec.groupId();
+
+ if (storeOnly && gId != METASTORAGE_CACHE_ID)
+ continue;
+
final int pId = destroyRec.partitionId();
- PageMemoryEx pageMem = getPageMemoryForCacheGroup(gId);
+ PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
pageMem.clearAsync(new P3<Integer, Long, Integer>() {
@Override public boolean apply(Integer cacheId, Long pageId, Integer tag) {
@@ -1539,9 +1809,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
PageDeltaRecord r = (PageDeltaRecord)rec;
int grpId = r.groupId();
+
+ if (storeOnly && grpId != METASTORAGE_CACHE_ID)
+ continue;
+
long pageId = r.pageId();
- PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId);
+ PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
// Here we do not require tag check because we may be applying memory changes after
// several repetitive restarts and the same pages may have changed several times.
@@ -1567,6 +1841,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
}
+ if (storeOnly)
+ return null;
+
if (status.needRestoreMemory()) {
if (apply)
throw new IgniteCheckedException("Failed to restore memory state (checkpoint marker is present " +
@@ -1607,16 +1884,87 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
+ * Apply update from some iterator and with specific filters.
+ *
+ * @param it WalIterator.
+ * @param recPredicate Wal record filter.
+ * @param entryPredicate Entry filter.
+ * @param partStates Partition to restore state.
+ */
+ public void applyUpdatesOnRecovery(
+ WALIterator it,
+ IgnitePredicate<IgniteBiTuple<WALPointer, WALRecord>> recPredicate,
+ IgnitePredicate<DataEntry> entryPredicate,
+ Map<T2<Integer, Integer>, T2<Integer, Long>> partStates
+ ) throws IgniteCheckedException {
+ while (it.hasNextX()) {
+ IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
+
+ WALRecord rec = next.get2();
+
+ if (!recPredicate.apply(next))
+ break;
+
+ switch (rec.type()) {
+ case DATA_RECORD:
+ checkpointReadLock();
+
+ try {
+ DataRecord dataRec = (DataRecord) rec;
+
+ for (DataEntry dataEntry : dataRec.writeEntries()) {
+ if (entryPredicate.apply(dataEntry)) {
+ checkpointReadLock();
+
+ try {
+ int cacheId = dataEntry.cacheId();
+
+ GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+
+ if (cacheCtx != null)
+ applyUpdate(cacheCtx, dataEntry);
+ else if (log != null)
+ log.warning("Cache (cacheId=" + cacheId + ") is not started, can't apply updates.");
+ }
+ finally {
+ checkpointReadUnlock();
+ }
+ }
+ }
+ }
+ finally {
+ checkpointReadUnlock();
+ }
+
+ break;
+
+ default:
+ // Skip other records.
+ }
+ }
+
+ checkpointReadLock();
+
+ try {
+ restorePartitionState(partStates);
+ }
+ finally {
+ checkpointReadUnlock();
+ }
+ }
+
+ /**
* @param status Last registered checkpoint status.
* @throws IgniteCheckedException If failed to apply updates.
* @throws StorageException If IO exception occurred while reading write-ahead log.
*/
- private void applyLastUpdates(CheckpointStatus status) throws IgniteCheckedException {
+ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) throws IgniteCheckedException {
if (log.isInfoEnabled())
log.info("Applying lost cache updates since last checkpoint record [lastMarked="
+ status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
- cctx.kernalContext().query().skipFieldLookup(true);
+ if (!metastoreOnly)
+ cctx.kernalContext().query().skipFieldLookup(true);
long start = U.currentTimeMillis();
int applied = 0;
@@ -1631,6 +1979,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
switch (rec.type()) {
case DATA_RECORD:
+ if (metastoreOnly)
+ continue;
+
DataRecord dataRec = (DataRecord)rec;
for (DataEntry dataEntry : dataRec.writeEntries()) {
@@ -1646,6 +1997,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
break;
case PART_META_UPDATE_STATE:
+ if (metastoreOnly)
+ continue;
+
PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord)rec;
partStates.put(new T2<>(metaStateRecord.groupId(), metaStateRecord.partitionId()),
@@ -1653,15 +2007,52 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
break;
+ case METASTORE_DATA_RECORD:
+ MetastoreDataRecord metastoreDataRecord = (MetastoreDataRecord)rec;
+
+ metaStorage.applyUpdate(metastoreDataRecord.key(), metastoreDataRecord.value());
+
+ break;
+
+ case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
+ case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
+ case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
+ if (metastoreOnly)
+ continue;
+
+ PageDeltaRecord rec0 = (PageDeltaRecord) rec;
+
+ PageMemoryEx pageMem = getPageMemoryForCacheGroup(rec0.groupId());
+
+ long page = pageMem.acquirePage(rec0.groupId(), rec0.pageId(), true);
+
+ try {
+ long addr = pageMem.writeLock(rec0.groupId(), rec0.pageId(), page, true);
+
+ try {
+ rec0.applyDelta(pageMem, addr);
+ }
+ finally {
+ pageMem.writeUnlock(rec0.groupId(), rec0.pageId(), page, null, true, true);
+ }
+ }
+ finally {
+ pageMem.releasePage(rec0.groupId(), rec0.pageId(), page);
+ }
+
+ break;
+
default:
// Skip other records.
}
}
- restorePartitionState(partStates);
+ if (!metastoreOnly)
+ restorePartitionState(partStates);
}
finally {
- cctx.kernalContext().query().skipFieldLookup(false);
+ if (!metastoreOnly)
+ cctx.kernalContext().query().skipFieldLookup(false);
}
if (log.isInfoEnabled())
@@ -1690,6 +2081,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
for (int i = 0; i < grp.affinity().partitions(); i++) {
+ T2<Integer, Long> restore = partStates.get(new T2<>(grpId, i));
+
if (storeMgr.exists(grpId, i)) {
storeMgr.ensure(grpId, i);
@@ -1703,41 +2096,99 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
// TODO: https://issues.apache.org/jira/browse/IGNITE-6097
grp.offheap().onPartitionInitialCounterUpdated(i, 0);
- long partMetaId = pageMem.partitionMetaPageId(grpId, i);
- long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
+ checkpointReadLock();
try {
- long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
-
- boolean changed = false;
+ long partMetaId = pageMem.partitionMetaPageId(grpId, i);
+ long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
try {
- PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
+ long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
- T2<Integer, Long> fromWal = partStates.get(new T2<>(grpId, i));
+ boolean changed = false;
- if (fromWal != null) {
- int stateId = fromWal.get1();
+ try {
+ PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
+
+ if (restore != null) {
+ int stateId = restore.get1();
- io.setPartitionState(pageAddr, (byte)stateId);
+ io.setPartitionState(pageAddr, (byte)stateId);
- changed = updateState(part, stateId);
+ changed = updateState(part, stateId);
- if (stateId == GridDhtPartitionState.OWNING.ordinal()) {
- grp.offheap().onPartitionInitialCounterUpdated(i, fromWal.get2());
+ if (stateId == GridDhtPartitionState.OWNING.ordinal()
+ || (stateId == GridDhtPartitionState.MOVING.ordinal()
- if (part.initialUpdateCounter() < fromWal.get2()) {
- part.initialUpdateCounter(fromWal.get2());
+ && part.initialUpdateCounter() < restore.get2())) {
+ part.initialUpdateCounter(restore.get2());
changed = true;
}
}
+ else
+ updateState(part, (int)io.getPartitionState(pageAddr));
+ }
+ finally {
+ pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, partMetaId, partMetaPage);
+ }
+ }
+ finally {
+ checkpointReadUnlock();
+ }
+ }
+ else if (restore != null) {
+ GridDhtLocalPartition part = grp.topology().forceCreatePartition(i);
+
+ assert part != null;
+
+ // TODO: https://issues.apache.org/jira/browse/IGNITE-6097
+ grp.offheap().onPartitionInitialCounterUpdated(i, 0);
+
+ updateState(part, restore.get1());
+ }
+ }
+ }
+ }
+
+ /**
+ * @param grpCtx Group context.
+ * @param partId Partition ID.
+ * @return Partition state.
+ */
+ public GridDhtPartitionState readPartitionState(CacheGroupContext grpCtx, int partId) {
+ int grpId = grpCtx.groupId();
+ PageMemoryEx pageMem = (PageMemoryEx)grpCtx.dataRegion().pageMemory();
+
+ try {
+ if (storeMgr.exists(grpId, partId)) {
+ storeMgr.ensure(grpId, partId);
+
+ if (storeMgr.pages(grpId, partId) > 1) {
+ long partMetaId = pageMem.partitionMetaPageId(grpId, partId);
+ long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
+
+ try {
+ long pageAddr = pageMem.readLock(grpId, partMetaId, partMetaPage);
+
+ try {
+ if (PageIO.getType(pageAddr) == PageIO.T_PART_META) {
+ PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
+
+ GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal((int)io.getPartitionState(pageAddr));
+
+ if (state == null)
+ state = GridDhtPartitionState.MOVING;
+
+ return state;
}
- else
- changed = updateState(part, (int)io.getPartitionState(pageAddr));
}
finally {
- pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
+ pageMem.readUnlock(grpId, partMetaId, partMetaPage);
}
}
finally {
@@ -1746,6 +2197,21 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
}
}
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to read partition state (will default to MOVING) [grp=" + grpCtx +
+ ", partId=" + partId + "]", e);
+ }
+
+ return GridDhtPartitionState.MOVING;
+ }
+
+ /**
+ * Wal truncate callBack.
+ *
+ * @param highBound WALPointer.
+ */
+ public void onWalTruncated(WALPointer highBound) {
+ checkpointHist.onWalTruncated(highBound);
}
/**
@@ -1805,6 +2271,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
break;
+ case READ:
+ // do nothing
+ break;
+
default:
throw new IgniteCheckedException("Invalid operation for WAL entry update: " + dataEntry.op());
}
@@ -2330,8 +2800,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
CacheState state = new CacheState(locParts.size());
- for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions())
- state.addPartitionState(part.id(), part.dataStore().fullSize(), part.updateCounter());
+ for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions()) {
+ state.addPartitionState(
+ part.id(),
+ part.dataStore().fullSize(),
+ part.updateCounter(),
+ (byte)part.state().ordinal()
+ );
+ }
cpRec.addCacheGroupState(grp.groupId(), state);
}
@@ -2340,7 +2816,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
hasPages = hasPageForWrite(cpPagesTuple.get1());
- if (hasPages) {
+ if (hasPages || curr.nextSnapshot) {
// No page updates for this checkpoint are allowed from now on.
cpPtr = cctx.wal().log(cpRec);
@@ -2631,15 +3107,22 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
int grpId = fullId.groupId();
- CacheGroupContext grp = context().cache().cacheGroup(grpId);
+ PageMemoryEx pageMem;
- if (grp == null)
- continue;
+ if (grpId != MetaStorage.METASTORAGE_CACHE_ID) {
+ CacheGroupContext grp = context().cache().cacheGroup(grpId);
- if (!grp.dataRegion().config().isPersistenceEnabled())
- continue;
+ if (grp == null)
+ continue;
+
+ if (!grp.dataRegion().config().isPersistenceEnabled())
+ continue;
+
+ pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
+ }
+ else
+ pageMem = (PageMemoryEx)metaStorage.pageMemory();
- PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
Integer tag = pageMem.getForCheckpoint(
fullId, tmpWriteBuf, persStoreMetrics.metricsEnabled() ? tracker : null);
@@ -2935,15 +3418,18 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
- * Clears checkpoint history.
+ * Callback on truncate wal.
*/
- private void onCheckpointFinished(Checkpoint chp) {
- int deleted = 0;
+ private void onWalTruncated(WALPointer ptr) {
+ FileWALPointer highBound = (FileWALPointer)ptr;
- while (histMap.size() > persistenceCfg.getWalHistorySize()) {
- Map.Entry<Long, CheckpointEntry> entry = histMap.firstEntry();
+ List<CheckpointEntry> cpToRemove = new ArrayList<>();
- CheckpointEntry cpEntry = entry.getValue();
+ for (CheckpointEntry cpEntry : histMap.values()) {
+ FileWALPointer cpPnt = (FileWALPointer)cpEntry.checkpointMark();
+
+ if (highBound.compareTo(cpPnt) <= 0)
+ break;
if (cctx.wal().reserved(cpEntry.checkpointMark())) {
U.warn(log, "Could not clear historyMap due to WAL reservation on cpEntry " + cpEntry.cpId +
@@ -2952,27 +3438,39 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
break;
}
- File startFile = new File(cpDir.getAbsolutePath(), cpEntry.startFile());
- File endFile = new File(cpDir.getAbsolutePath(), cpEntry.endFile());
+ if (!removeCheckpointFiles(cpEntry))
+ cpToRemove.add(cpEntry);
+ }
+
+ for (CheckpointEntry cpEntry : cpToRemove)
+ histMap.remove(cpEntry.cpTs);
+ }
- boolean rmvdStart = !startFile.exists() || startFile.delete();
- boolean rmvdEnd = !endFile.exists() || endFile.delete();
+ /**
+ * Clears checkpoint history.
+ */
+ private void onCheckpointFinished(Checkpoint chp) {
+ int deleted = 0;
- boolean fail = !rmvdStart || !rmvdEnd;
+ boolean dropWal = persistenceCfg.getWalHistorySize() != Integer.MAX_VALUE;
- if (fail) {
- U.warn(log, "Failed to remove stale checkpoint files [startFile=" + startFile.getAbsolutePath() +
- ", endFile=" + endFile.getAbsolutePath() + ']');
+ while (histMap.size() > maxCpHistMemSize) {
+ Map.Entry<Long, CheckpointEntry> entry = histMap.firstEntry();
- if (histMap.size() > 2 * persistenceCfg.getWalHistorySize()) {
- U.error(log, "Too many stale checkpoint entries in the map, will truncate WAL archive anyway.");
+ CheckpointEntry cpEntry = entry.getValue();
- fail = false;
- }
+ if (cctx.wal().reserved(cpEntry.checkpointMark())) {
+ U.warn(log, "Could not clear historyMap due to WAL reservation on cpEntry " + cpEntry.cpId +
+ ", history map size is " + histMap.size());
+
+ break;
}
+ boolean fail = removeCheckpointFiles(cpEntry);
+
if (!fail) {
- deleted += cctx.wal().truncate(cpEntry.checkpointMark());
+ if (dropWal)
+ deleted += cctx.wal().truncate(null, cpEntry.checkpointMark());
histMap.remove(entry.getKey());
}
@@ -2987,6 +3485,33 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
+ * @param cpEntry Checkpoint entry.
+ * @return {True} if delete fail.
+ */
+ private boolean removeCheckpointFiles(CheckpointEntry cpEntry) {
+ File startFile = new File(cpDir.getAbsolutePath(), cpEntry.startFile());
+ File endFile = new File(cpDir.getAbsolutePath(), cpEntry.endFile());
+
+ boolean rmvdStart = !startFile.exists() || startFile.delete();
+ boolean rmvdEnd = !endFile.exists() || endFile.delete();
+
+ boolean fail = !rmvdStart || !rmvdEnd;
+
+ if (fail) {
+ U.warn(log, "Failed to remove stale checkpoint files [startFile=" + startFile.getAbsolutePath() +
+ ", endFile=" + endFile.getAbsolutePath() + ']');
+
+ if (histMap.size() > 2 * maxCpHistMemSize) {
+ U.error(log, "Too many stale checkpoint entries in the map, will truncate WAL archive anyway.");
+
+ fail = false;
+ }
+ }
+
+ return fail;
+ }
+
+ /**
* @param cacheId Cache ID.
* @param partId Partition ID.
* @return Reserved counter or null if couldn't reserve.
@@ -3360,4 +3885,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
public DataStorageMetricsImpl persistentStoreMetricsImpl() {
return persStoreMetrics;
}
-}
\ No newline at end of file
+
+ /** {@inheritDoc} */
+ @Override public MetaStorage metaStorage() {
+ return metaStorage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 5ccbfb9..f8fd86c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageSupport;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
@@ -49,8 +50,7 @@ import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
import org.apache.ignite.internal.processors.cache.persistence.partstate.PagesAllocationRange;
@@ -67,7 +67,6 @@ import org.apache.ignite.internal.processors.cache.tree.CacheDataTree;
import org.apache.ignite.internal.processors.cache.tree.PendingEntriesTree;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.query.GridQueryRowCacheCleaner;
-import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.lang.GridCursor;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.T2;
@@ -76,12 +75,16 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
+
/**
* Used when persistence enabled.
*/
public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl implements DbCheckpointListener {
/** */
- private MetaStore metaStore;
+ private IndexStorage indexStorage;
/** */
private ReuseListImpl reuseList;
@@ -101,7 +104,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
RootPage metastoreRoot = metas.treeRoot;
- metaStore = new MetadataStorage(grp.dataRegion().pageMemory(),
+ indexStorage = new IndexStorageImpl(grp.dataRegion().pageMemory(),
ctx.wal(),
globalRemoveId(),
grp.groupId(),
@@ -122,7 +125,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
try {
final String name = "PendingEntries";
- RootPage pendingRootPage = metaStore.getOrAllocateForTree(name);
+ RootPage pendingRootPage = indexStorage.getOrAllocateForTree(name);
pendingEntries = new PendingEntriesTree(
grp,
@@ -169,8 +172,12 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
* @param store Store to save metadata.
* @throws IgniteCheckedException If failed.
*/
- private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean saveMeta,
- boolean beforeDestroy) throws IgniteCheckedException {
+ private boolean saveStoreMetadata(
+ CacheDataStore store,
+ Context ctx,
+ boolean saveMeta,
+ boolean beforeDestroy
+ ) throws IgniteCheckedException {
RowStore rowStore0 = store.rowStore();
boolean needSnapshot = ctx != null && ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName());
@@ -178,7 +185,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
boolean wasSaveToMeta = false;
if (rowStore0 != null) {
- FreeListImpl freeList = (FreeListImpl)rowStore0.freeList();
+ CacheFreeListImpl freeList = (CacheFreeListImpl)rowStore0.freeList();
freeList.saveMetadata();
@@ -192,13 +199,15 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (size > 0 || updCntr > 0) {
GridDhtPartitionState state = null;
+ // localPartition will not acquire writeLock here because create=false.
+ GridDhtLocalPartition part = null;
+
if (!grp.isLocal()) {
if (beforeDestroy)
state = GridDhtPartitionState.EVICTED;
else {
- // localPartition will not acquire writeLock here because create=false.
- GridDhtLocalPartition part = grp.topology().localPartition(store.partId(),
- AffinityTopologyVersion.NONE, false, true);
+ part = grp.topology().localPartition(store.partId(),
+ AffinityTopologyVersion.NONE, false, true);
if (part != null && part.state() != GridDhtPartitionState.EVICTED)
state = part.state();
@@ -241,63 +250,21 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
long cntrsPageId;
if (grp.sharedGroup()) {
- cntrsPageId = io.getCountersPageId(partMetaPageAddr);
-
- byte[] data = serializeCacheSizes(store.cacheSizes());
-
- int items = data.length / 12;
- int written = 0;
- int pageSize = pageMem.pageSize();
-
- boolean init = cntrsPageId == 0;
-
- if (init && items > 0) {
- cntrsPageId = pageMem.allocatePage(grpId, store.partId(), PageIdAllocator.FLAG_DATA);
-
- io.setCountersPageId(partMetaPageAddr, cntrsPageId);
-
- changed = true;
- }
-
- long nextId = cntrsPageId;
-
- while (written != items) {
- final long curId = nextId;
- final long curPage = pageMem.acquirePage(grpId, curId);
-
- try {
- final long curAddr = pageMem.writeLock(grpId, curId, curPage);
-
- assert curAddr != 0;
-
- try {
- PagePartitionCountersIO partMetaIo;
+ long initCntrPageId = io.getCountersPageId(partMetaPageAddr);
- if (init) {
- partMetaIo = PagePartitionCountersIO.VERSIONS.latest();
+ Map<Integer, Long> newSizes = store.cacheSizes();
+ Map<Integer, Long> prevSizes = readSharedGroupCacheSizes(pageMem, grpId, initCntrPageId);
- partMetaIo.initNewPage(curAddr, curId, pageSize);
- }
- else
- partMetaIo = PageIO.getPageIO(curAddr);
+ if (prevSizes != null && prevSizes.equals(newSizes))
+ cntrsPageId = initCntrPageId; // Preventing modification of sizes pages for store
+ else {
+ cntrsPageId = writeSharedGroupCacheSizes(pageMem, grpId, initCntrPageId,
+ store.partId(), newSizes);
- written += partMetaIo.writeCacheSizes(pageSize, curAddr, data, written);
+ if (initCntrPageId == 0 && cntrsPageId != 0) {
+ io.setCountersPageId(partMetaPageAddr, cntrsPageId);
- nextId = partMetaIo.getNextCountersPageId(curAddr);
-
- if(written != items && (init = nextId == 0)) {
- //allocate new counters page
- nextId = pageMem.allocatePage(grpId, store.partId(), PageIdAllocator.FLAG_DATA);
- partMetaIo.setNextCountersPageId(curAddr, nextId);
- }
- }
- finally {
- // Write full page
- pageMem.writeUnlock(grpId, curId, curPage, Boolean.TRUE, true);
- }
- }
- finally {
- pageMem.releasePage(grpId, curId, curPage);
+ changed = true;
}
}
}
@@ -311,49 +278,35 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
io.setCandidatePageCount(partMetaPageAddr, pageCnt);
if (saveMeta) {
- long metaPageId = pageMem.metaPageId(grpId);
- long metaPage = pageMem.acquirePage(grpId, metaPageId);
-
- try {
- long metaPageAddr = pageMem.writeLock(grpId, metaPageId, metaPage);
-
- try {
- PageMetaIO metaIo = PageMetaIO.getPageIO(metaPageAddr);
-
- long nextSnapshotTag = metaIo.getNextSnapshotTag(metaPageAddr);
-
- metaIo.setNextSnapshotTag(metaPageAddr, nextSnapshotTag + 1);
-
- if (log != null && log.isDebugEnabled())
- log.debug("Save next snapshot before checkpoint start for grId = " + grpId
- + ", nextSnapshotTag = " + nextSnapshotTag);
-
- if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaPageId,
- metaPage, wal, null))
- wal.log(new MetaPageUpdateNextSnapshotId(grpId, metaPageId,
- nextSnapshotTag + 1));
-
- if (state == GridDhtPartitionState.OWNING)
- addPartition(ctx.partitionStatMap(), metaPageAddr, metaIo, grpId, PageIdAllocator.INDEX_PARTITION,
- this.ctx.kernalContext().cache().context().pageStore().pages(grpId, PageIdAllocator.INDEX_PARTITION));
- }
- finally {
- pageMem.writeUnlock(grpId, metaPageId, metaPage, null, true);
- }
- }
- finally {
- pageMem.releasePage(grpId, metaPageId, metaPage);
- }
+ saveMeta(ctx);
wasSaveToMeta = true;
}
- GridDhtPartitionMap partMap = grp.topology().localPartitionMap();
-
- if (partMap.containsKey(store.partId()) &&
- partMap.get(store.partId()) == GridDhtPartitionState.OWNING)
- addPartition(ctx.partitionStatMap(), partMetaPageAddr, io, grpId, store.partId(),
- this.ctx.pageStore().pages(grpId, store.partId()));
+ if (state == OWNING) {
+ assert part != null;
+
+ if(!addPartition(
+ part,
+ ctx.partitionStatMap(),
+ partMetaPageAddr,
+ io,
+ grpId,
+ store.partId(),
+ this.ctx.pageStore().pages(grpId, store.partId()),
+ store.fullSize()
+ ))
+ U.warn(log,"Partition was concurrently evicted grpId=" + grpId +
+ ", partitionId=" + part.id());
+ }
+ else if (state == MOVING || state == RENTING) {
+ if (ctx.partitionStatMap().forceSkipIndexPartition(grpId)) {
+ if (log.isInfoEnabled())
+ log.info("Will not include SQL indexes to snapshot because there is " +
+ "a partition not in " + OWNING + " state [grp=" + grp.cacheOrGroupName() +
+ ", partId=" + store.partId() + ", state=" + state + ']');
+ }
+ }
changed = true;
}
@@ -386,47 +339,211 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
/**
- * @param cacheSizes Cache sizes.
- * @return Serialized cache sizes
+ * Loads cache sizes for all caches in shared group.
+ *
+ * @param pageMem page memory to perform operations on pages.
+ * @param grpId Cache group ID.
+ * @param cntrsPageId Counters page ID, if zero is provided that means no counters page exist.
+ * @return Cache sizes if store belongs to group containing multiple caches and sizes are available in memory. May
+ * return null if counter page does not exist.
+ * @throws IgniteCheckedException If page memory operation failed.
+ */
+ @Nullable private static Map<Integer, Long> readSharedGroupCacheSizes(PageSupport pageMem, int grpId,
+ long cntrsPageId) throws IgniteCheckedException {
+
+ if (cntrsPageId == 0L)
+ return null;
+
+ Map<Integer, Long> cacheSizes = new HashMap<>();
+
+ long nextId = cntrsPageId;
+
+ while (true){
+ final long curId = nextId;
+ final long curPage = pageMem.acquirePage(grpId, curId);
+
+ try {
+ final long curAddr = pageMem.readLock(grpId, curId, curPage);
+
+ assert curAddr != 0;
+
+ try {
+ PagePartitionCountersIO cntrsIO = PageIO.getPageIO(curAddr);
+
+ if (cntrsIO.readCacheSizes(curAddr, cacheSizes))
+ break;
+
+ nextId = cntrsIO.getNextCountersPageId(curAddr);
+
+ assert nextId != 0;
+ }
+ finally {
+ pageMem.readUnlock(grpId, curId, curPage);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, curId, curPage);
+ }
+ }
+ return cacheSizes;
+ }
+
+ /**
+ * Saves cache sizes for all caches in shared group. Unconditionally marks pages as dirty.
+ *
+ * @param pageMem page memory to perform operations on pages.
+ * @param grpId Cache group ID.
+ * @param cntrsPageId Counters page ID, if zero is provided that means no counters page exist.
+ * @param partId Partition ID.
+ * @param sizes Cache sizes of all caches in group. Not null.
+ * @return new counter page Id. Same as {@code cntrsPageId} or new value if cache size pages were initialized.
+ * @throws IgniteCheckedException if page memory operation failed.
*/
- private byte[] serializeCacheSizes(Map<Integer, Long> cacheSizes) {
- // Item size = 4 bytes (cache ID) + 8 bytes (cache size) = 12 bytes
- byte[] data = new byte[cacheSizes.size() * 12];
- long off = GridUnsafe.BYTE_ARR_OFF;
+ private static long writeSharedGroupCacheSizes(PageMemory pageMem, int grpId,
+ long cntrsPageId, int partId, Map<Integer, Long> sizes) throws IgniteCheckedException {
+ byte[] data = PagePartitionCountersIO.VERSIONS.latest().serializeCacheSizes(sizes);
+
+ int items = data.length / PagePartitionCountersIO.ITEM_SIZE;
+ boolean init = cntrsPageId == 0;
+
+ if (init && !sizes.isEmpty())
+ cntrsPageId = pageMem.allocatePage(grpId, partId, PageIdAllocator.FLAG_DATA);
+
+ long nextId = cntrsPageId;
+ int written = 0;
+
+ while (written != items) {
+ final long curId = nextId;
+ final long curPage = pageMem.acquirePage(grpId, curId);
+
+ try {
+ final long curAddr = pageMem.writeLock(grpId, curId, curPage);
+
+ int pageSize = pageMem.pageSize();
+
+ assert curAddr != 0;
+
+ try {
+ PagePartitionCountersIO partCntrIo;
+
+ if (init) {
+ partCntrIo = PagePartitionCountersIO.VERSIONS.latest();
+
+ partCntrIo.initNewPage(curAddr, curId, pageSize);
+ }
+ else
+ partCntrIo = PageIO.getPageIO(curAddr);
- for (Map.Entry<Integer, Long> entry : cacheSizes.entrySet()) {
- GridUnsafe.putInt(data, off, entry.getKey()); off += 4;
- GridUnsafe.putLong(data, off, entry.getValue()); off += 8;
+ written += partCntrIo.writeCacheSizes(pageSize, curAddr, data, written);
+
+ nextId = partCntrIo.getNextCountersPageId(curAddr);
+
+ if (written != items && (init = nextId == 0)) {
+ //allocate new counters page
+ nextId = pageMem.allocatePage(grpId, partId, PageIdAllocator.FLAG_DATA);
+ partCntrIo.setNextCountersPageId(curAddr, nextId);
+ }
+ }
+ finally {
+ // Write full page
+ pageMem.writeUnlock(grpId, curId, curPage, Boolean.TRUE, true);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, curId, curPage);
+ }
}
- return data;
+ return cntrsPageId;
}
/**
+ * @param ctx Context.
+ */
+ private void saveMeta(Context ctx) throws IgniteCheckedException {
+ int grpId = grp.groupId();
+ PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
+ IgniteWriteAheadLogManager wal = this.ctx.wal();
+
+ long metaPageId = pageMem.metaPageId(grpId);
+ long metaPage = pageMem.acquirePage(grpId, metaPageId);
+
+ try {
+ long metaPageAddr = pageMem.writeLock(grpId, metaPageId, metaPage);
+
+ try {
+ PageMetaIO metaIo = PageMetaIO.getPageIO(metaPageAddr);
+
+ long nextSnapshotTag = metaIo.getNextSnapshotTag(metaPageAddr);
+
+ metaIo.setNextSnapshotTag(metaPageAddr, nextSnapshotTag + 1);
+
+ if (log != null && log.isDebugEnabled())
+ log.debug("Save next snapshot before checkpoint start for grId = " + grpId
+ + ", nextSnapshotTag = " + nextSnapshotTag);
+
+ if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaPageId,
+ metaPage, wal, null))
+ wal.log(new MetaPageUpdateNextSnapshotId(grpId, metaPageId,
+ nextSnapshotTag + 1));
+
+ addPartition(
+ null,
+ ctx.partitionStatMap(),
+ metaPageAddr,
+ metaIo,
+ grpId,
+ PageIdAllocator.INDEX_PARTITION,
+ this.ctx.pageStore().pages(grpId, PageIdAllocator.INDEX_PARTITION),
+ -1);
+ }
+ finally {
+ pageMem.writeUnlock(grpId, metaPageId, metaPage, null, true);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, metaPageId, metaPage);
+ }
+ }
+
+ /**
+ * @param part
* @param map Map to add values to.
* @param metaPageAddr Meta page address
* @param io Page Meta IO
* @param cacheId Cache ID.
- * @param partId Partition ID. Or {@link PageIdAllocator#INDEX_PARTITION} for index partition
* @param currAllocatedPageCnt total number of pages allocated for partition <code>[partition, cacheId]</code>
*/
- private static void addPartition(
- final PartitionAllocationMap map,
- final long metaPageAddr,
- final PageMetaIO io,
- final int cacheId,
- final int partId,
- final int currAllocatedPageCnt
+ private static boolean addPartition(
+ GridDhtLocalPartition part,
+ final PartitionAllocationMap map,
+ final long metaPageAddr,
+ final PageMetaIO io,
+ final int cacheId,
+ final int partId,
+ final int currAllocatedPageCnt,
+ final int partSize
) {
- if (currAllocatedPageCnt <= 1)
- return;
+ if (part != null) {
+ boolean reserved = part.reserve();
+
+ if(!reserved)
+ return false;
+ }
+ else
+ assert partId == PageIdAllocator.INDEX_PARTITION : partId;
assert PageIO.getPageId(metaPageAddr) != 0;
int lastAllocatedPageCnt = io.getLastAllocatedPageCount(metaPageAddr);
+
+ int curPageCnt = lastAllocatedPageCnt == 0 && partSize == 0 ? 0 : currAllocatedPageCnt;
+
map.put(
new GroupPartitionId(cacheId, partId),
- new PagesAllocationRange(lastAllocatedPageCnt, currAllocatedPageCnt));
+ new PagesAllocationRange(lastAllocatedPageCnt, curPageCnt));
+
+ return true;
}
/** {@inheritDoc} */
@@ -485,7 +602,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (grp.sharedGroup())
idxName = Integer.toString(cacheId) + "_" + idxName;
- return metaStore.getOrAllocateForTree(idxName);
+ return indexStorage.getOrAllocateForTree(idxName);
}
/** {@inheritDoc} */
@@ -493,7 +610,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
if (grp.sharedGroup())
idxName = Integer.toString(cacheId) + "_" + idxName;
- metaStore.dropRootPage(idxName);
+ indexStorage.dropRootPage(idxName);
}
/** {@inheritDoc} */
@@ -613,7 +730,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
for (CacheDataStore store : partDataStores.values()) {
assert store instanceof GridCacheDataStore;
- FreeListImpl freeList = ((GridCacheDataStore)store).freeList;
+ CacheFreeListImpl freeList = ((GridCacheDataStore)store).freeList;
if (freeList == null)
continue;
@@ -863,7 +980,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
private String name;
/** */
- private volatile FreeListImpl freeList;
+ private volatile CacheFreeListImpl freeList;
/** */
private volatile CacheDataStore delegate;
@@ -913,7 +1030,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
RootPage reuseRoot = metas.reuseListRoot;
- freeList = new FreeListImpl(
+ freeList = new CacheFreeListImpl(
grp.groupId(),
grp.cacheOrGroupName() + "-" + partId,
grp.dataRegion().memoryMetrics(),
@@ -964,43 +1081,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
Map<Integer, Long> cacheSizes = null;
- if (grp.sharedGroup()) {
- long cntrsPageId = io.getCountersPageId(pageAddr);
-
- if (cntrsPageId != 0L) {
- cacheSizes = new HashMap<>();
-
- long nextId = cntrsPageId;
-
- while (true){
- final long curId = nextId;
- final long curPage = pageMem.acquirePage(grpId, curId);
-
- try {
- final long curAddr = pageMem.readLock(grpId, curId, curPage);
-
- assert curAddr != 0;
-
- try {
- PagePartitionCountersIO cntrsIO = PageIO.getPageIO(curAddr);
-
- if (cntrsIO.readCacheSizes(curAddr, cacheSizes))
- break;
-
- nextId = cntrsIO.getNextCountersPageId(curAddr);
-
- assert nextId != 0;
- }
- finally {
- pageMem.readUnlock(grpId, curId, curPage);
- }
- }
- finally {
- pageMem.releasePage(grpId, curId, curPage);
- }
- }
- }
- }
+ if (grp.sharedGroup())
+ cacheSizes = readSharedGroupCacheSizes(pageMem, grpId, io.getCountersPageId(pageAddr));
delegate0.init(io.getSize(pageAddr), io.getUpdateCounter(pageAddr), cacheSizes);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 8771f6a..1260147 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.management.InstanceNotFoundException;
import org.apache.ignite.DataRegionMetrics;
import org.apache.ignite.DataStorageMetrics;
import org.apache.ignite.IgniteCheckedException;
@@ -51,8 +52,9 @@ import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictio
import org.apache.ignite.internal.processors.cache.persistence.evict.Random2LruPageEvictionTracker;
import org.apache.ignite.internal.processors.cache.persistence.evict.RandomLruPageEvictionTracker;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport;
import org.apache.ignite.internal.util.typedef.F;
@@ -83,7 +85,10 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
private static final long MAX_PAGE_MEMORY_INIT_SIZE_32_BIT = 2L * 1024 * 1024 * 1024;
/** */
- protected Map<String, DataRegion> dataRegionMap;
+ protected volatile Map<String, DataRegion> dataRegionMap;
+
+ /** */
+ private volatile boolean dataRegionsInitialized;
/** */
protected Map<String, DataRegionMetrics> memMetricsMap;
@@ -92,10 +97,10 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
protected DataRegion dfltDataRegion;
/** */
- protected Map<String, FreeListImpl> freeListMap;
+ protected Map<String, CacheFreeListImpl> freeListMap;
/** */
- private FreeListImpl dfltFreeList;
+ private CacheFreeListImpl dfltFreeList;
/** Page size from memory configuration, may be set only for fake(standalone) IgniteCacheDataBaseSharedManager */
private int pageSize;
@@ -112,6 +117,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
validateConfiguration(memCfg);
pageSize = memCfg.getPageSize();
+
+ initDataRegions(memCfg);
}
/**
@@ -172,7 +179,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
boolean persistenceEnabled = memPlcCfg.isPersistenceEnabled();
- FreeListImpl freeList = new FreeListImpl(0,
+ CacheFreeListImpl freeList = new CacheFreeListImpl(0,
cctx.igniteInstanceName(),
memMetrics,
memPlc,
@@ -210,21 +217,25 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
* @throws IgniteCheckedException If failed to initialize swap path.
*/
protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteCheckedException {
+ if (dataRegionsInitialized)
+ return;
+
DataRegionConfiguration[] dataRegionCfgs = memCfg.getDataRegionConfigurations();
int dataRegions = dataRegionCfgs == null ? 0 : dataRegionCfgs.length;
- dataRegionMap = U.newHashMap(2 + dataRegions);
- memMetricsMap = U.newHashMap(2 + dataRegions);
+ dataRegionMap = U.newHashMap(3 + dataRegions);
+ memMetricsMap = U.newHashMap(3 + dataRegions);
if (dataRegionCfgs != null) {
for (DataRegionConfiguration dataRegionCfg : dataRegionCfgs)
- addDataRegion(memCfg, dataRegionCfg);
+ addDataRegion(memCfg, dataRegionCfg, dataRegionCfg.isPersistenceEnabled());
}
addDataRegion(
memCfg,
- memCfg.getDefaultDataRegionConfiguration()
+ memCfg.getDefaultDataRegionConfiguration(),
+ memCfg.getDefaultDataRegionConfiguration().isPersistenceEnabled()
);
addDataRegion(
@@ -233,8 +244,11 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
memCfg.getSystemRegionInitialSize(),
memCfg.getSystemRegionMaxSize(),
CU.isPersistenceEnabled(memCfg)
- )
+ ),
+ CU.isPersistenceEnabled(memCfg)
);
+
+ dataRegionsInitialized = true;
}
/**
@@ -242,9 +256,10 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
* @param dataRegionCfg Data region config.
* @throws IgniteCheckedException If failed to initialize swap path.
*/
- private void addDataRegion(
+ protected void addDataRegion(
DataStorageConfiguration dataStorageCfg,
- DataRegionConfiguration dataRegionCfg
+ DataRegionConfiguration dataRegionCfg,
+ boolean trackable
) throws IgniteCheckedException {
String dataRegionName = dataRegionCfg.getName();
@@ -255,7 +270,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
DataRegionMetricsImpl memMetrics = new DataRegionMetricsImpl(dataRegionCfg, fillFactorProvider(dataRegionCfg));
- DataRegion memPlc = initMemory(dataStorageCfg, dataRegionCfg, memMetrics);
+ DataRegion memPlc = initMemory(dataStorageCfg, dataRegionCfg, memMetrics, trackable);
dataRegionMap.put(dataRegionName, memPlc);
@@ -278,11 +293,11 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
final String dataRegName = dataRegCfg.getName();
return new IgniteOutClosure<Float>() {
- private FreeListImpl freeList;
+ private CacheFreeListImpl freeList;
@Override public Float apply() {
if (freeList == null) {
- FreeListImpl freeList0 = freeListMap.get(dataRegName);
+ CacheFreeListImpl freeList0 = freeListMap.get(dataRegName);
if (freeList0 == null)
return (float) 0;
@@ -526,7 +541,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
*/
public void dumpStatistics(IgniteLogger log) {
if (freeListMap != null) {
- for (FreeListImpl freeList : freeListMap.values())
+ for (CacheFreeListImpl freeList : freeListMap.values())
freeList.dumpStatistics(log);
}
}
@@ -641,6 +656,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
dataRegionMap.clear();
dataRegionMap = null;
+
+ dataRegionsInitialized = false;
}
}
@@ -661,6 +678,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
"DataRegionMetrics", name
));
}
+ catch (InstanceNotFoundException ignored) {
+ // We tried to unregister a non-existing MBean, not a big deal.
+ }
catch (Throwable e) {
U.error(log, "Failed to unregister MBean for memory metrics: " +
name, e);
@@ -798,7 +818,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
int sysPageSize = pageMem.systemPageSize();
- FreeListImpl freeListImpl = freeListMap.get(plcCfg.getName());
+ CacheFreeListImpl freeListImpl = freeListMap.get(plcCfg.getName());
for (;;) {
long allocatedPagesCnt = pageMem.loadedPages();
@@ -826,7 +846,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
private DataRegion initMemory(
DataStorageConfiguration memCfg,
DataRegionConfiguration plcCfg,
- DataRegionMetricsImpl memMetrics
+ DataRegionMetricsImpl memMetrics,
+ boolean trackable
) throws IgniteCheckedException {
File allocPath = buildAllocPath(plcCfg);
@@ -836,7 +857,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
log,
allocPath);
- PageMemory pageMem = createPageMemory(memProvider, memCfg, plcCfg, memMetrics);
+ PageMemory pageMem = createPageMemory(memProvider, memCfg, plcCfg, memMetrics, trackable);
return new DataRegion(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem));
}
@@ -845,7 +866,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
* @param plc data region Configuration.
* @param pageMem Page memory.
*/
- private PageEvictionTracker createPageEvictionTracker(DataRegionConfiguration plc, PageMemory pageMem) {
+ protected PageEvictionTracker createPageEvictionTracker(DataRegionConfiguration plc, PageMemory pageMem) {
if (plc.getPageEvictionMode() == DataPageEvictionMode.DISABLED || plc.isPersistenceEnabled())
return new NoOpPageEvictionTracker();
@@ -873,7 +894,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
*
* @throws IgniteCheckedException If resolving swap directory fails.
*/
- @Nullable private File buildAllocPath(DataRegionConfiguration plc) throws IgniteCheckedException {
+ @Nullable protected File buildAllocPath(DataRegionConfiguration plc) throws IgniteCheckedException {
String path = plc.getSwapPath();
if (path == null)
@@ -901,7 +922,8 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
DirectMemoryProvider memProvider,
DataStorageConfiguration memCfg,
DataRegionConfiguration memPlcCfg,
- DataRegionMetricsImpl memMetrics
+ DataRegionMetricsImpl memMetrics,
+ boolean trackable
) {
memMetrics.persistenceEnabled(false);
@@ -969,4 +991,11 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
protected void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
+
+ /**
+ * @return MetaStorage
+ */
+ public MetaStorage metaStorage() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java
new file mode 100644
index 0000000..5141b04
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorage.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Meta store.
+ */
+public interface IndexStorage {
+ /**
+ * Get or allocate initial page for an index.
+ *
+ * @param idxName Index name.
+ * @return {@link RootPage} that keeps pageId, allocated flag that shows whether the page
+ * was newly allocated, and rootId that is counter which increments each time new page allocated.
+ * @throws IgniteCheckedException If failed.
+ */
+ public RootPage getOrAllocateForTree(String idxName) throws IgniteCheckedException;
+
+ /**
+ * Deallocate index page and remove from tree.
+ *
+ * @param idxName Index name.
+ * @return Root ID or -1 if no page was removed.
+ * @throws IgniteCheckedException If failed.
+ */
+ public RootPage dropRootPage(String idxName) throws IgniteCheckedException;
+
+ /**
+ * Destroy this meta store.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void destroy() throws IgniteCheckedException;
+}
[08/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index 962c4ca..d30dddb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -21,7 +21,6 @@ import java.io.File;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
@@ -46,9 +45,9 @@ public class IgniteWalIteratorFactory {
@Nullable private File binaryMetadataFileStoreDir;
/**
- * Folder specifying location of marshaller mapping file store. {@code null} means no specific folder is configured. <br>
- * This folder should be specified for converting data entries into BinaryObjects.
- * Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects will be provided
+ * Folder specifying location of marshaller mapping file store. {@code null} means no specific folder is configured.
+ * <br> This folder should be specified for converting data entries into BinaryObjects. Providing {@code null} will
+ * disable unmarshall for non primitive objects, BinaryObjects will be provided
*/
@Nullable private File marshallerMappingFileStoreDir;
@@ -56,7 +55,10 @@ public class IgniteWalIteratorFactory {
private boolean keepBinary;
/** Factory to provide I/O interfaces for read/write operations with files */
- private final FileIOFactory ioFactory;
+ private FileIOFactory ioFactory;
+
+ /** Wal records iterator buffer size */
+ private int bufSize = StandaloneWalRecordsIterator.DFLT_BUF_SIZE;
/**
* Creates WAL files iterator factory.
@@ -69,8 +71,8 @@ public class IgniteWalIteratorFactory {
* subfolder and consistent ID subfolder. Note Consistent ID should be already masked and should not contain special
* symbols. Providing {@code null} means no specific folder is configured. <br>
* @param marshallerMappingFileStoreDir Folder specifying location of marshaller mapping file store. Should include
- * "marshaller" subfolder. Providing {@code null} will disable unmarshall for non primitive objects,
- * BinaryObjects will be provided
+ * "marshaller" subfolder. Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects
+ * will be provided
* @param keepBinary {@code true} disables complex object unmarshall into source classes
*/
public IgniteWalIteratorFactory(
@@ -148,8 +150,10 @@ public class IgniteWalIteratorFactory {
* @return closable WAL records iterator, should be closed when non needed
* @throws IgniteCheckedException if failed to read folder
*/
- public WALIterator iteratorArchiveDirectory(@NotNull final File walDirWithConsistentId) throws IgniteCheckedException {
- return new StandaloneWalRecordsIterator(walDirWithConsistentId, log, prepareSharedCtx(), ioFactory, keepBinary);
+ public WALIterator iteratorArchiveDirectory(
+ @NotNull final File walDirWithConsistentId) throws IgniteCheckedException {
+ return new StandaloneWalRecordsIterator(
+ walDirWithConsistentId, log, prepareSharedCtx(), ioFactory, keepBinary, bufSize);
}
/**
@@ -166,7 +170,7 @@ public class IgniteWalIteratorFactory {
* @throws IgniteCheckedException if failed to read files
*/
public WALIterator iteratorArchiveFiles(@NotNull final File... files) throws IgniteCheckedException {
- return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, false, keepBinary, files);
+ return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, false, keepBinary, bufSize, files);
}
/**
@@ -183,7 +187,7 @@ public class IgniteWalIteratorFactory {
* @throws IgniteCheckedException if failed to read files
*/
public WALIterator iteratorWorkFiles(@NotNull final File... files) throws IgniteCheckedException {
- return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, true, keepBinary, files);
+ return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, true, keepBinary, bufSize, files);
}
/**
@@ -202,4 +206,18 @@ public class IgniteWalIteratorFactory {
null, null, null, null,
null, null, null);
}
+
+ /**
+ * @param ioFactory New factory to provide I/O interfaces for read/write operations with files
+ */
+ public void ioFactory(FileIOFactory ioFactory) {
+ this.ioFactory = ioFactory;
+ }
+
+ /**
+ * @param bufSize New wal records iterator buffer size
+ */
+ public void bufferSize(int bufSize) {
+ this.bufSize = bufSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index 80dfc5b..fa3f7f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -77,6 +77,7 @@ import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
import org.apache.ignite.internal.processors.segmentation.GridSegmentationProcessor;
import org.apache.ignite.internal.processors.service.GridServiceProcessor;
import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
+import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
import org.apache.ignite.internal.processors.task.GridTaskProcessor;
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
@@ -609,6 +610,11 @@ public class StandaloneGridKernalContext implements GridKernalContext {
}
/** {@inheritDoc} */
+ @Override public GridInternalSubscriptionProcessor internalSubscriptionProcessor() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public PdsFoldersResolver pdsFolderResolver() {
return new PdsFoldersResolver() {
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index b36c2db..7740f28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -60,12 +60,12 @@ import static org.apache.ignite.internal.processors.cache.persistence.wal.serial
* Operates over one directory, does not provide start and end boundaries
*/
class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
+ /** Record buffer size */
+ public static final int DFLT_BUF_SIZE = 2 * 1024 * 1024;
+
/** */
private static final long serialVersionUID = 0L;
- /** Record buffer size */
- private static final int BUF_SIZE = 2 * 1024 * 1024;
-
/**
* WAL files directory. Should already contain 'consistent ID' as subfolder.
* <code>null</code> value means file-by-file iteration mode
@@ -80,38 +80,31 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
@Nullable
private List<FileWriteAheadLogManager.FileDescriptor> walFileDescriptors;
- /**
- * True if this iterator used for work dir, false for archive.
- * In work dir mode exceptions come from record reading are ignored (file may be not completed).
- * Index of file is taken from file itself, not from file name
- */
- private boolean workDir;
-
/** Keep binary. This flag disables converting of non primitive types (BinaryObjects) */
private boolean keepBinary;
/**
* Creates iterator in directory scan mode
- * @param walFilesDir Wal files directory. Should already contain node consistent ID as subfolder
+ * @param walFilesDir Wal files directory. Should already contain node consistent ID as subfolder
* @param log Logger.
- * @param sharedCtx Shared context. Cache processor is to be configured if Cache Object Key & Data Entry is
- * required.
+ * @param sharedCtx Shared context. Cache processor is to be configured if Cache Object Key & Data Entry is required.
* @param ioFactory File I/O factory.
* @param keepBinary Keep binary. This flag disables converting of non primitive types
- * (BinaryObjects will be used instead)
+ * @param bufSize Buffer size.
*/
StandaloneWalRecordsIterator(
@NotNull File walFilesDir,
@NotNull IgniteLogger log,
@NotNull GridCacheSharedContext sharedCtx,
@NotNull FileIOFactory ioFactory,
- boolean keepBinary
+ boolean keepBinary,
+ int bufSize
) throws IgniteCheckedException {
super(log,
sharedCtx,
new RecordSerializerFactoryImpl(sharedCtx),
ioFactory,
- BUF_SIZE);
+ bufSize);
this.keepBinary = keepBinary;
init(walFilesDir, false, null);
advance();
@@ -119,7 +112,7 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
/**
* Creates iterator in file-by-file iteration mode. Directory
- * @param log Logger.
+ * @param log Logger.
* @param sharedCtx Shared context. Cache processor is to be configured if Cache Object Key & Data Entry is
* required.
* @param ioFactory File I/O factory.
@@ -134,14 +127,14 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
@NotNull FileIOFactory ioFactory,
boolean workDir,
boolean keepBinary,
+ int bufSize,
@NotNull File... walFiles) throws IgniteCheckedException {
super(log,
sharedCtx,
new RecordSerializerFactoryImpl(sharedCtx),
ioFactory,
- BUF_SIZE);
+ bufSize);
- this.workDir = workDir;
this.keepBinary = keepBinary;
init(null, workDir, walFiles);
advance();
@@ -163,10 +156,8 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
FileWriteAheadLogManager.FileDescriptor[] descs = loadFileDescriptors(walFilesDir);
curWalSegmIdx = !F.isEmpty(descs) ? descs[0].getIdx() : 0;
this.walFilesDir = walFilesDir;
- this.workDir = false;
}
else {
- this.workDir = workDir;
if (workDir)
walFileDescriptors = scanIndexesFromFileHeaders(walFiles);
@@ -244,9 +235,14 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
final FileWriteAheadLogManager.FileDescriptor fd;
if (walFilesDir != null) {
- fd = new FileWriteAheadLogManager.FileDescriptor(
- new File(walFilesDir,
- FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx)));
+ File segmentFile = new File(walFilesDir,
+ FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx));
+
+ if (!segmentFile.exists())
+ segmentFile = new File(walFilesDir,
+ FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx) + ".zip");
+
+ fd = new FileWriteAheadLogManager.FileDescriptor(segmentFile);
}
else {
if (walFileDescriptors.isEmpty())
@@ -285,6 +281,7 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
log.error("Failed to perform post processing for data record ", e);
}
}
+
return super.postProcessRecord(rec);
}
@@ -312,7 +309,13 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
postProcessedEntries.add(postProcessedEntry);
}
- return new DataRecord(postProcessedEntries, dataRec.timestamp());
+
+ DataRecord res = new DataRecord(postProcessedEntries, dataRec.timestamp());
+
+ res.size(dataRec.size());
+ res.position(dataRec.position());
+
+ return res;
}
/**
@@ -368,18 +371,6 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
}
/** {@inheritDoc} */
- @Override protected void handleRecordException(
- @NotNull final Exception e,
- @Nullable final FileWALPointer ptr) {
- super.handleRecordException(e, ptr);
- final RuntimeException ex = new RuntimeException("Record reading problem occurred at file pointer [" + ptr + "]:" + e.getMessage(), e);
-
- ex.printStackTrace();
- if (!workDir)
- throw ex;
- }
-
- /** {@inheritDoc} */
@Override protected void onClose() throws IgniteCheckedException {
super.onClose();
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java
new file mode 100644
index 0000000..94b51c5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.serializer;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.wal.record.BaselineTopologyRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * {@link BaselineTopologyRecord} WAL serializer.
+ */
+public class BaselineTopologyRecordSerializer {
+ /** Cache shared context. */
+ private GridCacheSharedContext cctx;
+
+ /** Class loader to unmarshal consistent IDs. */
+ private ClassLoader clsLdr;
+
+ /**
+ * Create an instance of serializer.
+ *
+ * @param cctx Cache shared context.
+ */
+ public BaselineTopologyRecordSerializer(GridCacheSharedContext cctx) {
+ this.cctx = cctx;
+
+ clsLdr = U.resolveClassLoader(cctx.gridConfig());
+ }
+
+ /**
+ * Writes {@link BaselineTopologyRecord} to given buffer.
+ *
+ * @param rec {@link BaselineTopologyRecord} instance.
+ * @param buf Byte buffer.
+ * @throws IgniteCheckedException In case of fail.
+ */
+ public void write(BaselineTopologyRecord rec, ByteBuffer buf) throws IgniteCheckedException {
+ buf.putInt(rec.id());
+
+ Map<Short, Object> mapping = rec.mapping();
+
+ if (mapping != null && !mapping.isEmpty()) {
+ buf.putInt(mapping.size());
+
+ for (Map.Entry<Short, Object> e : mapping.entrySet()) {
+ buf.putShort(e.getKey());
+
+ writeConsistentId(e.getValue(), buf);
+ }
+ }
+ else
+ buf.putInt(0);
+ }
+
+ /**
+ * Reads {@link BaselineTopologyRecord} from given input.
+ *
+ * @param in Input
+ * @return BaselineTopologyRecord instance.
+ * @throws IOException In case of fail.
+ * @throws IgniteCheckedException In case of fail.
+ */
+ public BaselineTopologyRecord read(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
+ int id = in.readInt();
+
+ int size = in.readInt();
+
+ Map<Short, Object> mapping = size > 0 ? U.<Short, Object>newHashMap(size) : null;
+
+ for (int i = 0; i < size; i++) {
+ short compactId = in.readShort();
+
+ Object consistentId = readConsistentId(in);
+
+ mapping.put(compactId, consistentId);
+ }
+
+ return new BaselineTopologyRecord(id, mapping);
+ }
+
+ /**
+ * Returns size of marshalled {@link BaselineTopologyRecord} in bytes.
+ *
+ * @param rec BaselineTopologyRecord instance.
+ * @return Size of BaselineTopologyRecord instance in bytes.
+ * @throws IgniteCheckedException In case of fail.
+ */
+ public int size(BaselineTopologyRecord rec) throws IgniteCheckedException {
+ int size = 0;
+
+ size += /* Baseline topology ID. */ 4;
+
+ size += /* Consistent ID mapping size. */ 4;
+
+ if (rec.mapping() != null) {
+ for (Object consistentId : rec.mapping().values()) {
+ size += /* Compact ID size */ 2;
+
+ size += marshalConsistentId(consistentId).length;
+ }
+ }
+
+ return size;
+ }
+
+ /**
+ * Write consistent id to given buffer.
+ *
+ * @param consistentId Consistent id.
+ * @param buf Byte buffer.
+ * @throws IgniteCheckedException In case of fail.
+ */
+ private void writeConsistentId(Object consistentId, ByteBuffer buf) throws IgniteCheckedException {
+ byte[] content = marshalConsistentId(consistentId);
+
+ buf.putInt(content.length);
+ buf.put(content);
+ }
+
+ /**
+ * Read consistent id from given input.
+ *
+ * @param in Input.
+ * @return Consistent id.
+ * @throws IOException In case of fail.
+ * @throws IgniteCheckedException In case of fail.
+ */
+ private Object readConsistentId(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
+ int len = in.readInt();
+ in.ensure(len);
+
+ byte[] content = new byte[len];
+ in.readFully(content);
+
+ return cctx.marshaller().unmarshal(content, clsLdr);
+ }
+
+ /**
+ * Marshal consistent id to byte array.
+ *
+ * @param consistentId Consistent id.
+ * @return Marshalled byte array.
+ * @throws IgniteCheckedException In case of fail.
+ */
+ private byte[] marshalConsistentId(Object consistentId) throws IgniteCheckedException {
+ return cctx.marshaller().marshal(consistentId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
index 12992a1..e8d116b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry;
import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
@@ -112,11 +113,11 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
private TxRecordSerializer txRecordSerializer;
/**
- * @param cctx Cctx.
+ * @param cctx Cache shared context.
*/
public RecordDataV1Serializer(GridCacheSharedContext cctx) {
this.cctx = cctx;
- this.txRecordSerializer = new TxRecordSerializer(cctx);
+ this.txRecordSerializer = new TxRecordSerializer();
this.co = cctx.kernalContext().cacheObjects();
this.pageSize = cctx.database().pageSize();
}
@@ -161,6 +162,12 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
return 4 + dataSize(dataRec);
+ case METASTORE_DATA_RECORD:
+ MetastoreDataRecord metastoreDataRec = (MetastoreDataRecord)record;
+
+ return 4 + metastoreDataRec.key().getBytes().length + 4 +
+ (metastoreDataRec.value() != null ? metastoreDataRec.value().length : 0);
+
case HEADER_RECORD:
return HEADER_RECORD_DATA_SIZE;
@@ -283,7 +290,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
return 0;
case TX_RECORD:
- return txRecordSerializer.sizeOfTxRecord((TxRecord)record);
+ return txRecordSerializer.size((TxRecord)record);
default:
throw new UnsupportedOperationException("Type: " + record.type());
@@ -384,6 +391,31 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
break;
+ case METASTORE_DATA_RECORD:
+ int strLen = in.readInt();
+
+ byte[] strBytes = new byte[strLen];
+
+ in.readFully(strBytes);
+
+ String key = new String(strBytes);
+
+ int valLen = in.readInt();
+
+ assert valLen >= 0;
+
+ byte[] val;
+
+ if (valLen > 0) {
+ val = new byte[valLen];
+
+ in.readFully(val);
+ }
+ else
+ val = null;
+
+ return new MetastoreDataRecord(key, val);
+
case HEADER_RECORD:
long magic = in.readLong();
@@ -783,13 +815,16 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
case PART_META_UPDATE_STATE:
cacheId = in.readInt();
+
partId = in.readInt();
state = in.readByte();
long updateCntr = in.readLong();
- res = new PartitionMetaStateRecord(cacheId, partId, GridDhtPartitionState.fromOrdinal(state), updateCntr);
+ GridDhtPartitionState partState = GridDhtPartitionState.fromOrdinal(state);
+
+ res = new PartitionMetaStateRecord(cacheId, partId, partState, updateCntr);
break;
@@ -804,7 +839,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
throw new EOFException("END OF SEGMENT");
case TX_RECORD:
- res = txRecordSerializer.readTxRecord(in);
+ res = txRecordSerializer.read(in);
break;
@@ -906,6 +941,29 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
break;
+ case METASTORE_DATA_RECORD:
+ MetastoreDataRecord metastoreDataRecord = (MetastoreDataRecord)rec;
+
+ byte[] strBytes = metastoreDataRecord.key().getBytes();
+
+ buf.putInt(strBytes.length);
+ buf.put(strBytes);
+ if (metastoreDataRecord.value() != null) {
+ buf.putInt(metastoreDataRecord.value().length);
+ buf.put(metastoreDataRecord.value());
+ }
+ else
+ buf.putInt(0);
+
+ break;
+
+ case HEADER_RECORD:
+ buf.putLong(HeaderRecord.REGULAR_MAGIC);
+
+ buf.putInt(((HeaderRecord)rec).version());
+
+ break;
+
case DATA_PAGE_INSERT_RECORD:
DataPageInsertRecord diRec = (DataPageInsertRecord)rec;
@@ -1294,7 +1352,7 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
break;
case TX_RECORD:
- txRecordSerializer.writeTxRecord((TxRecord)rec, buf);
+ txRecordSerializer.write((TxRecord)rec, buf);
break;
@@ -1307,6 +1365,15 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
}
/**
+ * Return shared cache context.
+ *
+ * @return Shared cache context.
+ */
+ public GridCacheSharedContext cctx() {
+ return cctx;
+ }
+
+ /**
* @param buf Buffer to write to.
* @param entry Data entry.
*/
@@ -1558,5 +1625,4 @@ public class RecordDataV1Serializer implements RecordDataSerializer {
return size;
}
-
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
index 0e574f8..efba611 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
@@ -17,16 +17,27 @@
package org.apache.ignite.internal.processors.cache.persistence.wal.serializer;
+import java.io.DataInput;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.wal.record.BaselineTopologyRecord;
+import org.apache.ignite.internal.pagemem.wal.record.CacheState;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord;
import org.apache.ignite.internal.pagemem.wal.record.SnapshotRecord;
+import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
/**
@@ -39,6 +50,12 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
/** V1 data serializer delegate. */
private final RecordDataV1Serializer delegateSerializer;
+ /** Serializer of {@link TxRecord} records. */
+ private final TxRecordSerializer txRecordSerializer;
+
+ /** Serializer of {@link BaselineTopologyRecord} records. */
+ private final BaselineTopologyRecordSerializer bltRecSerializer;
+
/**
* Create an instance of V2 data serializer.
*
@@ -46,22 +63,45 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
*/
public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) {
this.delegateSerializer = delegateSerializer;
+ this.txRecordSerializer = new TxRecordSerializer();
+ this.bltRecSerializer = new BaselineTopologyRecordSerializer(delegateSerializer.cctx());
}
/** {@inheritDoc} */
- @Override public int size(WALRecord record) throws IgniteCheckedException {
- switch (record.type()) {
+ @Override public int size(WALRecord rec) throws IgniteCheckedException {
+ switch (rec.type()) {
case HEADER_RECORD:
return HEADER_RECORD_DATA_SIZE;
+ case CHECKPOINT_RECORD:
+ CheckpointRecord cpRec = (CheckpointRecord)rec;
+
+ assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer :
+ "Invalid WAL record: " + cpRec;
+
+ int cacheStatesSize = cacheStatesSize(cpRec.cacheGroupStates());
+
+ FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark();
+
+ return 18 + cacheStatesSize + (walPtr == null ? 0 : 16);
+
case DATA_RECORD:
- return delegateSerializer.size(record) + 8/*timestamp*/;
+ return delegateSerializer.size(rec) + 8/*timestamp*/;
case SNAPSHOT:
return 8 + 1;
+ case EXCHANGE:
+ return 4 /*type*/ + 8 /*timestamp*/ + 2 /*constId*/;
+
+ case TX_RECORD:
+ return txRecordSerializer.size((TxRecord)rec);
+
+ case BASELINE_TOP_RECORD:
+ return bltRecSerializer.size((BaselineTopologyRecord)rec);
+
default:
- return delegateSerializer.size(record);
+ return delegateSerializer.size(rec);
}
}
@@ -71,6 +111,26 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
ByteBufferBackedDataInput in
) throws IOException, IgniteCheckedException {
switch (type) {
+ case CHECKPOINT_RECORD:
+ long msb = in.readLong();
+ long lsb = in.readLong();
+ boolean hasPtr = in.readByte() != 0;
+ int idx0 = hasPtr ? in.readInt() : 0;
+ int off = hasPtr ? in.readInt() : 0;
+ int len = hasPtr ? in.readInt() : 0;
+
+ Map<Integer, CacheState> states = readPartitionStates(in);
+
+ boolean end = in.readByte() != 0;
+
+ FileWALPointer walPtr = hasPtr ? new FileWALPointer(idx0, off, len) : null;
+
+ CheckpointRecord cpRec = new CheckpointRecord(new UUID(msb, lsb), walPtr, end);
+
+ cpRec.cacheGroupStates(states);
+
+ return cpRec;
+
case DATA_RECORD:
int entryCnt = in.readInt();
long timeStamp = in.readLong();
@@ -81,26 +141,65 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
entries.add(delegateSerializer.readDataEntry(in));
return new DataRecord(entries, timeStamp);
-
case SNAPSHOT:
long snpId = in.readLong();
byte full = in.readByte();
return new SnapshotRecord(snpId, full == 1);
+ case EXCHANGE:
+ int idx = in.readInt();
+ short constId = in.readShort();
+ long ts = in.readLong();
+
+ return new ExchangeRecord(constId, ExchangeRecord.Type.values()[idx], ts);
+
+ case TX_RECORD:
+ return txRecordSerializer.read(in);
+
+ case BASELINE_TOP_RECORD:
+ return bltRecSerializer.read(in);
+
default:
return delegateSerializer.readRecord(type, in);
}
+
}
/** {@inheritDoc} */
- @Override public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException {
- if (record instanceof HeaderRecord)
+ @Override public void writeRecord(WALRecord rec, ByteBuffer buf) throws IgniteCheckedException {
+ if (rec instanceof HeaderRecord)
throw new UnsupportedOperationException("Writing header records is forbidden since version 2 of serializer");
- switch (record.type()) {
+ switch (rec.type()) {
+ case CHECKPOINT_RECORD:
+ CheckpointRecord cpRec = (CheckpointRecord)rec;
+
+ assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer :
+ "Invalid WAL record: " + cpRec;
+
+ FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark();
+ UUID cpId = cpRec.checkpointId();
+
+ buf.putLong(cpId.getMostSignificantBits());
+ buf.putLong(cpId.getLeastSignificantBits());
+
+ buf.put(walPtr == null ? (byte)0 : 1);
+
+ if (walPtr != null) {
+ buf.putLong(walPtr.index());
+ buf.putInt(walPtr.fileOffset());
+ buf.putInt(walPtr.length());
+ }
+
+ putCacheStates(buf, cpRec.cacheGroupStates());
+
+ buf.put(cpRec.end() ? (byte)1 : 0);
+
+ break;
+
case DATA_RECORD:
- DataRecord dataRec = (DataRecord)record;
+ DataRecord dataRec = (DataRecord)rec;
buf.putInt(dataRec.writeEntries().size());
buf.putLong(dataRec.timestamp());
@@ -111,15 +210,118 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
break;
case SNAPSHOT:
- SnapshotRecord snpRec = (SnapshotRecord)record;
+ SnapshotRecord snpRec = (SnapshotRecord)rec;
buf.putLong(snpRec.getSnapshotId());
buf.put(snpRec.isFull() ? (byte)1 : 0);
break;
+ case EXCHANGE:
+ ExchangeRecord r = (ExchangeRecord)rec;
+
+ buf.putInt(r.getType().ordinal());
+ buf.putShort(r.getConstId());
+ buf.putLong(r.timestamp());
+
+ break;
+
+ case TX_RECORD:
+ txRecordSerializer.write((TxRecord)rec, buf);
+
+ break;
+
+ case BASELINE_TOP_RECORD:
+ bltRecSerializer.write((BaselineTopologyRecord)rec, buf);
+
+ break;
+
default:
- delegateSerializer.writeRecord(record, buf);
+ delegateSerializer.writeRecord(rec, buf);
+ }
+ }
+
+ /**
+ * @param buf Buffer to read from.
+ * @return Read map.
+ */
+ private Map<Integer, CacheState> readPartitionStates(DataInput buf) throws IOException {
+ int caches = buf.readShort() & 0xFFFF;
+
+ if (caches == 0)
+ return Collections.emptyMap();
+
+ Map<Integer, CacheState> states = new HashMap<>(caches, 1.0f);
+
+ for (int i = 0; i < caches; i++) {
+ int cacheId = buf.readInt();
+
+ int parts = buf.readShort() & 0xFFFF;
+
+ CacheState state = new CacheState(parts);
+
+ for (int p = 0; p < parts; p++) {
+ int partId = buf.readShort() & 0xFFFF;
+ long size = buf.readLong();
+ long partCntr = buf.readLong();
+ byte partState = buf.readByte();
+
+ state.addPartitionState(partId, size, partCntr, partState);
+ }
+
+ states.put(cacheId, state);
}
+
+ return states;
+ }
+
+ /**
+ * @param states Cache states.
+ */
+ private static void putCacheStates(ByteBuffer buf, Map<Integer, CacheState> states) {
+ buf.putShort((short)states.size());
+
+ for (Map.Entry<Integer, CacheState> entry : states.entrySet()) {
+ buf.putInt(entry.getKey());
+
+ CacheState state = entry.getValue();
+
+ // Need 2 bytes for the number of partitions.
+ buf.putShort((short)state.size());
+
+ for (int i = 0; i < state.size(); i++) {
+ short partIdx = (short)state.partitionByIndex(i);
+
+ buf.putShort(partIdx);
+
+ buf.putLong(state.partitionSizeByIndex(i));
+ buf.putLong(state.partitionCounterByIndex(i));
+ buf.put(state.stateByIndex(i));
+ }
+ }
+ }
+
+ /**
+ * @param states Partition states.
+ * @return Size required to write partition states.
+ */
+ private int cacheStatesSize(Map<Integer, CacheState> states) {
+ // Need 4 bytes for the number of caches.
+ int size = 2;
+
+ for (Map.Entry<Integer, CacheState> entry : states.entrySet()) {
+ // Cache ID.
+ size += 4;
+
+ // Need 2 bytes for the number of partitions.
+ size += 2;
+
+ CacheState state = entry.getValue();
+
+ // 2 bytes partition ID, size and counter per partition, part state.
+ size += 19 * state.size();
+ }
+
+ return size;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
index 11bd16b..5ae269b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -125,8 +125,10 @@ public class RecordV1Serializer implements RecordSerializer {
final WALRecord rec = dataSerializer.readRecord(recType, in);
+ rec.position(ptr);
+
if (recordFilter != null && !recordFilter.apply(rec.type(), ptr))
- return new FilteredRecord();
+ return FilteredRecord.INSTANCE;
else if (marshalledMode) {
ByteBuffer buf = heapTlb.get();
@@ -137,8 +139,6 @@ public class RecordV1Serializer implements RecordSerializer {
else
buf.clear();
- rec.position(ptr);
-
writeRecord(rec, buf);
buf.flip();
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
index adfd701..feeb810 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
@@ -116,7 +116,7 @@ public class RecordV2Serializer implements RecordSerializer {
if (in.skipBytes(toSkip) < toSkip)
throw new EOFException("Reached end of file while reading record: " + ptr);
- return new FilteredRecord();
+ return FilteredRecord.INSTANCE;
}
else if (marshalledMode) {
ByteBuffer buf = heapTlb.get();
@@ -144,8 +144,13 @@ public class RecordV2Serializer implements RecordSerializer {
return new MarshalledRecord(recType, ptr, buf);
}
- else
- return dataSerializer.readRecord(recType, in);
+ else {
+ WALRecord rec = dataSerializer.readRecord(recType, in);
+
+ rec.position(ptr);
+
+ return rec;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/TxRecordSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/TxRecordSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/TxRecordSerializer.java
index e8b324d..2f6d4c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/TxRecordSerializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/TxRecordSerializer.java
@@ -21,11 +21,9 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
import java.util.Map;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -36,60 +34,38 @@ import org.apache.ignite.transactions.TransactionState;
* {@link TxRecord} WAL serializer.
*/
public class TxRecordSerializer {
- /** Cache shared context. */
- private GridCacheSharedContext cctx;
-
- /** Class loader to unmarshal consistent ids. */
- private ClassLoader classLoader;
-
- /**
- * Create an instance of serializer.
- *
- * @param cctx Cache shared context.
- */
- public TxRecordSerializer(GridCacheSharedContext cctx) {
- this.cctx = cctx;
-
- classLoader = U.resolveClassLoader(cctx.gridConfig());
- }
-
/**
* Writes {@link TxRecord} to given buffer.
*
- * @param record TxRecord.
+ * @param rec TxRecord.
* @param buf Byte buffer.
* @throws IgniteCheckedException In case of fail.
*/
- public void writeTxRecord(TxRecord record, ByteBuffer buf) throws IgniteCheckedException {
- buf.put((byte) record.state().ordinal());
- RecordV1Serializer.putVersion(buf, record.nearXidVersion(), true);
- RecordV1Serializer.putVersion(buf, record.writeVersion(), true);
+ public void write(TxRecord rec, ByteBuffer buf) throws IgniteCheckedException {
+ buf.put((byte)rec.state().ordinal());
+ RecordV1Serializer.putVersion(buf, rec.nearXidVersion(), true);
+ RecordV1Serializer.putVersion(buf, rec.writeVersion(), true);
+
+ Map<Short, Collection<Short>> participatingNodes = rec.participatingNodes();
- if (record.participatingNodes() != null) {
- buf.putInt(record.participatingNodes().keySet().size());
+ if (participatingNodes != null && !participatingNodes.isEmpty()) {
+ buf.putInt(participatingNodes.size());
- for (Object primaryNode : record.participatingNodes().keySet()) {
- writeConsistentId(primaryNode, buf);
+ for (Map.Entry<Short, Collection<Short>> e : participatingNodes.entrySet()) {
+ buf.putShort(e.getKey());
- Collection<Object> backupNodes = record.participatingNodes().get(primaryNode);
+ Collection<Short> backupNodes = e.getValue();
buf.putInt(backupNodes.size());
- for (Object backupNode : backupNodes)
- writeConsistentId(backupNode, buf);
+ for (short backupNode : backupNodes)
+ buf.putShort(backupNode);
}
}
- else {
- // Put zero size of participating nodes.
- buf.putInt(0);
- }
-
- buf.put((byte) (record.remote() ? 1 : 0));
+ else
+ buf.putInt(0); // Put zero size of participating nodes.
- if (record.remote())
- writeConsistentId(record.primaryNode(), buf);
-
- buf.putLong(record.timestamp());
+ buf.putLong(rec.timestamp());
}
/**
@@ -100,7 +76,7 @@ public class TxRecordSerializer {
* @throws IOException In case of fail.
* @throws IgniteCheckedException In case of fail.
*/
- public TxRecord readTxRecord(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
+ public TxRecord read(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
byte txState = in.readByte();
TransactionState state = TransactionState.fromOrdinal(txState);
@@ -108,17 +84,18 @@ public class TxRecordSerializer {
GridCacheVersion writeVer = RecordV1Serializer.readVersion(in, true);
int participatingNodesSize = in.readInt();
- Map<Object, Collection<Object>> participatingNodes = new HashMap<>(2 * participatingNodesSize);
+
+ Map<Short, Collection<Short>> participatingNodes = U.newHashMap(participatingNodesSize);
for (int i = 0; i < participatingNodesSize; i++) {
- Object primaryNode = readConsistentId(in);
+ short primaryNode = in.readShort();
int backupNodesSize = in.readInt();
- Collection<Object> backupNodes = new ArrayList<>(backupNodesSize);
+ Collection<Short> backupNodes = new ArrayList<>(backupNodesSize);
for (int j = 0; j < backupNodesSize; j++) {
- Object backupNode = readConsistentId(in);
+ short backupNode = in.readShort();
backupNodes.add(backupNode);
}
@@ -126,102 +103,41 @@ public class TxRecordSerializer {
participatingNodes.put(primaryNode, backupNodes);
}
- boolean hasRemote = in.readByte() == 1;
-
- Object primaryNode = null;
-
- if (hasRemote)
- primaryNode = readConsistentId(in);
-
- long timestamp = in.readLong();
+ long ts = in.readLong();
- return new TxRecord(state, nearXidVer, writeVer, participatingNodes, primaryNode, timestamp);
+ return new TxRecord(state, nearXidVer, writeVer, participatingNodes, ts);
}
/**
* Returns size of marshalled {@link TxRecord} in bytes.
*
- * @param record TxRecord.
+ * @param rec TxRecord.
* @return Size of TxRecord in bytes.
* @throws IgniteCheckedException In case of fail.
*/
- public int sizeOfTxRecord(TxRecord record) throws IgniteCheckedException {
+ public int size(TxRecord rec) throws IgniteCheckedException {
int size = 0;
size += /* transaction state. */ 1;
- size += CacheVersionIO.size(record.nearXidVersion(), true);
- size += CacheVersionIO.size(record.writeVersion(), true);
+ size += CacheVersionIO.size(rec.nearXidVersion(), true);
+ size += CacheVersionIO.size(rec.writeVersion(), true);
size += /* primary nodes count. */ 4;
- if (record.participatingNodes() != null) {
- for (Object primaryNode : record.participatingNodes().keySet()) {
- size += /* byte array length. */ 4;
- size += marshalConsistentId(primaryNode).length;
+ Map<Short, Collection<Short>> participatingNodes = rec.participatingNodes();
- Collection<Object> backupNodes = record.participatingNodes().get(primaryNode);
+ if (participatingNodes != null && !participatingNodes.isEmpty()) {
+ for (Collection<Short> backupNodes : participatingNodes.values()) {
+ size += /* Compact ID. */ 2;
size += /* size of backup nodes. */ 4;
- for (Object backupNode : backupNodes) {
- size += /* byte array length. */ 4;
- size += marshalConsistentId(backupNode).length;
- }
+ size += /* Compact ID. */ 2 * backupNodes.size();
}
}
- size += /* Is primary node exist. */ 1;
-
- if (record.remote()) {
- size += /* byte array length. */ 4;
- size += marshalConsistentId(record.primaryNode()).length;
- }
-
size += /* Timestamp */ 8;
return size;
}
-
- /**
- * Marshal consistent id to byte array.
- *
- * @param consistentId Consistent id.
- * @return Marshalled byte array.
- * @throws IgniteCheckedException In case of fail.
- */
- private byte[] marshalConsistentId(Object consistentId) throws IgniteCheckedException {
- return cctx.marshaller().marshal(consistentId);
- }
-
- /**
- * Read consistent id from given input.
- *
- * @param in Input.
- * @return Consistent id.
- * @throws IOException In case of fail.
- * @throws IgniteCheckedException In case of fail.
- */
- private Object readConsistentId(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
- int len = in.readInt();
- in.ensure(len);
-
- byte[] content = new byte[len];
- in.readFully(content);
-
- return cctx.marshaller().unmarshal(content, classLoader);
- }
-
- /**
- * Write consistent id to given buffer.
- *
- * @param consistentId Consistent id.
- * @param buf Byte buffer.
- * @throws IgniteCheckedException In case of fail.
- */
- private void writeConsistentId(Object consistentId, ByteBuffer buf) throws IgniteCheckedException {
- byte[] content = marshalConsistentId(consistentId);
-
- buf.putInt(content.length);
- buf.put(content);
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index e862c0a..6c5922a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -230,6 +230,12 @@ public abstract class GridCacheStoreManagerAdapter extends GridCacheManagerAdapt
"Write-behind mode for the cache store also requires CacheConfiguration.setWriteThrough(true) " +
"property. Fix configuration for the cache: " + cfg.getName());
}
+
+ if (cctx.group().persistenceEnabled() && (cfg.isWriteThrough() || cfg.isReadThrough()))
+ U.quietAndWarn(log,
+ "Both Ignite native persistence and CacheStore are configured for cache '" + cfg.getName() + "'. " +
+ "This configuration does not guarantee strict consistency between CacheStore and Ignite data " +
+ "storage upon restarts. Consult documentation for more details.");
}
sesLsnrs = CU.startStoreSessionListeners(cctx.kernalContext(), cfg.getCacheStoreSessionListenerFactories());
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 00c637e..0fe2dd4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -45,6 +45,7 @@ import org.apache.ignite.cache.store.CacheStore;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.managers.discovery.ConsistentIdMapper;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheInvokeEntry;
@@ -64,6 +65,7 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheLazyPlainVer
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
import org.apache.ignite.internal.util.GridSetWrapper;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -1001,6 +1003,8 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
boolean notify = false;
+ WALPointer ptr = null;
+
synchronized (this) {
prev = this.state;
@@ -1089,48 +1093,63 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
log.debug("Invalid transaction state transition [invalid=" + state + ", cur=" + this.state +
", tx=" + this + ']');
}
- }
-
- if (notify) {
- GridFutureAdapter<IgniteInternalTx> fut = finFut;
- if (fut != null)
- fut.onDone(this);
- }
+ if (valid) {
+ // Seal transactions maps.
+ if (state != ACTIVE && state != SUSPENDED)
+ seal();
- if (valid) {
- // Seal transactions maps.
- if (state != ACTIVE && state != SUSPENDED)
- seal();
+ if (cctx.wal() != null && cctx.tm().logTxRecords()) {
+ // Log tx state change to WAL.
+ if (state == PREPARED || state == COMMITTED || state == ROLLED_BACK) {
+ assert txNodes != null || state == ROLLED_BACK : "txNodes=" + txNodes + " state=" + state;
- if (cctx.wal() != null && cctx.tm().logTxRecords()) {
- // Log tx state change to WAL.
- if (state == PREPARED || state == COMMITTED || state == ROLLED_BACK) {
- assert txNodes != null || state == ROLLED_BACK;
+ BaselineTopology baselineTop = cctx.kernalContext().state().clusterState().baselineTopology();
- Map<Object, Collection<Object>> participatingNodes = consistentIdMapper
- .mapToConsistentIds(topVer, txNodes);
+ Map<Short, Collection<Short>> participatingNodes = consistentIdMapper
+ .mapToCompactIds(topVer, txNodes, baselineTop);
- TxRecord txRecord = new TxRecord(
+ TxRecord txRecord = new TxRecord(
state,
nearXidVersion(),
writeVersion(),
- participatingNodes,
- remote() ? nodeId() : null
- );
+ participatingNodes
+ );
- try {
- cctx.wal().log(txRecord);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to log TxRecord: " + txRecord, e);
+ try {
+ ptr = cctx.wal().log(txRecord);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to log TxRecord: " + txRecord, e);
- throw new IgniteException("Failed to log TxRecord: " + txRecord, e);
+ throw new IgniteException("Failed to log TxRecord: " + txRecord, e);
+ }
}
}
}
}
+ if (valid) {
+ if (ptr != null && (state == COMMITTED || state == ROLLED_BACK))
+ try {
+ cctx.wal().fsync(ptr);
+ }
+ catch (IgniteCheckedException e) {
+ String msg = "Failed to fsync ptr: " + ptr;
+
+ U.error(log, msg, e);
+
+ throw new IgniteException(msg, e);
+ }
+ }
+
+ if (notify) {
+ GridFutureAdapter<IgniteInternalTx> fut = finFut;
+
+ if (fut != null)
+ fut.onDone(this);
+ }
+
return valid;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 7da4898..2a0b176 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -487,7 +487,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
checkValid();
- Collection<IgniteTxEntry> commitEntries = near() ? allEntries() : writeEntries();
+ Collection<IgniteTxEntry> commitEntries = (near() || cctx.snapshot().needTxReadLogging()) ? allEntries() : writeEntries();
boolean empty = F.isEmpty(commitEntries);
@@ -648,19 +648,6 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
if (dhtVer == null)
dhtVer = explicitVer != null ? explicitVer : writeVersion();
- if (cacheCtx.group().persistenceEnabled() && !writeEntries().isEmpty()
- && op != NOOP && op != RELOAD && op != READ)
- ptr = cctx.wal().log(new DataRecord(new DataEntry(
- cacheCtx.cacheId(),
- txEntry.key(),
- val,
- op,
- nearXidVersion(),
- writeVersion(),
- 0,
- txEntry.key().partition(),
- txEntry.updateCounter())));
-
if (op == CREATE || op == UPDATE) {
assert val != null : txEntry;
@@ -690,6 +677,9 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
if (updRes.success())
txEntry.updateCounter(updRes.updatePartitionCounter());
+ if (updRes.loggedPointer() != null)
+ ptr = updRes.loggedPointer();
+
if (nearCached != null && updRes.success()) {
nearCached.innerSet(
null,
@@ -738,6 +728,9 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
if (updRes.success())
txEntry.updateCounter(updRes.updatePartitionCounter());
+ if (updRes.loggedPointer() != null)
+ ptr = updRes.loggedPointer();
+
if (nearCached != null && updRes.success()) {
nearCached.innerRemove(
null,
@@ -766,6 +759,19 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
nearCached.innerReload();
}
else if (op == READ) {
+ if (cacheCtx.group().persistenceEnabled() && cctx.snapshot().needTxReadLogging()) {
+ ptr = cctx.wal().log(new DataRecord(new DataEntry(
+ cacheCtx.cacheId(),
+ txEntry.key(),
+ val,
+ op,
+ nearXidVersion(),
+ writeVersion(),
+ 0,
+ txEntry.key().partition(),
+ txEntry.updateCounter())));
+ }
+
ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
if (expiry != null) {
@@ -866,7 +872,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
}
}
- if (ptr != null)
+ if (ptr != null && !cctx.tm().logTxRecords())
cctx.wal().fsync(ptr);
}
catch (StorageException e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index a9353f4..932acf6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -1589,6 +1589,8 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
GridCacheContext cacheCtx = txEntry1.context();
while (true) {
+ cctx.database().checkpointReadLock();
+
try {
GridCacheEntryEx entry1 = txEntry1.cached();
@@ -1641,6 +1643,9 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
throw new IgniteCheckedException("Entry lock has been cancelled for transaction: " + tx);
}
+ finally {
+ cctx.database().checkpointReadUnlock();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
new file mode 100644
index 0000000..7651f6e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
@@ -0,0 +1,235 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
+import org.apache.ignite.compute.ComputeTaskAdapter;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.lang.GridIterator;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+@GridInternal
+public class CollectConflictPartitionKeysTask extends ComputeTaskAdapter<PartitionKey,
+ Map<PartitionHashRecord, List<PartitionEntryHashRecord>>> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Injected logger. */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
+ List<ClusterNode> subgrid, PartitionKey partKey) throws IgniteException {
+ Map<ComputeJob, ClusterNode> jobs = new HashMap<>();
+
+ for (ClusterNode node : subgrid)
+ jobs.put(new CollectConflictPartitionKeysTask.CollectPartitionEntryHashesJob(partKey), node);
+
+ return jobs;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Map<PartitionHashRecord, List<PartitionEntryHashRecord>> reduce(List<ComputeJobResult> results)
+ throws IgniteException {
+ Map<PartitionHashRecord, List<PartitionEntryHashRecord>> totalRes = new HashMap<>();
+
+ for (ComputeJobResult res : results) {
+ Map<PartitionHashRecord, List<PartitionEntryHashRecord>> nodeRes = res.getData();
+
+ totalRes.putAll(nodeRes);
+ }
+
+ Set<PartitionEntryHashRecord> commonEntries = null;
+
+ for (List<PartitionEntryHashRecord> nodeEntryHashRecords : totalRes.values()) {
+ HashSet<PartitionEntryHashRecord> set = new HashSet<>(nodeEntryHashRecords);
+
+ if (commonEntries == null)
+ commonEntries = set;
+ else
+ commonEntries.retainAll(set);
+ }
+
+ if (commonEntries == null)
+ return Collections.emptyMap();
+
+ Map<PartitionHashRecord, List<PartitionEntryHashRecord>> conflictsRes = new HashMap<>();
+
+ for (Map.Entry<PartitionHashRecord, List<PartitionEntryHashRecord>> e : totalRes.entrySet()) {
+ HashSet<PartitionEntryHashRecord> conflicts = new HashSet<>(e.getValue());
+
+ conflicts.removeAll(commonEntries);
+
+ if (!conflicts.isEmpty())
+ conflictsRes.put(e.getKey(), new ArrayList<>(conflicts));
+ }
+
+ return conflictsRes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
+ ComputeJobResultPolicy superRes = super.result(res, rcvd);
+
+ // Deny failover.
+ if (superRes == ComputeJobResultPolicy.FAILOVER) {
+ superRes = ComputeJobResultPolicy.WAIT;
+
+ log.warning("CollectPartitionEntryHashesJob failed on node " +
+ "[consistentId=" + res.getNode().consistentId() + "]", res.getException());
+ }
+
+ return superRes;
+ }
+
+ /**
+ *
+ */
+ public static class CollectPartitionEntryHashesJob extends ComputeJobAdapter {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Ignite instance. */
+ @IgniteInstanceResource
+ private IgniteEx ignite;
+
+ /** Injected logger. */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** Partition key. */
+ private PartitionKey partKey;
+
+ /**
+ * @param partKey Partition key.
+ */
+ private CollectPartitionEntryHashesJob(PartitionKey partKey) {
+ this.partKey = partKey;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<PartitionHashRecord, List<PartitionEntryHashRecord>> execute() throws IgniteException {
+ CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(partKey.groupId());
+
+ if (grpCtx == null)
+ return Collections.emptyMap();
+
+ partKey.groupName(grpCtx.cacheOrGroupName());
+
+ GridDhtLocalPartition part = grpCtx.topology().localPartition(partKey.partitionId());
+
+ if (part == null || !part.reserve())
+ return Collections.emptyMap();
+
+ int partHash = 0;
+ long partSize;
+ long updateCntrBefore;
+ List<PartitionEntryHashRecord> partEntryHashRecords;
+
+ try {
+ if (part.state() != GridDhtPartitionState.OWNING)
+ return Collections.emptyMap();
+
+ updateCntrBefore = part.updateCounter();
+
+ partSize = part.dataStore().fullSize();
+
+ GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
+
+ partEntryHashRecords = new ArrayList<>();
+
+ while (it.hasNextX()) {
+ CacheDataRow row = it.nextX();
+
+ partHash += row.key().hashCode();
+
+ int valHash = Arrays.hashCode(row.value().valueBytes(grpCtx.cacheObjectContext()));
+ partHash += valHash;
+
+ int cacheId = row.cacheId() == 0 ? grpCtx.groupId() : row.cacheId();
+ DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheId);
+
+ assert desc != null;
+
+ Object o = CacheObjectUtils.unwrapBinaryIfNeeded(grpCtx.cacheObjectContext(), row.key(), true, true);
+
+ partEntryHashRecords.add(new PartitionEntryHashRecord(
+ cacheId, desc.cacheName(), row.key(), o.toString(),
+ row.key().valueBytes(grpCtx.cacheObjectContext()), row.version(), valHash));
+ }
+
+ long updateCntrAfter = part.updateCounter();
+
+ if (updateCntrBefore != updateCntrAfter) {
+ throw new IgniteException("Cluster is not idle: update counter of partition " + partKey.toString() +
+ " changed during hash calculation [before=" + updateCntrBefore +
+ ", after=" + updateCntrAfter + "]");
+ }
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Can't calculate partition hash " + partKey.toString(), e);
+
+ return Collections.emptyMap();
+ }
+ finally {
+ part.release();
+ }
+
+ Object consId = ignite.context().discovery().localNode().consistentId();
+
+ boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
+
+ PartitionHashRecord partHashRec = new PartitionHashRecord(
+ partKey, isPrimary, consId, partHash, updateCntrBefore, partSize);
+
+ Map<PartitionHashRecord, List<PartitionEntryHashRecord>> res = new HashMap<>();
+
+ res.put(partHashRec, partEntryHashRecords);
+
+ return res;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionEntryHashRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionEntryHashRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionEntryHashRecord.java
new file mode 100644
index 0000000..6b0d345
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionEntryHashRecord.java
@@ -0,0 +1,237 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class PartitionEntryHashRecord implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Cache id. */
+ @GridToStringExclude
+ private final int cacheId;
+
+ /** Cache name. */
+ private final String cacheName;
+
+ /** Binary key. */
+ @GridToStringExclude
+ private final KeyCacheObject key;
+
+ /** Key string. */
+ @GridToStringExclude
+ private final String keyStr;
+
+ /** Key bytes. */
+ private final byte[] keyBytes;
+
+ /** Grid Cache Version. */
+ private final GridCacheVersion ver;
+
+ /** Value. */
+ @GridToStringExclude
+ private volatile CacheObject val;
+
+ /** Value bytes. */
+ private volatile byte[] valBytes;
+
+ /** Value string. */
+ @GridToStringExclude
+ private volatile String valStr;
+
+ /** Value hash. */
+ @GridToStringExclude
+ private final int valHash;
+
+ /**
+ * @param cacheId Cache id.
+ * @param cacheName Cache name.
+ * @param key Key.
+ * @param keyStr Key string.
+ * @param keyBytes Key bytes.
+ * @param ver Version.
+ * @param valHash Value hash.
+ * @param val Value.
+ * @param valStr Value string.
+ * @param valBytes Value bytes.
+ */
+ public PartitionEntryHashRecord(int cacheId, String cacheName, KeyCacheObject key, String keyStr,
+ byte[] keyBytes, GridCacheVersion ver, int valHash, CacheObject val, String valStr, byte[] valBytes) {
+ this.cacheId = cacheId;
+ this.cacheName = cacheName;
+ this.key = key;
+ this.keyStr = keyStr;
+ this.keyBytes = keyBytes;
+ this.ver = ver;
+ this.val = val;
+ this.valStr = valStr;
+ this.valHash = valHash;
+ this.valBytes = valBytes;
+ }
+
+ /**
+ * @param cacheId Cache id.
+ * @param cacheName Cache name.
+ * @param key Key.
+ * @param keyStr Key string.
+ * @param keyBytes Key bytes.
+ * @param ver Version.
+ * @param valHash Value hash.
+ */
+ public PartitionEntryHashRecord(int cacheId, String cacheName, KeyCacheObject key,
+ String keyStr, byte[] keyBytes, GridCacheVersion ver, int valHash) {
+ this(cacheId, cacheName, key, keyStr, keyBytes, ver, valHash, null, null, null);
+ }
+
+ /**
+ * @return Cache id.
+ */
+ public int cacheId() {
+ return cacheId;
+ }
+
+ /**
+ * @return Cache name.
+ */
+ public String cacheName() {
+ return cacheName;
+ }
+
+ /**
+ * @return Binary key.
+ */
+ public KeyCacheObject key() {
+ return key;
+ }
+
+ /**
+ * @return Key bytes.
+ */
+ public byte[] keyBytes() {
+ return keyBytes;
+ }
+
+ /**
+ * @return Grid Cache Version.
+ */
+ public GridCacheVersion version() {
+ return ver;
+ }
+
+ /**
+ * @return Value hash.
+ */
+ public int valueHash() {
+ return valHash;
+ }
+
+ /**
+ * @return Key string.
+ */
+ public String keyString() {
+ return keyStr;
+ }
+
+ /**
+ * @return Binary value.
+ */
+ public CacheObject value() {
+ return val;
+ }
+
+ /**
+ * @param val Value.
+ */
+ public void value(CacheObject val) {
+ this.val = val;
+ }
+
+ /**
+ * @return Value string.
+ */
+ public String valueString() {
+ return valStr;
+ }
+
+ /**
+ * @param valStr Value string.
+ */
+ public void valueString(String valStr) {
+ this.valStr = valStr;
+ }
+
+ /**
+ * @param valBytes Value bytes.
+ */
+ public void valueBytes(byte[] valBytes) {
+ this.valBytes = valBytes;
+ }
+
+ /**
+ * @return Value bytes.
+ */
+ public byte[] valueBytes() {
+ return valBytes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ PartitionEntryHashRecord that = (PartitionEntryHashRecord)o;
+
+ if (cacheId != that.cacheId)
+ return false;
+ if (valHash != that.valHash)
+ return false;
+ if (!Arrays.equals(keyBytes, that.keyBytes))
+ return false;
+ return ver != null ? ver.equals(that.ver) : that.ver == null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = cacheId;
+ res = 31 * res + Arrays.hashCode(keyBytes);
+ res = 31 * res + (ver != null ? ver.hashCode() : 0);
+ res = 31 * res + valHash;
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(PartitionEntryHashRecord.class, this,
+ "key", keyStr,
+ "value", valStr,
+ "keyBytes", keyBytes != null ? U.byteArray2HexString(keyBytes) : null,
+ "valueBytes", valBytes != null ? U.byteArray2HexString(valBytes) : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java
new file mode 100644
index 0000000..0f6e2e4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java
@@ -0,0 +1,130 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Record containing partition checksum, primary flag and consistent ID of owner.
+ */
+public class PartitionHashRecord implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Partition key. */
+ @GridToStringExclude
+ private final PartitionKey partKey;
+
+ /** Is primary flag. */
+ private final boolean isPrimary;
+
+ /** Consistent id. */
+ private final Object consistentId;
+
+ /** Partition hash. */
+ private final int partHash;
+
+ /** Update counter. */
+ private final long updateCntr;
+
+ /** Size. */
+ private final long size;
+
+ /**
+ * @param partKey Partition key.
+ * @param isPrimary Is primary.
+ * @param consistentId Consistent id.
+ * @param partHash Partition hash.
+ * @param updateCntr Update counter.
+ * @param size Size.
+ */
+ public PartitionHashRecord(PartitionKey partKey, boolean isPrimary,
+ Object consistentId, int partHash, long updateCntr, long size) {
+ this.partKey = partKey;
+ this.isPrimary = isPrimary;
+ this.consistentId = consistentId;
+ this.partHash = partHash;
+ this.updateCntr = updateCntr;
+ this.size = size;
+ }
+
+ /**
+ * @return Partition key.
+ */
+ public PartitionKey partitionKey() {
+ return partKey;
+ }
+
+ /**
+ * @return Is primary.
+ */
+ public boolean isPrimary() {
+ return isPrimary;
+ }
+
+ /**
+ * @return Consistent id.
+ */
+ public Object consistentId() {
+ return consistentId;
+ }
+
+ /**
+ * @return Partition hash.
+ */
+ public int partitionHash() {
+ return partHash;
+ }
+
+ /**
+ * @return Update counter.
+ */
+ public long updateCounter() {
+ return updateCntr;
+ }
+
+ /**
+ * @return Size.
+ */
+ public long size() {
+ return size;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(PartitionHashRecord.class, this, "consistentId", consistentId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ PartitionHashRecord record = (PartitionHashRecord)o;
+
+ return consistentId.equals(record.consistentId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return consistentId.hashCode();
+ }
+}
[06/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 927fd90..54e0f56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -17,22 +17,29 @@
package org.apache.ignite.internal.processors.cluster;
+import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.BaselineNode;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
@@ -46,6 +53,9 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.GridChangeGlobalStateMessageResponse;
import org.apache.ignite.internal.processors.cache.StateChangeRequest;
import org.apache.ignite.internal.processors.cache.StoredCacheData;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -58,8 +68,12 @@ import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.IgniteNodeValidationResult;
import org.apache.ignite.spi.discovery.DiscoveryDataBag;
import org.jetbrains.annotations.Nullable;
@@ -71,13 +85,31 @@ import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYS
/**
*
*/
-public class GridClusterStateProcessor extends GridProcessorAdapter {
+public class GridClusterStateProcessor extends GridProcessorAdapter implements IGridClusterStateProcessor, MetastorageLifecycleListener {
+ /** */
+ private static final String METASTORE_CURR_BLT_KEY = "metastoreBltKey";
+
+ /** */
+ private boolean inMemoryMode;
+
+ /**
+ * Compatibility mode flag. When node detects it runs in heterogeneous cluster (nodes of different versions),
+ * it should skip baseline topology operations.
+ */
+ private volatile boolean compatibilityMode;
+
/** */
private volatile DiscoveryDataClusterState globalState;
+ /** */
+ private final BaselineTopologyHistory bltHist = new BaselineTopologyHistory();
+
/** Local action future. */
private final AtomicReference<GridChangeGlobalStateFuture> stateChangeFut = new AtomicReference<>();
+ /** */
+ private final ConcurrentMap<UUID, GridFutureAdapter<Void>> transitionFuts = new ConcurrentHashMap<>();
+
/** Future initialized if node joins when cluster state change is in progress. */
private TransitionOnJoinWaitFuture joinFut;
@@ -89,6 +121,13 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
@GridToStringExclude
private GridCacheSharedContext<?, ?> sharedCtx;
+ /** Fully initialized metastorage. */
+ @GridToStringExclude
+ private ReadWriteMetastorage metastorage;
+
+ /** */
+ private final JdkMarshaller marsh = new JdkMarshaller();
+
/** Listener. */
private final GridLocalEventListener lsr = new GridLocalEventListener() {
@Override public void onEvent(Event evt) {
@@ -115,12 +154,20 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
*/
public GridClusterStateProcessor(GridKernalContext ctx) {
super(ctx);
+
+ ctx.internalSubscriptionProcessor().registerMetastorageListener(this);
}
/**
- * @return Cluster state to be used on public API.
+ * @return {@code True} if {@link IGridClusterStateProcessor} has detected that cluster is working
+ * in compatibility mode (nodes of different versions are joined to the cluster).
*/
- public boolean publicApiActiveState() {
+ public boolean compatibilityMode() {
+ return compatibilityMode;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean publicApiActiveState(boolean waitForTransition) {
if (ctx.isDaemon())
return sendComputeCheckGlobalState();
@@ -133,19 +180,120 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
if (transitionRes != null)
return transitionRes;
- else
- return false;
+ else {
+ if (waitForTransition) {
+ GridFutureAdapter<Void> fut = transitionFuts.get(globalState.transitionRequestId());
+
+ if (fut != null) {
+ try {
+ fut.get();
+ }
+ catch (IgniteCheckedException ex) {
+ throw new IgniteException(ex);
+ }
+ }
+
+ transitionRes = globalState.transitionResult();
+
+ assert transitionRes != null;
+
+ return transitionRes;
+ }
+ else
+ return false;
+ }
}
else
return globalState.active();
}
/** {@inheritDoc} */
+ @Override public void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException {
+ BaselineTopology blt = (BaselineTopology) metastorage.read(METASTORE_CURR_BLT_KEY);
+
+ if (blt != null) {
+ if (log.isInfoEnabled())
+ U.log(log, "Restoring history for BaselineTopology[id=" + blt.id() + "]");
+
+ bltHist.restoreHistory(metastorage, blt.id());
+ }
+
+ onStateRestored(blt);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onReadyForReadWrite(ReadWriteMetastorage metastorage) throws IgniteCheckedException {
+ this.metastorage = metastorage;
+
+ if (compatibilityMode) {
+ if (log.isInfoEnabled())
+ log.info("BaselineTopology won't be stored as this node is running in compatibility mode");
+
+ return;
+ }
+
+ writeBaselineTopology(globalState.baselineTopology(), null);
+
+ bltHist.flushHistoryItems(metastorage);
+ }
+
+ /**
+ * Resets branching history on current BaselineTopology.
+ *
+ * @throws IgniteCheckedException If write to metastore has failed.
+ */
+ public void resetBranchingHistory(long newBranchingHash) throws IgniteCheckedException {
+ globalState.baselineTopology().resetBranchingHistory(newBranchingHash);
+
+ writeBaselineTopology(globalState.baselineTopology(), null);
+
+ U.log(log,
+ String.format("Branching history of current BaselineTopology is reset to the value %d", newBranchingHash));
+ }
+
+ /**
+ * @param blt Blt.
+ */
+ private void writeBaselineTopology(BaselineTopology blt, BaselineTopologyHistoryItem prevBltHistItem) throws IgniteCheckedException {
+ assert metastorage != null;
+
+ sharedCtx.database().checkpointReadLock();
+
+ try {
+ if (blt != null) {
+ if (log.isInfoEnabled()) {
+ U.log(log, "Writing BaselineTopology[id=" + blt.id() + "]");
+
+ if (prevBltHistItem != null)
+ U.log(log, "Writing BaselineTopologyHistoryItem[id=" + prevBltHistItem.id() + "]");
+ }
+
+ bltHist.writeHistoryItem(metastorage, prevBltHistItem);
+
+ metastorage.write(METASTORE_CURR_BLT_KEY, blt);
+ }
+ else {
+ if (log.isInfoEnabled())
+ U.log(log, "Removing BaselineTopology and history");
+
+ metastorage.remove(METASTORE_CURR_BLT_KEY);
+
+ bltHist.removeHistory(metastorage);
+ }
+ }
+ finally {
+ sharedCtx.database().checkpointReadUnlock();
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public void start() throws IgniteCheckedException {
+ inMemoryMode = !CU.isPersistenceEnabled(ctx.config());
+
// Start first node as inactive if persistence is enabled.
- boolean activeOnStart = !CU.isPersistenceEnabled(ctx.config()) && ctx.config().isActiveOnStart();
+ boolean activeOnStart = inMemoryMode && ctx.config().isActiveOnStart();
- globalState = DiscoveryDataClusterState.createState(activeOnStart);
+ globalState = DiscoveryDataClusterState.createState(activeOnStart, null);
ctx.event().addLocalEventListener(lsr, EVT_NODE_LEFT, EVT_NODE_FAILED);
}
@@ -160,25 +308,46 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
super.onKernalStop(cancel);
}
- /**
- * @param discoCache Discovery data cache.
- * @return If transition is in progress returns future which is completed when transition finishes.
- */
- @Nullable public IgniteInternalFuture<Boolean> onLocalJoin(DiscoCache discoCache) {
- if (globalState.transition()) {
- joinFut = new TransitionOnJoinWaitFuture(globalState, discoCache);
+ /** {@inheritDoc} */
+ @Override @Nullable public IgniteInternalFuture<Boolean> onLocalJoin(DiscoCache discoCache) {
+ final DiscoveryDataClusterState state = globalState;
+
+ if (state.transition()) {
+ joinFut = new TransitionOnJoinWaitFuture(state, discoCache);
return joinFut;
}
+ else if (!ctx.clientNode()
+ && !ctx.isDaemon()
+ && ctx.config().isAutoActivationEnabled()
+ && !state.active()
+ && isBaselineSatisfied(state.baselineTopology(), discoCache.serverNodes()))
+ changeGlobalState0(true, state.baselineTopology(), false);
return null;
}
/**
- * @param node Failed node.
- * @return Message if cluster state changed.
+ * Checks whether all conditions to meet BaselineTopology are satisfied.
*/
- @Nullable public ChangeGlobalStateFinishMessage onNodeLeft(ClusterNode node) {
+ private boolean isBaselineSatisfied(BaselineTopology blt, List<ClusterNode> serverNodes) {
+ if (blt == null)
+ return false;
+
+ if (blt.consistentIds() == null)
+ return false;
+
+ if (//only node participating in BaselineTopology is allowed to send activation command...
+ blt.consistentIds().contains(ctx.discovery().localNode().consistentId())
+ //...and with this node BaselineTopology is reached
+ && blt.isSatisfied(serverNodes))
+ return true;
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override @Nullable public ChangeGlobalStateFinishMessage onNodeLeft(ClusterNode node) {
if (globalState.transition()) {
Set<UUID> nodes = globalState.transitionNodes();
@@ -187,7 +356,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
"Switching to inactive state.");
ChangeGlobalStateFinishMessage msg =
- new ChangeGlobalStateFinishMessage(globalState.transitionRequestId(), false);
+ new ChangeGlobalStateFinishMessage(globalState.transitionRequestId(), false, false);
onStateFinishMessage(msg);
@@ -198,14 +367,16 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
return null;
}
- /**
- * @param msg Message.
- */
- public void onStateFinishMessage(ChangeGlobalStateFinishMessage msg) {
- if (msg.requestId().equals(globalState.transitionRequestId())) {
+ /** {@inheritDoc} */
+ @Override public void onStateFinishMessage(ChangeGlobalStateFinishMessage msg) {
+ DiscoveryDataClusterState state = globalState;
+
+ if (msg.requestId().equals(state.transitionRequestId())) {
log.info("Received state change finish message: " + msg.clusterActive());
- globalState = DiscoveryDataClusterState.createState(msg.clusterActive());
+ globalState = globalState.finish(msg.success());
+
+ afterStateChangeFinished(msg.id(), msg.success());
ctx.cache().onStateChangeFinish(msg);
@@ -213,22 +384,39 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
if (joinFut != null)
joinFut.onDone(false);
+
+ GridFutureAdapter<Void> transitionFut = transitionFuts.remove(state.transitionRequestId());
+
+ if (transitionFut != null) {
+ state.setTransitionResult(msg.requestId(), msg.clusterActive());
+
+ transitionFut.onDone();
+ }
}
else
U.warn(log, "Received state finish message with unexpected ID: " + msg);
}
- /**
- * @param topVer Current topology version.
- * @param msg Message.
- * @param discoCache Current nodes.
- * @return {@code True} if need start state change process.
- */
- public boolean onStateChangeMessage(AffinityTopologyVersion topVer,
+ /** */
+ protected void afterStateChangeFinished(IgniteUuid msgId, boolean success) {
+ // no-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean onStateChangeMessage(
+ AffinityTopologyVersion topVer,
ChangeGlobalStateMessage msg,
- DiscoCache discoCache) {
- if (globalState.transition()) {
- if (globalState.active() != msg.activate()) {
+ DiscoCache discoCache
+ ) {
+ DiscoveryDataClusterState state = globalState;
+
+ if (log.isInfoEnabled())
+ U.log(log, "Received " + prettyStr(msg.activate()) + " request with BaselineTopology" +
+ (msg.baselineTopology() == null ? ": null"
+ : "[id=" + msg.baselineTopology().id() + "]"));
+
+ if (state.transition()) {
+ if (isApplicable(msg, state)) {
GridChangeGlobalStateFuture fut = changeStateFuture(msg);
if (fut != null)
@@ -237,27 +425,31 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
else {
final GridChangeGlobalStateFuture stateFut = changeStateFuture(msg);
- if (stateFut != null) {
- IgniteInternalFuture<?> exchFut = ctx.cache().context().exchange().affinityReadyFuture(
- globalState.transitionTopologyVersion());
+ GridFutureAdapter<Void> transitionFut = transitionFuts.get(state.transitionRequestId());
- if (exchFut == null)
- exchFut = new GridFinishedFuture<>();
+ if (stateFut != null && transitionFut != null) {
+ transitionFut.listen(new IgniteInClosure<IgniteInternalFuture<Void>>() {
+ @Override public void apply(IgniteInternalFuture<Void> fut) {
+ try {
+ fut.get();
+
+ stateFut.onDone();
+ }
+ catch (Exception ex) {
+ stateFut.onDone(ex);
+ }
- exchFut.listen(new CI1<IgniteInternalFuture<?>>() {
- @Override public void apply(IgniteInternalFuture<?> exchFut) {
- stateFut.onDone();
}
});
}
}
}
else {
- if (globalState.active() != msg.activate()) {
+ if (isApplicable(msg, state)) {
ExchangeActions exchangeActions;
try {
- exchangeActions = ctx.cache().onStateChangeRequest(msg, topVer);
+ exchangeActions = ctx.cache().onStateChangeRequest(msg, topVer, state);
}
catch (IgniteCheckedException e) {
GridChangeGlobalStateFuture fut = changeStateFuture(msg);
@@ -281,14 +473,25 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
if (log.isInfoEnabled())
log.info("Started state transition: " + msg.activate());
- globalState = DiscoveryDataClusterState.createTransitionState(msg.activate(),
+ BaselineTopologyHistoryItem bltHistItem = BaselineTopologyHistoryItem.fromBaseline(
+ globalState.baselineTopology());
+
+ transitionFuts.put(msg.requestId(), new GridFutureAdapter<Void>());
+
+ globalState = DiscoveryDataClusterState.createTransitionState(
+ globalState,
+ msg.activate(),
+ msg.activate() ? msg.baselineTopology() : globalState.baselineTopology(),
msg.requestId(),
topVer,
nodeIds);
+ if (msg.forceChangeBaselineTopology())
+ globalState.setTransitionResult(msg.requestId(), msg.activate());
+
AffinityTopologyVersion stateChangeTopVer = topVer.nextMinorVersion();
- StateChangeRequest req = new StateChangeRequest(msg, stateChangeTopVer);
+ StateChangeRequest req = new StateChangeRequest(msg, bltHistItem, msg.activate() != state.active(), stateChangeTopVer);
exchangeActions.stateChangeRequest(req);
@@ -309,12 +512,34 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
}
/**
- * @return Current cluster state, should be called only from discovery thread.
+ * @param msg State change message.
+ * @param state Current cluster state.
+ * @return {@code True} if state change from message can be applied to the current state.
+ */
+ protected boolean isApplicable(ChangeGlobalStateMessage msg, DiscoveryDataClusterState state) {
+ return !isEquivalent(msg, state);
+ }
+
+ /**
+ * @param msg State change message.
+ * @param state Current cluster state.
+ * @return {@code True} if states are equivalent.
*/
- public DiscoveryDataClusterState clusterState() {
+ protected static boolean isEquivalent(ChangeGlobalStateMessage msg, DiscoveryDataClusterState state) {
+ return (msg.activate() == state.active() && BaselineTopology.equals(msg.baselineTopology(), state.baselineTopology()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public DiscoveryDataClusterState clusterState() {
return globalState;
}
+ /** {@inheritDoc} */
+ @Override public DiscoveryDataClusterState pendingState(ChangeGlobalStateMessage stateMsg) {
+ return DiscoveryDataClusterState.createState(stateMsg.activate() || stateMsg.forceChangeBaselineTopology(),
+ stateMsg.baselineTopology());
+ }
+
/**
* @param msg State change message.
* @return Local future for state change process.
@@ -346,15 +571,13 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
* @param activate New state.
* @return State change error.
*/
- private IgniteCheckedException concurrentStateChangeError(boolean activate) {
+ protected IgniteCheckedException concurrentStateChangeError(boolean activate) {
return new IgniteCheckedException("Failed to " + prettyStr(activate) +
", because another state change operation is currently in progress: " + prettyStr(!activate));
}
- /**
- *
- */
- public void cacheProcessorStarted() {
+ /** {@inheritDoc} */
+ @Override public void cacheProcessorStarted() {
cacheProc = ctx.cache();
sharedCtx = cacheProc.context();
@@ -391,40 +614,192 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
}
/** {@inheritDoc} */
+ @Override public void collectJoiningNodeData(DiscoveryDataBag dataBag) {
+ try {
+ byte[] marshalledState = marsh.marshal(globalState);
+
+ dataBag.addJoiningNodeData(discoveryDataType().ordinal(), marshalledState);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public void collectGridNodeData(DiscoveryDataBag dataBag) {
- if (!dataBag.commonDataCollectedFor(STATE_PROC.ordinal()))
- dataBag.addGridCommonData(STATE_PROC.ordinal(), globalState);
+ if (!dataBag.commonDataCollectedFor(STATE_PROC.ordinal())) {
+ DiscoveryDataBag.JoiningNodeDiscoveryData joiningNodeData = dataBag.newJoinerDiscoveryData(STATE_PROC.ordinal());
+
+ BaselineTopologyHistory historyToSend = null;
+
+ if (joiningNodeData != null) {
+ if (!joiningNodeData.hasJoiningNodeData()) {
+ //compatibility mode: old nodes don't send any data on join, so coordinator of new version
+ //doesn't send BaselineTopology history, only its current globalState
+ dataBag.addGridCommonData(STATE_PROC.ordinal(), globalState);
+
+ return;
+ }
+
+ DiscoveryDataClusterState joiningNodeState = null;
+
+ try {
+ if (joiningNodeData.joiningNodeData() != null)
+ joiningNodeState = marsh.unmarshal(
+ (byte[]) joiningNodeData.joiningNodeData(),
+ U.resolveClassLoader(ctx.config()));
+ } catch (IgniteCheckedException e) {
+ U.error(log, "Failed to unmarshal disco data from joining node: " + joiningNodeData.joiningNodeId());
+
+ return;
+ }
+
+ if (!bltHist.isEmpty()) {
+ if (joiningNodeState != null && joiningNodeState.baselineTopology() != null) {
+ int lastId = joiningNodeState.baselineTopology().id();
+
+ historyToSend = bltHist.tailFrom(lastId);
+ }
+ else
+ historyToSend = bltHist;
+ }
+
+ dataBag.addGridCommonData(STATE_PROC.ordinal(), new BaselineStateAndHistoryData(globalState, historyToSend));
+ }
+ }
}
/** {@inheritDoc} */
@Override public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) {
- DiscoveryDataClusterState state = (DiscoveryDataClusterState)data.commonData();
+ if (data.commonData() instanceof DiscoveryDataClusterState) {
+ if (globalState != null && globalState.baselineTopology() != null)
+ //node with BaselineTopology is not allowed to join mixed cluster
+ // (where some nodes don't support BaselineTopology)
+ throw new IgniteException("Node with BaselineTopology cannot join" +
+ " mixed cluster running in compatibility mode");
+
+ globalState = (DiscoveryDataClusterState) data.commonData();
+
+ compatibilityMode = true;
+
+ return;
+ }
+
+ BaselineStateAndHistoryData stateDiscoData = (BaselineStateAndHistoryData)data.commonData();
+
+ if (stateDiscoData != null) {
+ DiscoveryDataClusterState state = stateDiscoData.globalState;
+
+ if (state.transition())
+ transitionFuts.put(state.transitionRequestId(), new GridFutureAdapter<Void>());
- if (state != null)
globalState = state;
+
+ if (stateDiscoData.recentHistory != null) {
+ for (BaselineTopologyHistoryItem item : stateDiscoData.recentHistory.history())
+ bltHist.bufferHistoryItemForStore(item);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInternalFuture<?> changeGlobalState(
+ final boolean activate,
+ Collection<? extends BaselineNode> baselineNodes,
+ boolean forceChangeBaselineTopology
+ ) {
+ if (inMemoryMode)
+ return changeGlobalState0(activate, null, false);
+
+ BaselineTopology newBlt = compatibilityMode ? null :
+ calculateNewBaselineTopology(activate, baselineNodes, forceChangeBaselineTopology);
+
+ return changeGlobalState0(activate, newBlt, forceChangeBaselineTopology);
}
/**
- * @param activate New cluster state.
- * @return State change future.
+ *
*/
- public IgniteInternalFuture<?> changeGlobalState(final boolean activate) {
+ private BaselineTopology calculateNewBaselineTopology(final boolean activate,
+ Collection<? extends BaselineNode> baselineNodes,
+ boolean forceChangeBaselineTopology) {
+ BaselineTopology newBlt;
+
+ BaselineTopology currentBlt = globalState.baselineTopology();
+
+ int newBltId = 0;
+
+ if (currentBlt != null)
+ newBltId = activate ? currentBlt.id() + 1 : currentBlt.id();
+
+ if (baselineNodes != null && !baselineNodes.isEmpty()) {
+ List<BaselineNode> baselineNodes0 = new ArrayList<>();
+
+ for (BaselineNode node : baselineNodes) {
+ if (node instanceof ClusterNode) {
+ ClusterNode clusterNode = (ClusterNode) node;
+
+ if (!clusterNode.isClient() && !clusterNode.isDaemon())
+ baselineNodes0.add(node);
+ }
+ else
+ baselineNodes0.add(node);
+ }
+
+ baselineNodes = baselineNodes0;
+ }
+
+ if (forceChangeBaselineTopology)
+ newBlt = BaselineTopology.build(baselineNodes, newBltId);
+ else if (activate) {
+ if (baselineNodes == null)
+ baselineNodes = baselineNodes();
+
+ if (currentBlt == null)
+ newBlt = BaselineTopology.build(baselineNodes, newBltId);
+ else {
+ newBlt = currentBlt;
+
+ newBlt.updateHistory(baselineNodes);
+ }
+ }
+ else
+ newBlt = null;
+
+ return newBlt;
+ }
+
+ /** */
+ private Collection<BaselineNode> baselineNodes() {
+ List<ClusterNode> clNodes = ctx.discovery().serverNodes(AffinityTopologyVersion.NONE);
+
+ ArrayList<BaselineNode> bltNodes = new ArrayList<>(clNodes.size());
+
+ for (ClusterNode clNode : clNodes)
+ bltNodes.add(clNode);
+
+ return bltNodes;
+ }
+
+ /** */
+ private IgniteInternalFuture<?> changeGlobalState0(final boolean activate,
+ BaselineTopology blt, boolean forceChangeBaselineTopology) {
if (ctx.isDaemon() || ctx.clientNode()) {
GridFutureAdapter<Void> fut = new GridFutureAdapter<>();
- sendComputeChangeGlobalState(activate, fut);
+ sendComputeChangeGlobalState(activate, blt, forceChangeBaselineTopology, fut);
return fut;
}
if (cacheProc.transactions().tx() != null || sharedCtx.lockedTopologyVersion(null) != null) {
return new GridFinishedFuture<>(new IgniteCheckedException("Failed to " + prettyStr(activate) +
- " cluster (must invoke the method outside of an active transaction or lock)."));
+ " cluster (must invoke the method outside of an active transaction)."));
}
DiscoveryDataClusterState curState = globalState;
- if (!curState.transition() && curState.active() == activate)
+ if (!curState.transition() && curState.active() == activate && BaselineTopology.equals(curState.baselineTopology(), blt))
return new GridFinishedFuture<>();
GridChangeGlobalStateFuture startedFut = null;
@@ -473,9 +848,15 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
ChangeGlobalStateMessage msg = new ChangeGlobalStateMessage(startedFut.requestId,
ctx.localNodeId(),
storedCfgs,
- activate);
+ activate,
+ blt,
+ forceChangeBaselineTopology,
+ System.currentTimeMillis());
try {
+ if (log.isInfoEnabled())
+ U.log(log, "Sending " + prettyStr(activate) + " request with BaselineTopology " + blt);
+
ctx.discovery().sendCustomEvent(msg);
if (ctx.isStopping())
@@ -488,14 +869,115 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
startedFut.onDone(e);
}
- return startedFut;
+ return wrapStateChangeFuture(startedFut, msg);
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag.JoiningNodeDiscoveryData discoData) {
+ if (node.isClient() || node.isDaemon())
+ return null;
+
+ if (discoData.joiningNodeData() == null) {
+ if (globalState.baselineTopology() != null) {
+ String msg = "Node not supporting BaselineTopology" +
+ " is not allowed to join the cluster with BaselineTopology";
+
+ return new IgniteNodeValidationResult(node.id(), msg, msg);
+ }
+
+ return null;
+ }
+
+ DiscoveryDataClusterState joiningNodeState;
+
+ try {
+ joiningNodeState = marsh.unmarshal((byte[]) discoData.joiningNodeData(), Thread.currentThread().getContextClassLoader());
+ } catch (IgniteCheckedException e) {
+ String msg = "Error on unmarshalling discovery data " +
+ "from node " + node.consistentId() + ": " + e.getMessage() +
+ "; node is not allowed to join";
+
+ return new IgniteNodeValidationResult(node.id(), msg , msg);
+ }
+
+ if (joiningNodeState == null || joiningNodeState.baselineTopology() == null)
+ return null;
+
+ if (globalState == null || globalState.baselineTopology() == null) {
+ if (joiningNodeState != null && joiningNodeState.baselineTopology() != null) {
+ String msg = "Node with set up BaselineTopology is not allowed to join cluster without one: " + node.consistentId();
+
+ return new IgniteNodeValidationResult(node.id(), msg, msg);
+ }
+ }
+
+ BaselineTopology joiningNodeBlt = joiningNodeState.baselineTopology();
+ BaselineTopology clusterBlt = globalState.baselineTopology();
+
+ String recommendation = " Consider cleaning persistent storage of the node and adding it to the cluster again.";
+
+ if (joiningNodeBlt.id() > clusterBlt.id()) {
+ String msg = "BaselineTopology of joining node ("
+ + node.consistentId()
+ + ") is not compatible with BaselineTopology in the cluster."
+ + " Joining node BlT id (" + joiningNodeBlt.id()
+ + ") is greater than cluster BlT id (" + clusterBlt.id() + ")."
+ + " New BaselineTopology was set on joining node with set-baseline command."
+ + recommendation;
+
+ return new IgniteNodeValidationResult(node.id(), msg, msg);
+ }
+
+ if (joiningNodeBlt.id() == clusterBlt.id()) {
+ if (!clusterBlt.isCompatibleWith(joiningNodeBlt)) {
+ String msg = "BaselineTopology of joining node ("
+ + node.consistentId()
+ + " ) is not compatible with BaselineTopology in the cluster."
+ + " Branching history of cluster BlT (" + clusterBlt.branchingHistory()
+ + ") doesn't contain branching point hash of joining node BlT ("
+ + joiningNodeBlt.branchingPointHash()
+ + ")." + recommendation;
+
+ return new IgniteNodeValidationResult(node.id(), msg, msg);
+ }
+ }
+ else if (joiningNodeBlt.id() < clusterBlt.id()) {
+ if (!bltHist.isCompatibleWith(joiningNodeBlt)) {
+ String msg = "BaselineTopology of joining node ("
+ + node.consistentId()
+ + ") is not compatible with BaselineTopology in the cluster."
+ + " BlT id of joining node (" + joiningNodeBlt.id()
+ + ") less than BlT id of cluster (" + clusterBlt.id()
+ + ") but cluster's BaselineHistory doesn't contain branching point hash of joining node BlT ("
+ + joiningNodeBlt.branchingPointHash()
+ + ")." + recommendation;
+
+ return new IgniteNodeValidationResult(node.id(), msg, msg);
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * @param fut Original state change future.
+ * @param msg State change message.
+ * @return Wrapped state change future.
+ */
+ protected IgniteInternalFuture<?> wrapStateChangeFuture(IgniteInternalFuture fut, ChangeGlobalStateMessage msg) {
+ return fut;
}
/**
* @param activate New cluster state.
* @param resFut State change future.
*/
- private void sendComputeChangeGlobalState(boolean activate, final GridFutureAdapter<Void> resFut) {
+ private void sendComputeChangeGlobalState(
+ boolean activate,
+ BaselineTopology blt,
+ boolean forceBlt,
+ final GridFutureAdapter<Void> resFut
+ ) {
AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx();
if (log.isInfoEnabled()) {
@@ -507,7 +989,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
IgniteCompute comp = ((ClusterGroupAdapter)ctx.cluster().get().forServers()).compute();
- IgniteFuture<Void> fut = comp.runAsync(new ClientChangeGlobalStateComputeRequest(activate));
+ IgniteFuture<Void> fut = comp.runAsync(new ClientChangeGlobalStateComputeRequest(activate, blt, forceBlt));
fut.listen(new CI1<IgniteFuture>() {
@Override public void apply(IgniteFuture fut) {
@@ -549,32 +1031,31 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
});
}
- /**
- * @param errs Errors.
- * @param req State change request.
- */
- public void onStateChangeError(Map<UUID, Exception> errs, StateChangeRequest req) {
+ /** {@inheritDoc} */
+ @Override public void onStateChangeError(Map<UUID, Exception> errs, StateChangeRequest req) {
assert !F.isEmpty(errs);
// Revert caches start if activation request fail.
- if (req.activate()) {
- try {
- cacheProc.onKernalStopCaches(true);
+ if (req.activeChanged()) {
+ if (req.activate()) {
+ try {
+ cacheProc.onKernalStopCaches(true);
- cacheProc.stopCaches(true);
+ cacheProc.stopCaches(true);
- sharedCtx.affinity().removeAllCacheInfo();
+ sharedCtx.affinity().removeAllCacheInfo();
- if (!ctx.clientNode())
- sharedCtx.deactivate();
+ if (!ctx.clientNode())
+ sharedCtx.deactivate();
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to revert activation request changes", e);
+ }
}
- catch (Exception e) {
- U.error(log, "Failed to revert activation request changes", e);
+ else {
+ //todo https://issues.apache.org/jira/browse/IGNITE-5480
}
}
- else {
- //todo https://issues.apache.org/jira/browse/IGNITE-5480
- }
GridChangeGlobalStateFuture fut = changeStateFuture(req.initiatorNodeId(), req.requestId());
@@ -596,6 +1077,8 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
* @param req State change request.
*/
private void onFinalActivate(final StateChangeRequest req) {
+ ctx.dataStructures().onBeforeActivate();
+
ctx.closure().runLocalSafe(new Runnable() {
@Override public void run() {
boolean client = ctx.clientNode();
@@ -618,37 +1101,44 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
+ ctx.localNodeId() + ", client=" + client + ", topVer=" + req.topologyVersion() + "]");
}
catch (Exception ex) {
- e = new IgniteCheckedException("Failed to perform final activation steps", ex);
-
- U.error(log, "Failed to perform final activation steps [nodeId=" + ctx.localNodeId() +
- ", client=" + client + ", topVer=" + req.topologyVersion() + "]", ex);
- }
- finally {
- globalState.setTransitionResult(req.requestId(), true);
-
- sendChangeGlobalStateResponse(req.requestId(), req.initiatorNodeId(), e);
+ throw new IgniteException(ex);
}
}
});
}
- /**
- * @param req State change request.
- */
- private void onFinalDeActivate(final StateChangeRequest req) {
- globalState.setTransitionResult(req.requestId(), false);
+ /** {@inheritDoc} */
+ @Override public void onStateChangeExchangeDone(StateChangeRequest req) {
+ try {
+ if (req.activeChanged()) {
+ if (req.activate())
+ onFinalActivate(req);
+
+ globalState.setTransitionResult(req.requestId(), req.activate());
+ }
+
+ sendChangeGlobalStateResponse(req.requestId(), req.initiatorNodeId(), null);
+ }
+ catch (Exception ex) {
+ Exception e = new IgniteCheckedException("Failed to perform final activation steps", ex);
- sendChangeGlobalStateResponse(req.requestId(), req.initiatorNodeId(), null);
+ U.error(log, "Failed to perform final activation steps [nodeId=" + ctx.localNodeId() +
+ ", client=" + ctx.clientNode() + ", topVer=" + req.topologyVersion() + "]", ex);
+
+ sendChangeGlobalStateResponse(req.requestId(), req.initiatorNodeId(), e);
+ }
}
- /**
- * @param req State change request.
- */
- public void onStateChangeExchangeDone(StateChangeRequest req) {
- if (req.activate())
- onFinalActivate(req);
- else
- onFinalDeActivate(req);
+ /** {@inheritDoc} */
+ @Override public void onBaselineTopologyChanged(BaselineTopology blt, BaselineTopologyHistoryItem prevBltHistItem) throws IgniteCheckedException {
+ if (compatibilityMode) {
+ if (log.isDebugEnabled())
+ log.info("BaselineTopology won't be stored as this node is running in compatibility mode");
+
+ return;
+ }
+
+ writeBaselineTopology(blt, prevBltHistItem);
}
/**
@@ -718,6 +1208,27 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
}
}
+ /** */
+ private void onStateRestored(BaselineTopology blt) {
+ DiscoveryDataClusterState state = globalState;
+
+ if (!state.active() && !state.transition() && state.baselineTopology() == null) {
+ DiscoveryDataClusterState newState = DiscoveryDataClusterState.createState(false, blt);
+
+ globalState = newState;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onExchangeFinishedOnCoordinator(IgniteInternalFuture exchangeFuture, boolean hasMovingPartitions) {
+ // no-op
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean evictionsAllowed() {
+ return true;
+ }
+
/**
* @param activate Activate.
* @return Activate flag string.
@@ -734,7 +1245,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
/**
*
*/
- private static class GridChangeGlobalStateFuture extends GridFutureAdapter<Void> {
+ private class GridChangeGlobalStateFuture extends GridFutureAdapter<Void> {
/** Request id. */
@GridToStringInclude
private final UUID requestId;
@@ -868,7 +1379,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
/** {@inheritDoc} */
@Override public boolean onDone(@Nullable Void res, @Nullable Throwable err) {
if (super.onDone(res, err)) {
- ctx.state().stateChangeFut.compareAndSet(this, null);
+ stateChangeFut.compareAndSet(this, null);
return true;
}
@@ -892,20 +1403,37 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
/** */
private final boolean activate;
+ /** */
+ private final BaselineTopology baselineTopology;
+
+ /** */
+ private final boolean forceChangeBaselineTopology;
+
/** Ignite. */
@IgniteInstanceResource
- private Ignite ig;
+ private IgniteEx ig;
/**
* @param activate New cluster state.
*/
- private ClientChangeGlobalStateComputeRequest(boolean activate) {
+ private ClientChangeGlobalStateComputeRequest(boolean activate, BaselineTopology blt, boolean forceBlt) {
this.activate = activate;
+ this.baselineTopology = blt;
+ this.forceChangeBaselineTopology = forceBlt;
}
/** {@inheritDoc} */
@Override public void run() {
- ig.active(activate);
+ try {
+ ig.context().state().changeGlobalState(
+ activate,
+ baselineTopology != null ? baselineTopology.currentBaseline() : null,
+ forceChangeBaselineTopology
+ ).get();
+ }
+ catch (IgniteCheckedException ex) {
+ throw new IgniteException(ex);
+ }
}
}
@@ -961,4 +1489,22 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
return false;
}
}
+
+ /** */
+ private static class BaselineStateAndHistoryData implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final DiscoveryDataClusterState globalState;
+
+ /** */
+ private final BaselineTopologyHistory recentHistory;
+
+ /** */
+ BaselineStateAndHistoryData(DiscoveryDataClusterState globalState, BaselineTopologyHistory recentHistory) {
+ this.globalState = globalState;
+ this.recentHistory = recentHistory;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java
new file mode 100644
index 0000000..bc72a51
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cluster;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.processors.GridProcessor;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.StateChangeRequest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public interface IGridClusterStateProcessor extends GridProcessor {
+ /**
+ * @return Cluster state to be used on public API.
+ */
+ boolean publicApiActiveState(boolean waitForTransition);
+
+ /**
+ * @param discoCache Discovery data cache.
+ * @return If transition is in progress returns future which is completed when transition finishes.
+ */
+ @Nullable IgniteInternalFuture<Boolean> onLocalJoin(DiscoCache discoCache);
+
+ /**
+ * @param node Failed node.
+ * @return Message if cluster state changed.
+ */
+ @Nullable ChangeGlobalStateFinishMessage onNodeLeft(ClusterNode node);
+
+ /**
+ * @param msg Message.
+ */
+ void onStateFinishMessage(ChangeGlobalStateFinishMessage msg);
+
+ /**
+ * @param topVer Current topology version.
+ * @param msg Message.
+ * @param discoCache Current nodes.
+ * @return {@code True} if need start state change process.
+ */
+ boolean onStateChangeMessage(AffinityTopologyVersion topVer,
+ ChangeGlobalStateMessage msg,
+ DiscoCache discoCache);
+
+ /**
+ * @return Current cluster state, should be called only from discovery thread.
+ */
+ DiscoveryDataClusterState clusterState();
+
+ /**
+ * @return Pending cluster state which will be used when state transition is finished.
+ */
+ DiscoveryDataClusterState pendingState(ChangeGlobalStateMessage stateMsg);
+
+ /**
+ *
+ */
+ void cacheProcessorStarted();
+
+ /**
+ * @param activate New cluster state.
+ * @return State change future.
+ */
+ IgniteInternalFuture<?> changeGlobalState(
+ boolean activate,
+ Collection<? extends BaselineNode> baselineNodes,
+ boolean forceChangeBaselineTopology
+ );
+
+ /**
+ * @param errs Errors.
+ * @param req State change request.
+ */
+ void onStateChangeError(Map<UUID, Exception> errs, StateChangeRequest req);
+
+ /**
+ * @param req State change request.
+ */
+ void onStateChangeExchangeDone(StateChangeRequest req);
+
+ /**
+ * @param blt New baseline topology.
+ * @param prevBltHistItem Previous baseline history item.
+ */
+ void onBaselineTopologyChanged(BaselineTopology blt, BaselineTopologyHistoryItem prevBltHistItem) throws IgniteCheckedException;
+
+ /**
+ * @param exchangeFuture Exchange future.
+ * @param hasMovingPartitions {@code True} if there are moving partitions.
+ */
+ void onExchangeFinishedOnCoordinator(IgniteInternalFuture exchangeFuture, boolean hasMovingPartitions);
+
+ /**
+ * @return {@code True} if partition evictions are allowed in current state.
+ */
+ boolean evictionsAllowed();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/AtomicDataStructureProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/AtomicDataStructureProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/AtomicDataStructureProxy.java
index 8355210..2c336a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/AtomicDataStructureProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/AtomicDataStructureProxy.java
@@ -89,8 +89,8 @@ public abstract class AtomicDataStructureProxy<V extends AtomicDataStructureValu
* @return Datastructure name.
*/
public String name() {
- return name;
- }
+ return name;
+ }
/**
* @return Key value.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
index acd8c11..99b2fa8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/DataStructuresProcessor.java
@@ -17,7 +17,6 @@
package org.apache.ignite.internal.processors.datastructures;
-import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -74,7 +73,6 @@ import org.apache.ignite.internal.util.typedef.CX1;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.A;
-import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.GPR;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -105,10 +103,10 @@ import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_REA
*/
public final class DataStructuresProcessor extends GridProcessorAdapter implements IgniteChangeGlobalStateSupport {
/** */
- private static final String DEFAULT_DS_GROUP_NAME = "default-ds-group";
+ public static final String DEFAULT_VOLATILE_DS_GROUP_NAME = "default-volatile-ds-group";
/** */
- private static final String DEFAULT_VOLATILE_DS_GROUP_NAME = "default-volatile-ds-group";
+ private static final String DEFAULT_DS_GROUP_NAME = "default-ds-group";
/** */
private static final String DS_CACHE_NAME_PREFIX = "datastructures_";
@@ -172,13 +170,13 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
}
/** {@inheritDoc} */
- @Override public void start() throws IgniteCheckedException {
+ @Override public void start() {
ctx.event().addLocalEventListener(lsnr, EVT_NODE_LEFT, EVT_NODE_FAILED);
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public void onKernalStart(boolean active) throws IgniteCheckedException {
+ @Override public void onKernalStart(boolean active) {
if (ctx.config().isDaemon() || !active)
return;
@@ -188,7 +186,14 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
/**
*
*/
- private void onKernalStart0(){
+ public void onBeforeActivate() {
+ initLatch = new CountDownLatch(1);
+ }
+
+ /**
+ *
+ */
+ private void onKernalStart0() {
initLatch.countDown();
}
@@ -225,10 +230,14 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
((GridCacheLockEx)ds).onStop();
}
- if (initLatch.getCount() > 0) {
+ CountDownLatch init0 = initLatch;
+
+ if (init0 != null && init0.getCount() > 0) {
initFailed = true;
- initLatch.countDown();
+ init0.countDown();
+
+ initLatch = null;
}
Iterator<Map.Entry<Integer, UUID>> iter = qryIdMap.entrySet().iterator();
@@ -245,22 +254,20 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
}
/** {@inheritDoc} */
- @Override public void onActivate(GridKernalContext ctx) throws IgniteCheckedException {
+ @Override public void onActivate(GridKernalContext ctx) {
if (log.isDebugEnabled())
- log.debug("Activate data structure processor [nodeId=" + ctx.localNodeId() +
+ log.debug("Activating data structure processor [nodeId=" + ctx.localNodeId() +
" topVer=" + ctx.discovery().topologyVersionEx() + " ]");
initFailed = false;
- initLatch = new CountDownLatch(1);
-
qryIdMap.clear();
ctx.event().addLocalEventListener(lsnr, EVT_NODE_LEFT, EVT_NODE_FAILED);
- onKernalStart0();
-
restoreStructuresState(ctx);
+
+ onKernalStart0();
}
/**
@@ -284,12 +291,14 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
@Override public void onDeActivate(GridKernalContext ctx) {
if (log.isDebugEnabled())
log.debug("DeActivate data structure processor [nodeId=" + ctx.localNodeId() +
- " topVer=" + ctx.discovery().topologyVersionEx() + " ]");
+ ", topVer=" + ctx.discovery().topologyVersionEx() + "]");
ctx.event().removeLocalEventListener(lsnr, EVT_NODE_LEFT, EVT_NODE_FAILED);
onKernalStop(false);
+ initLatch = null;
+
for (GridCacheRemovable v : dsMap.values()) {
if (v instanceof IgniteChangeGlobalStateSupport)
((IgniteChangeGlobalStateSupport)v).onDeActivate(ctx);
@@ -1070,9 +1079,14 @@ public final class DataStructuresProcessor extends GridProcessorAdapter implemen
* Awaits for processor initialization.
*/
private void awaitInitialization() {
- if (initLatch.getCount() > 0) {
+ CountDownLatch latch0 = initLatch;
+
+ if (latch0 == null)
+ throw new IllegalStateException("Ignite cluster is not active");
+
+ if (latch0.getCount() > 0) {
try {
- U.await(initLatch);
+ U.await(latch0);
if (initFailed)
throw new IllegalStateException("Failed to initialize data structures processor.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index f53d4e2..1fd14f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -2071,7 +2071,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
if (qry.isLocal())
throw new IgniteException("Local query is not supported without specific cache.");
- if (!ctx.state().publicApiActiveState()) {
+ if (!ctx.state().publicApiActiveState(true)) {
throw new IgniteException("Can not perform the operation because the cluster is inactive. Note, that " +
"the cluster is considered inactive by default if Ignite Persistent Store is used to let all the nodes " +
"join the cluster. To activate the cluster call Ignite.active(true).");
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
index 6236026..7bb13d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
@@ -64,12 +64,12 @@ public class GridChangeStateCommandHandler extends GridRestCommandHandlerAdapter
try {
if (req.command().equals(CLUSTER_CURRENT_STATE)) {
- Boolean currentState = ctx.state().publicApiActiveState();
+ Boolean currentState = ctx.state().publicApiActiveState(false);
res.setResponse(currentState);
}
else
- ctx.state().changeGlobalState(req.active()).get();
+ ctx.grid().active(req.active());
fut.onDone(res);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6df107b..f2b2781 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -238,9 +238,7 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
if (ctx.deploy().enabled())
ctx.cache().context().deploy().ignoreOwnership(true);
- if (!ctx.clientNode()) {
- assert serviceCache.context().affinityNode();
-
+ if (!ctx.clientNode() && serviceCache.context().affinityNode()) {
serviceCache.context().continuousQueries().executeInternalQuery(
new ServiceEntriesListener(), null, true, true, false
);
@@ -1790,8 +1788,7 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
try {
svcName.set(dep.configuration().getName());
- ctx.cache().internalCache(UTILITY_CACHE_NAME).context().affinity().
- affinityReadyFuture(topVer).get();
+ ctx.cache().context().exchange().affinityReadyFuture(topVer).get();
reassign(dep, topVer);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java
new file mode 100644
index 0000000..6c0942a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.subscription;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Processor enables grid components to register listeners for events
+ * generated by other components on local node.
+ *
+ * It starts very first during node startup procedure so any components could use it.
+ *
+ * As only local events are supported, no network communication occurs during notification process.
+ */
+public class GridInternalSubscriptionProcessor extends GridProcessorAdapter {
+ /** */
+ private List<MetastorageLifecycleListener> metastorageListeners = new ArrayList<>();
+
+ /**
+ * @param ctx Kernal context.
+ */
+ public GridInternalSubscriptionProcessor(GridKernalContext ctx) {
+ super(ctx);
+ }
+
+ /** */
+ public void registerMetastorageListener(@NotNull MetastorageLifecycleListener metastorageListener) {
+ if (metastorageListener == null)
+ throw new NullPointerException("Metastorage subscriber should be not-null.");
+
+ metastorageListeners.add(metastorageListener);
+ }
+
+ /** */
+ public List<MetastorageLifecycleListener> getMetastorageSubscribers() {
+ return metastorageListeners;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredClosableIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredClosableIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredClosableIterator.java
new file mode 100644
index 0000000..0ca795c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredClosableIterator.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.lang;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ *
+ */
+public abstract class GridFilteredClosableIterator<T> extends GridFilteredIterator<T> implements AutoCloseable {
+ /**
+ * @param it Closeable iterator.
+ */
+ protected GridFilteredClosableIterator(GridCloseableIterator<? extends T> it) {
+ super(it);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ ((GridCloseableIterator)it).close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredIterator.java
index 8f074d3..452bdd3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFilteredIterator.java
@@ -25,7 +25,7 @@ import java.util.NoSuchElementException;
*/
public abstract class GridFilteredIterator<T> implements Iterator<T> {
/** */
- private final Iterator<? extends T> it;
+ protected final Iterator<? extends T> it;
/** */
private boolean hasNext;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
index 24b9d48..1c5a69a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
@@ -168,6 +168,13 @@ public class GridFunc {
/** */
private static final IgniteClosure<ClusterNode, UUID> NODE2ID = new ClusterNodeGetIdClosure();
+ /** */
+ private static final IgniteClosure<ClusterNode, Object> NODE2CONSISTENTID = new IgniteClosure<ClusterNode, Object>() {
+ @Override public Object apply(ClusterNode node) {
+ return node.consistentId();
+ }
+ };
+
/**
* Gets predicate that evaluates to {@code true} only for given local node ID.
*
@@ -324,6 +331,23 @@ public class GridFunc {
}
/**
+ * Convenient utility method that returns collection of node consistent IDs for a given
+ * collection of grid nodes.
+ * <p>
+ * Note that this method doesn't create a new collection but simply iterates
+ * over the input one.
+ *
+ * @param nodes Collection of grid nodes.
+ * @return Collection of node consistent IDs for given collection of grid nodes.
+ */
+ public static Collection<Object> nodeConsistentIds(@Nullable Collection<? extends ClusterNode> nodes) {
+ if (nodes == null || nodes.isEmpty())
+ return Collections.emptyList();
+
+ return F.viewReadOnly(nodes, NODE2CONSISTENTID);
+ }
+
+ /**
* Gets random value from given collection.
*
* @param c Input collection (no {@code null} and not emtpy).
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineNode.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineNode.java
new file mode 100644
index 0000000..b37a364
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineNode.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Map;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Data transfer object for {@link BaselineNode}.
+ */
+public class VisorBaselineNode extends VisorDataTransferObject {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private String consistentId;
+
+ /** */
+ private Map<String, Object> attrs;
+
+ /**
+ * Default constructor.
+ */
+ public VisorBaselineNode() {
+ // No-op.
+ }
+
+ /**
+ * Create data transfer object for baseline node.
+ *
+ * @param node Baseline node.
+ */
+ public VisorBaselineNode(BaselineNode node) {
+ consistentId = String.valueOf(node.consistentId());
+ attrs = node.attributes();
+ }
+
+ /**
+ * @return Node consistent ID.
+ */
+ public String getConsistentId() {
+ return consistentId;
+ }
+
+ /**
+ * @return Node attributes.
+ */
+ public Map<String, Object> getAttributes() {
+ return attrs;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+ U.writeString(out, consistentId);
+ U.writeMap(out, attrs);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+ consistentId = U.readString(in);
+ attrs = U.readMap(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorBaselineNode.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineOperation.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineOperation.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineOperation.java
new file mode 100644
index 0000000..b422417
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineOperation.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Baseline operation types.
+ **/
+public enum VisorBaselineOperation {
+ /** Collect baseline information. */
+ COLLECT,
+ /** Add nodes to baseline. */
+ ADD,
+ /** Remove nodes from baseline. */
+ REMOVE,
+ /** Set new baseline. */
+ SET,
+ /** Set baseline for topology version. */
+ VERSION;
+
+ /** Enumerated values. */
+ private static final VisorBaselineOperation[] VALS = values();
+
+ /**
+ * Efficiently gets enumerated value from its ordinal.
+ *
+ * @param ord Ordinal value.
+ * @return Enumerated value or {@code null} if ordinal out of range.
+ */
+ @Nullable public static VisorBaselineOperation fromOrdinal(int ord) {
+ return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
new file mode 100644
index 0000000..56c2dd9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task that will collect baseline topology information.
+ */
+@GridInternal
+public class VisorBaselineTask extends VisorOneNodeTask<VisorBaselineTaskArg, VisorBaselineTaskResult> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override protected VisorBaselineJob job(VisorBaselineTaskArg arg) {
+ return new VisorBaselineJob(arg, debug);
+ }
+
+ /**
+ * Job that will collect baseline topology information.
+ */
+ private static class VisorBaselineJob extends VisorJob<VisorBaselineTaskArg, VisorBaselineTaskResult> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * @param arg Formal job argument.
+ * @param debug Debug flag.
+ */
+ private VisorBaselineJob(VisorBaselineTaskArg arg, boolean debug) {
+ super(arg, debug);
+ }
+
+ /**
+ * Collect baseline and server nodes.
+ *
+ * @return Baseline descriptor.
+ */
+ private VisorBaselineTaskResult collect() {
+ IgniteClusterEx cluster = ignite.cluster();
+
+ Collection<? extends BaselineNode> baseline = cluster.currentBaselineTopology();
+
+ Collection<? extends BaselineNode> servers = cluster.forServers().nodes();
+
+ return new VisorBaselineTaskResult(ignite.active(), cluster.topologyVersion(),
+ F.isEmpty(baseline) ? null : baseline, servers);
+ }
+
+ /**
+ * Set new baseline.
+ *
+ * @param baselineTop Collection of baseline node.
+ * @return Baseline descriptor.
+ */
+ private VisorBaselineTaskResult set0(Collection<BaselineNode> baselineTop) {
+ ignite.cluster().setBaselineTopology(baselineTop);
+
+ return collect();
+ }
+
+ /**
+ * @return Current baseline.
+ */
+ private Map<String, BaselineNode> currentBaseLine() {
+ Collection<BaselineNode> baseline = ignite.cluster().currentBaselineTopology();
+
+ Map<String, BaselineNode> nodes = new HashMap<>();
+
+ for (BaselineNode node : baseline)
+ nodes.put(node.consistentId().toString(), node);
+
+ return nodes;
+ }
+
+ /**
+ * @return Current server nodes.
+ */
+ private Map<String, BaselineNode> currentServers() {
+ Map<String, BaselineNode> nodes = new HashMap<>();
+
+ for (ClusterNode node : ignite.cluster().forServers().nodes())
+ nodes.put(node.consistentId().toString(), node);
+
+ return nodes;
+ }
+
+ /**
+ * Set new baseline.
+ *
+ * @param consistentIds Collection of consistent IDs to set.
+ * @return New baseline.
+ */
+ private VisorBaselineTaskResult set(List<String> consistentIds) {
+ Map<String, BaselineNode> servers = currentServers();
+
+ Collection<BaselineNode> baselineTop = new ArrayList<>();
+
+ for (String consistentId : consistentIds) {
+ BaselineNode node = servers.get(consistentId);
+
+ if (node == null)
+ throw new IllegalStateException("Node not found for consistent ID: " + consistentId);
+
+ baselineTop.add(node);
+ }
+
+ return set0(baselineTop);
+ }
+
+ /**
+ * Add new nodes to baseline.
+ *
+ * @param consistentIds Collection of consistent IDs to add.
+ * @return New baseline.
+ */
+ private VisorBaselineTaskResult add(List<String> consistentIds) {
+ Map<String, BaselineNode> baseline = currentBaseLine();
+ Map<String, BaselineNode> servers = currentServers();
+
+ for (String consistentId : consistentIds) {
+ BaselineNode node = servers.get(consistentId);
+
+ if (node == null)
+ throw new IllegalStateException("Node not found for consistent ID: " + consistentId);
+
+ baseline.put(consistentId, node);
+ }
+
+ return set0(baseline.values());
+ }
+
+ /**
+ * Remove nodes from baseline.
+ *
+ * @param consistentIds Collection of consistent IDs to remove.
+ * @return New baseline.
+ */
+ private VisorBaselineTaskResult remove(List<String> consistentIds) {
+ Map<String, BaselineNode> baseline = currentBaseLine();
+
+ for (String consistentId : consistentIds) {
+ BaselineNode node = baseline.remove(consistentId);
+
+ if (node == null)
+ throw new IllegalStateException("Node not found for consistent ID: " + consistentId);
+ }
+
+ return set0(baseline.values());
+ }
+
+ /**
+ * Set baseline by topology version.
+ *
+ * @param targetVer Target topology version.
+ * @return New baseline.
+ */
+ private VisorBaselineTaskResult version(long targetVer) {
+ IgniteClusterEx cluster = ignite.cluster();
+
+ if (targetVer > cluster.topologyVersion())
+ throw new IllegalArgumentException("Topology version is ahead of time: " + targetVer);
+
+ cluster.setBaselineTopology(targetVer);
+
+ return collect();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected VisorBaselineTaskResult run(@Nullable VisorBaselineTaskArg arg) throws IgniteException {
+ switch (arg.getOperation()) {
+ case ADD:
+ return add(arg.getConsistentIds());
+
+ case REMOVE:
+ return remove(arg.getConsistentIds());
+
+ case SET:
+ return set(arg.getConsistentIds());
+
+ case VERSION:
+ return version(arg.getTopologyVersion());
+
+ default:
+ return collect();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorBaselineJob.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskArg.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskArg.java
new file mode 100644
index 0000000..092161e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskArg.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Argument for {@link VisorBaselineTask}.
+ */
+public class VisorBaselineTaskArg extends VisorDataTransferObject {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private VisorBaselineOperation op;
+
+ /** */
+ private long topVer;
+
+ /** */
+ private List<String> consistentIds;
+
+ /**
+ * Default constructor.
+ */
+ public VisorBaselineTaskArg() {
+ // No-op.
+ }
+
+ /**
+ * @param topVer Topology version.
+ */
+ public VisorBaselineTaskArg(VisorBaselineOperation op, long topVer, List<String> consistentIds) {
+ this.op = op;
+ this.topVer = topVer;
+ this.consistentIds = consistentIds;
+ }
+
+ /**
+ * @return Base line operation.
+ */
+ public VisorBaselineOperation getOperation() {
+ return op;
+ }
+
+ /**
+ * @return Topology version.
+ */
+ public long getTopologyVersion() {
+ return topVer;
+ }
+
+ /**
+ * @return Consistent IDs.
+ */
+ public List<String> getConsistentIds() {
+ return consistentIds;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+ U.writeEnum(out, op);
+ out.writeLong(topVer);
+ U.writeCollection(out, consistentIds);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+ op = VisorBaselineOperation.fromOrdinal(in.readByte());
+ topVer = in.readLong();
+ consistentIds = U.readList(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorBaselineTaskArg.class, this);
+ }
+}
[02/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java
new file mode 100644
index 0000000..80daff2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
+import org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.evict.NoOpPageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class CacheFreeListImplSelfTest extends GridCommonAbstractTest {
+ /** */
+ private static final int CPUS = Runtime.getRuntime().availableProcessors();
+
+ /** */
+ private static final long MB = 1024L * 1024L;
+
+ /** */
+ private PageMemory pageMem;
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ if (pageMem != null)
+ pageMem.stop();
+
+ pageMem = null;
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteSingleThreaded_1024() throws Exception {
+ checkInsertDeleteSingleThreaded(1024);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteSingleThreaded_2048() throws Exception {
+ checkInsertDeleteSingleThreaded(2048);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteSingleThreaded_4096() throws Exception {
+ checkInsertDeleteSingleThreaded(4096);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteSingleThreaded_8192() throws Exception {
+ checkInsertDeleteSingleThreaded(8192);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteSingleThreaded_16384() throws Exception {
+ checkInsertDeleteSingleThreaded(16384);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteMultiThreaded_1024() throws Exception {
+ checkInsertDeleteMultiThreaded(1024);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteMultiThreaded_2048() throws Exception {
+ checkInsertDeleteMultiThreaded(2048);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteMultiThreaded_4096() throws Exception {
+ checkInsertDeleteMultiThreaded(4096);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteMultiThreaded_8192() throws Exception {
+ checkInsertDeleteMultiThreaded(8192);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testInsertDeleteMultiThreaded_16384() throws Exception {
+ checkInsertDeleteMultiThreaded(16384);
+ }
+
+ /**
+ * @param pageSize Page size.
+ * @throws Exception If failed.
+ */
+ protected void checkInsertDeleteMultiThreaded(final int pageSize) throws Exception {
+ final FreeList list = createFreeList(pageSize);
+
+ Random rnd = new Random();
+
+ final ConcurrentMap<Long, TestDataRow> stored = new ConcurrentHashMap<>();
+
+ for (int i = 0; i < 100; i++) {
+ int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
+ int valSize = rnd.nextInt(pageSize * 5 / 2) + 10;
+
+ TestDataRow row = new TestDataRow(keySize, valSize);
+
+ list.insertDataRow(row);
+
+ assertTrue(row.link() != 0L);
+
+ TestDataRow old = stored.put(row.link(), row);
+
+ assertNull(old);
+ }
+
+ final AtomicBoolean grow = new AtomicBoolean(true);
+
+ GridTestUtils.runMultiThreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ Random rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 200_000; i++) {
+ boolean grow0 = grow.get();
+
+ if (grow0) {
+ if (stored.size() > 20_000) {
+ if (grow.compareAndSet(true, false))
+ info("Shrink... [" + stored.size() + ']');
+
+ grow0 = false;
+ }
+ }
+ else {
+ if (stored.size() < 1_000) {
+ if (grow.compareAndSet(false, true))
+ info("Grow... [" + stored.size() + ']');
+
+ grow0 = true;
+ }
+ }
+
+ boolean insert = rnd.nextInt(100) < 70 == grow0;
+
+ if (insert) {
+ int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
+ int valSize = rnd.nextInt(pageSize * 3 / 2) + 10;
+
+ TestDataRow row = new TestDataRow(keySize, valSize);
+
+ list.insertDataRow(row);
+
+ assertTrue(row.link() != 0L);
+
+ TestDataRow old = stored.put(row.link(), row);
+
+ assertNull(old);
+ }
+ else {
+ while (true) {
+ Iterator<TestDataRow> it = stored.values().iterator();
+
+ if (it.hasNext()) {
+ TestDataRow row = it.next();
+
+ TestDataRow rmvd = stored.remove(row.link);
+
+ if (rmvd != null) {
+ list.removeDataRowByLink(row.link);
+
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ return null;
+ }
+ }, 8, "runner");
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ protected void checkInsertDeleteSingleThreaded(int pageSize) throws Exception {
+ FreeList list = createFreeList(pageSize);
+
+ Random rnd = new Random();
+
+ Map<Long, TestDataRow> stored = new HashMap<>();
+
+ for (int i = 0; i < 100; i++) {
+ int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
+ int valSize = rnd.nextInt(pageSize * 5 / 2) + 10;
+
+ TestDataRow row = new TestDataRow(keySize, valSize);
+
+ list.insertDataRow(row);
+
+ assertTrue(row.link() != 0L);
+
+ TestDataRow old = stored.put(row.link(), row);
+
+ assertNull(old);
+ }
+
+ boolean grow = true;
+
+ for (int i = 0; i < 1_000_000; i++) {
+ if (grow) {
+ if (stored.size() > 20_000) {
+ grow = false;
+
+ info("Shrink... [" + stored.size() + ']');
+ }
+ }
+ else {
+ if (stored.size() < 1_000) {
+ grow = true;
+
+ info("Grow... [" + stored.size() + ']');
+ }
+ }
+
+ boolean insert = rnd.nextInt(100) < 70 == grow;
+
+ if (insert) {
+ int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
+ int valSize = rnd.nextInt(pageSize * 3 / 2) + 10;
+
+ TestDataRow row = new TestDataRow(keySize, valSize);
+
+ list.insertDataRow(row);
+
+ assertTrue(row.link() != 0L);
+
+ TestDataRow old = stored.put(row.link(), row);
+
+ assertNull(old);
+ }
+ else {
+ Iterator<TestDataRow> it = stored.values().iterator();
+
+ if (it.hasNext()) {
+ TestDataRow row = it.next();
+
+ TestDataRow rmvd = stored.remove(row.link);
+
+ assertTrue(rmvd == row);
+
+ list.removeDataRowByLink(row.link);
+ }
+ }
+ }
+ }
+
+ /**
+ * @return Page memory.
+ */
+ protected PageMemory createPageMemory(int pageSize, DataRegionConfiguration plcCfg) throws Exception {
+ PageMemory pageMem = new PageMemoryNoStoreImpl(log,
+ new UnsafeMemoryProvider(log),
+ null,
+ pageSize,
+ plcCfg,
+ new DataRegionMetricsImpl(plcCfg),
+ true);
+
+ pageMem.start();
+
+ return pageMem;
+ }
+
+ /**
+ * @param pageSize Page size.
+ * @return Free list.
+ * @throws Exception If failed.
+ */
+ protected FreeList createFreeList(int pageSize) throws Exception {
+ DataRegionConfiguration plcCfg = new DataRegionConfiguration()
+ .setInitialSize(1024 * MB)
+ .setMaxSize(1024 * MB);
+
+ pageMem = createPageMemory(pageSize, plcCfg);
+
+ long metaPageId = pageMem.allocatePage(1, 1, PageIdAllocator.FLAG_DATA);
+
+ DataRegionMetricsImpl regionMetrics = new DataRegionMetricsImpl(plcCfg);
+
+ DataRegion dataRegion = new DataRegion(pageMem, plcCfg, regionMetrics, new NoOpPageEvictionTracker());
+
+ return new CacheFreeListImpl(1, "freelist", regionMetrics, dataRegion, null, null, metaPageId, true);
+ }
+
+ /**
+ *
+ */
+ private static class TestDataRow implements CacheDataRow {
+ /** */
+ private long link;
+
+ /** */
+ private TestCacheObject key;
+
+ /** */
+ private TestCacheObject val;
+
+ /** */
+ private GridCacheVersion ver;
+
+ /**
+ * @param keySize Key size.
+ * @param valSize Value size.
+ */
+ private TestDataRow(int keySize, int valSize) {
+ key = new TestCacheObject(keySize);
+ val = new TestCacheObject(valSize);
+ ver = new GridCacheVersion(keySize, valSize, 1);
+ }
+
+ /** {@inheritDoc} */
+ @Override public KeyCacheObject key() {
+ return key;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void key(KeyCacheObject key) {
+ this.key = (TestCacheObject)key;
+ }
+
+ /** {@inheritDoc} */
+ @Override public CacheObject value() {
+ return val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridCacheVersion version() {
+ return ver;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long expireTime() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int partition() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long link() {
+ return link;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void link(long link) {
+ this.link = link;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hash() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int cacheId() {
+ return 0;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestCacheObject implements KeyCacheObject {
+ /** */
+ private byte[] data;
+
+ /**
+ * @param size Object size.
+ */
+ private TestCacheObject(int size) {
+ data = new byte[size];
+
+ Arrays.fill(data, (byte)size);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean internal() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int partition() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void partition(int part) {
+ assert false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public KeyCacheObject copy(int part) {
+ assert false;
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public <T> T value(CacheObjectValueContext ctx, boolean cpy) {
+ return (T)data;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte[] valueBytes(CacheObjectValueContext ctx) throws IgniteCheckedException {
+ return data;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int valueBytesLength(CacheObjectContext ctx) throws IgniteCheckedException {
+ return data.length;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean putValue(ByteBuffer buf) throws IgniteCheckedException {
+ buf.put(data);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int putValue(long addr) throws IgniteCheckedException {
+ PageUtils.putBytes(addr, 0, data);
+
+ return data.length;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean putValue(ByteBuffer buf, int off, int len) throws IgniteCheckedException {
+ buf.put(data, off, len);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte cacheObjectType() {
+ return 42;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isPlatformType() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public CacheObject prepareForCache(CacheObjectContext ctx) {
+ assert false;
+
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void finishUnmarshal(CacheObjectValueContext ctx, ClassLoader ldr)
+ throws IgniteCheckedException {
+ assert false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void prepareMarshal(CacheObjectValueContext ctx) throws IgniteCheckedException {
+ assert false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ assert false;
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ assert false;
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ assert false;
+
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ assert false;
+
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ assert false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java
deleted file mode 100644
index 72a1d81..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java
+++ /dev/null
@@ -1,565 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.database;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.CacheObjectContext;
-import org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
-import org.apache.ignite.internal.processors.cache.persistence.evict.NoOpPageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.plugin.extensions.communication.MessageReader;
-import org.apache.ignite.plugin.extensions.communication.MessageWriter;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.jetbrains.annotations.Nullable;
-
-/**
- *
- */
-public class FreeListImplSelfTest extends GridCommonAbstractTest {
- /** */
- private static final int CPUS = Runtime.getRuntime().availableProcessors();
-
- /** */
- private static final long MB = 1024L * 1024L;
-
- /** */
- private PageMemory pageMem;
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- if (pageMem != null)
- pageMem.stop();
-
- pageMem = null;
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testInsertDeleteSingleThreaded_1024() throws Exception {
- checkInsertDeleteSingleThreaded(1024);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testInsertDeleteSingleThreaded_2048() throws Exception {
- checkInsertDeleteSingleThreaded(2048);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testInsertDeleteSingleThreaded_4096() throws Exception {
- checkInsertDeleteSingleThreaded(4096);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testInsertDeleteSingleThreaded_8192() throws Exception {
- checkInsertDeleteSingleThreaded(8192);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testInsertDeleteSingleThreaded_16384() throws Exception {
- checkInsertDeleteSingleThreaded(16384);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testInsertDeleteMultiThreaded_1024() throws Exception {
- checkInsertDeleteMultiThreaded(1024);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testInsertDeleteMultiThreaded_2048() throws Exception {
- checkInsertDeleteMultiThreaded(2048);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testInsertDeleteMultiThreaded_4096() throws Exception {
- checkInsertDeleteMultiThreaded(4096);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testInsertDeleteMultiThreaded_8192() throws Exception {
- checkInsertDeleteMultiThreaded(8192);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testInsertDeleteMultiThreaded_16384() throws Exception {
- checkInsertDeleteMultiThreaded(16384);
- }
-
- /**
- * @param pageSize Page size.
- * @throws Exception If failed.
- */
- protected void checkInsertDeleteMultiThreaded(final int pageSize) throws Exception {
- final FreeList list = createFreeList(pageSize);
-
- Random rnd = new Random();
-
- final ConcurrentMap<Long, TestDataRow> stored = new ConcurrentHashMap<>();
-
- for (int i = 0; i < 100; i++) {
- int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
- int valSize = rnd.nextInt(pageSize * 5 / 2) + 10;
-
- TestDataRow row = new TestDataRow(keySize, valSize);
-
- list.insertDataRow(row);
-
- assertTrue(row.link() != 0L);
-
- TestDataRow old = stored.put(row.link(), row);
-
- assertNull(old);
- }
-
- final AtomicBoolean grow = new AtomicBoolean(true);
-
- GridTestUtils.runMultiThreaded(new Callable<Object>() {
- @Override public Object call() throws Exception {
- Random rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < 200_000; i++) {
- boolean grow0 = grow.get();
-
- if (grow0) {
- if (stored.size() > 20_000) {
- if (grow.compareAndSet(true, false))
- info("Shrink... [" + stored.size() + ']');
-
- grow0 = false;
- }
- }
- else {
- if (stored.size() < 1_000) {
- if (grow.compareAndSet(false, true))
- info("Grow... [" + stored.size() + ']');
-
- grow0 = true;
- }
- }
-
- boolean insert = rnd.nextInt(100) < 70 == grow0;
-
- if (insert) {
- int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
- int valSize = rnd.nextInt(pageSize * 3 / 2) + 10;
-
- TestDataRow row = new TestDataRow(keySize, valSize);
-
- list.insertDataRow(row);
-
- assertTrue(row.link() != 0L);
-
- TestDataRow old = stored.put(row.link(), row);
-
- assertNull(old);
- }
- else {
- while (true) {
- Iterator<TestDataRow> it = stored.values().iterator();
-
- if (it.hasNext()) {
- TestDataRow row = it.next();
-
- TestDataRow rmvd = stored.remove(row.link);
-
- if (rmvd != null) {
- list.removeDataRowByLink(row.link);
-
- break;
- }
- }
- }
- }
- }
-
- return null;
- }
- }, 8, "runner");
- }
-
- /**
- * @throws Exception if failed.
- */
- protected void checkInsertDeleteSingleThreaded(int pageSize) throws Exception {
- FreeList list = createFreeList(pageSize);
-
- Random rnd = new Random();
-
- Map<Long, TestDataRow> stored = new HashMap<>();
-
- for (int i = 0; i < 100; i++) {
- int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
- int valSize = rnd.nextInt(pageSize * 5 / 2) + 10;
-
- TestDataRow row = new TestDataRow(keySize, valSize);
-
- list.insertDataRow(row);
-
- assertTrue(row.link() != 0L);
-
- TestDataRow old = stored.put(row.link(), row);
-
- assertNull(old);
- }
-
- boolean grow = true;
-
- for (int i = 0; i < 1_000_000; i++) {
- if (grow) {
- if (stored.size() > 20_000) {
- grow = false;
-
- info("Shrink... [" + stored.size() + ']');
- }
- }
- else {
- if (stored.size() < 1_000) {
- grow = true;
-
- info("Grow... [" + stored.size() + ']');
- }
- }
-
- boolean insert = rnd.nextInt(100) < 70 == grow;
-
- if (insert) {
- int keySize = rnd.nextInt(pageSize * 3 / 2) + 10;
- int valSize = rnd.nextInt(pageSize * 3 / 2) + 10;
-
- TestDataRow row = new TestDataRow(keySize, valSize);
-
- list.insertDataRow(row);
-
- assertTrue(row.link() != 0L);
-
- TestDataRow old = stored.put(row.link(), row);
-
- assertNull(old);
- }
- else {
- Iterator<TestDataRow> it = stored.values().iterator();
-
- if (it.hasNext()) {
- TestDataRow row = it.next();
-
- TestDataRow rmvd = stored.remove(row.link);
-
- assertTrue(rmvd == row);
-
- list.removeDataRowByLink(row.link);
- }
- }
- }
- }
-
- /**
- * @return Page memory.
- */
- protected PageMemory createPageMemory(int pageSize, DataRegionConfiguration plcCfg) throws Exception {
- PageMemory pageMem = new PageMemoryNoStoreImpl(log,
- new UnsafeMemoryProvider(log),
- null,
- pageSize,
- plcCfg,
- new DataRegionMetricsImpl(plcCfg),
- true);
-
- pageMem.start();
-
- return pageMem;
- }
-
- /**
- * @param pageSize Page size.
- * @return Free list.
- * @throws Exception If failed.
- */
- protected FreeList createFreeList(int pageSize) throws Exception {
- DataRegionConfiguration plcCfg = new DataRegionConfiguration()
- .setInitialSize(1024 * MB)
- .setMaxSize(1024 * MB);
-
- pageMem = createPageMemory(pageSize, plcCfg);
-
- long metaPageId = pageMem.allocatePage(1, 1, PageIdAllocator.FLAG_DATA);
-
- DataRegionMetricsImpl metrics = new DataRegionMetricsImpl(plcCfg);
-
- DataRegion memPlc = new DataRegion(pageMem, plcCfg, metrics, new NoOpPageEvictionTracker());
-
- return new FreeListImpl(1, "freelist", metrics, memPlc, null, null, metaPageId, true);
- }
-
- /**
- *
- */
- private static class TestDataRow implements CacheDataRow {
- /** */
- private long link;
-
- /** */
- private TestCacheObject key;
-
- /** */
- private TestCacheObject val;
-
- /** */
- private GridCacheVersion ver;
-
- /**
- * @param keySize Key size.
- * @param valSize Value size.
- */
- private TestDataRow(int keySize, int valSize) {
- key = new TestCacheObject(keySize);
- val = new TestCacheObject(valSize);
- ver = new GridCacheVersion(keySize, valSize, 1);
- }
-
- /** {@inheritDoc} */
- @Override public KeyCacheObject key() {
- return key;
- }
-
- /** {@inheritDoc} */
- @Override public void key(KeyCacheObject key) {
- this.key = (TestCacheObject)key;
- }
-
- /** {@inheritDoc} */
- @Override public CacheObject value() {
- return val;
- }
-
- /** {@inheritDoc} */
- @Override public GridCacheVersion version() {
- return ver;
- }
-
- /** {@inheritDoc} */
- @Override public long expireTime() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public int partition() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public long link() {
- return link;
- }
-
- /** {@inheritDoc} */
- @Override public void link(long link) {
- this.link = link;
- }
-
- /** {@inheritDoc} */
- @Override public int hash() {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override public int cacheId() {
- return 0;
- }
- }
-
- /**
- *
- */
- private static class TestCacheObject implements KeyCacheObject {
- /** */
- private byte[] data;
-
- /**
- * @param size Object size.
- */
- private TestCacheObject(int size) {
- data = new byte[size];
-
- Arrays.fill(data, (byte)size);
- }
-
- /** {@inheritDoc} */
- @Override public boolean internal() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public int partition() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public void partition(int part) {
- assert false;
- }
-
- /** {@inheritDoc} */
- @Override public KeyCacheObject copy(int part) {
- assert false;
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public <T> T value(CacheObjectValueContext ctx, boolean cpy) {
- return (T)data;
- }
-
- /** {@inheritDoc} */
- @Override public byte[] valueBytes(CacheObjectValueContext ctx) throws IgniteCheckedException {
- return data;
- }
-
- /** {@inheritDoc} */
- @Override public int valueBytesLength(CacheObjectContext ctx) throws IgniteCheckedException {
- return data.length;
- }
-
- /** {@inheritDoc} */
- @Override public boolean putValue(ByteBuffer buf) throws IgniteCheckedException {
- buf.put(data);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public int putValue(long addr) throws IgniteCheckedException {
- PageUtils.putBytes(addr, 0, data);
-
- return data.length;
- }
-
- /** {@inheritDoc} */
- @Override public boolean putValue(ByteBuffer buf, int off, int len) throws IgniteCheckedException {
- buf.put(data, off, len);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public byte cacheObjectType() {
- return 42;
- }
-
- /** {@inheritDoc} */
- @Override public boolean isPlatformType() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public CacheObject prepareForCache(CacheObjectContext ctx) {
- assert false;
-
- return this;
- }
-
- /** {@inheritDoc} */
- @Override public void finishUnmarshal(CacheObjectValueContext ctx, ClassLoader ldr)
- throws IgniteCheckedException {
- assert false;
- }
-
- /** {@inheritDoc} */
- @Override public void prepareMarshal(CacheObjectValueContext ctx) throws IgniteCheckedException {
- assert false;
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- assert false;
-
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- assert false;
-
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public short directType() {
- assert false;
-
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public byte fieldsCount() {
- assert false;
-
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public void onAckReceived() {
- assert false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java
new file mode 100644
index 0000000..c76b6e2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbPutGetWithCacheStoreTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.cache.Cache;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.store.CacheStoreAdapter;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.IgniteReflectionFactory;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+
+/**
+ *
+ */
+public class IgniteDbPutGetWithCacheStoreTest extends GridCommonAbstractTest {
+ /** */
+ private static Map<Object, Object> storeMap = new ConcurrentHashMap<>();
+
+ /** */
+ private static final String CACHE_NAME = "cache";
+
+ /** */
+ private CacheAtomicityMode atomicityMode;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ DataStorageConfiguration dbCfg = new DataStorageConfiguration();
+
+ dbCfg
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(512 * 1024 * 1024)
+ .setPersistenceEnabled(true))
+ .setWalMode(WALMode.LOG_ONLY);
+
+ cfg.setDataStorageConfiguration(dbCfg);
+
+ CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(CACHE_NAME)
+ .setCacheStoreFactory(new IgniteReflectionFactory<>(TestStore.class))
+ .setAtomicityMode(atomicityMode)
+ .setBackups(1)
+ .setWriteThrough(true)
+ .setReadThrough(true);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ storeMap.clear();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ storeMap.clear();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWriteThrough() throws Exception {
+ checkWriteThrough(ATOMIC);
+ checkWriteThrough(TRANSACTIONAL);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testReadThrough() throws Exception {
+ checkReadThrough(ATOMIC);
+ checkReadThrough(TRANSACTIONAL);
+ }
+
+ /**
+ * @param atomicityMode Atomicity mode.
+ * @throws Exception if failed.
+ */
+ private void checkWriteThrough(CacheAtomicityMode atomicityMode) throws Exception {
+ this.atomicityMode = atomicityMode;
+
+ IgniteEx ig = startGrid(0);
+
+ try {
+ ig.active(true);
+
+ for (int i = 0; i < 2000; i++)
+ ig.cache(CACHE_NAME).put(i, i);
+
+ assertEquals(2000, storeMap.size());
+
+ stopAllGrids();
+
+ storeMap.clear();
+
+ ig = startGrid(0);
+
+ ig.active(true);
+
+ for (int i = 0; i < 2000; i++)
+ assertEquals(i, ig.cache(CACHE_NAME).get(i));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param atomicityMode Atomicity mode.
+ * @throws Exception if failed.
+ */
+ private void checkReadThrough(CacheAtomicityMode atomicityMode) throws Exception {
+ this.atomicityMode = atomicityMode;
+
+ IgniteEx ig = startGrid(0);
+
+ try {
+ ig.active(true);
+
+ for (int i = 0; i < 2000; i++)
+ storeMap.put(i, i);
+
+ for (int i = 0; i < 2000; i++)
+ assertEquals(i, ig.cache(CACHE_NAME).get(i));
+
+ stopAllGrids();
+
+ storeMap.clear();
+
+ ig = startGrid(0);
+
+ ig.active(true);
+
+ for (int i = 0; i < 2000; i++)
+ assertEquals(i, ig.cache(CACHE_NAME).get(i));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ *
+ */
+ public static class TestStore extends CacheStoreAdapter<Object, Object> {
+ /** {@inheritDoc} */
+ @Override public Object load(Object key) throws CacheLoaderException {
+ return storeMap.get(key);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(Cache.Entry<?, ?> entry) throws CacheWriterException {
+ storeMap.put(entry.getKey(), entry.getValue());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void delete(Object key) throws CacheWriterException {
+ storeMap.remove(key);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
new file mode 100644
index 0000000..04bea54
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.processors.cache.persistence.IndexStorageImpl;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.RootPage;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IndexStorageSelfTest extends GridCommonAbstractTest {
+ /** Make sure page is small enough to trigger multiple pages in a linked list. */
+ private static final int PAGE_SIZE = 1024;
+
+ /** */
+ private static File allocationPath;
+
+ /** */
+ private static final char[] ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ".toCharArray();
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testMetaIndexAllocation() throws Exception {
+ metaAllocation();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void metaAllocation() throws Exception {
+ PageMemory mem = memory(true);
+
+ int[] cacheIds = new int[]{1, "partitioned".hashCode(), "replicated".hashCode()};
+
+ Map<Integer, Map<String, RootPage>> allocatedIdxs = new HashMap<>();
+
+ mem.start();
+
+ try {
+ final Map<Integer, IndexStorageImpl> storeMap = new HashMap<>();
+
+ for (int i = 0; i < 1_000; i++) {
+ int cacheId = cacheIds[i % cacheIds.length];
+
+ Map<String, RootPage> idxMap = allocatedIdxs.get(cacheId);
+
+ if (idxMap == null) {
+ idxMap = new HashMap<>();
+
+ allocatedIdxs.put(cacheId, idxMap);
+ }
+
+ String idxName;
+
+ do {
+ idxName = randomName();
+ } while (idxMap.containsKey(idxName));
+
+ IndexStorageImpl metaStore = storeMap.get(cacheId);
+
+ if (metaStore == null) {
+ metaStore = new IndexStorageImpl(mem, null, new AtomicLong(), cacheId,
+ PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX,
+ null, mem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX), true);
+
+ storeMap.put(cacheId, metaStore);
+ }
+
+ final RootPage rootPage = metaStore.getOrAllocateForTree(idxName);
+
+ assertTrue(rootPage.isAllocated());
+
+ idxMap.put(idxName, rootPage);
+ }
+
+ for (int cacheId : cacheIds) {
+ Map<String, RootPage> idxMap = allocatedIdxs.get(cacheId);
+
+ for (Map.Entry<String, RootPage> entry : idxMap.entrySet()) {
+ String idxName = entry.getKey();
+ FullPageId rootPageId = entry.getValue().pageId();
+
+ final RootPage rootPage = storeMap.get(cacheId).getOrAllocateForTree(idxName);
+
+ assertEquals("Invalid root page ID restored [cacheId=" + cacheId + ", idxName=" + idxName + ']',
+ rootPageId, rootPage.pageId());
+
+ assertFalse("Root page already allocated [cacheId=" + cacheId + ", idxName=" + idxName + ']',
+ rootPage.isAllocated());
+ }
+ }
+ }
+ finally {
+ mem.stop();
+ }
+ }
+
+ /**
+ * @return Random name.
+ */
+ private static String randomName() {
+ StringBuilder sb = new StringBuilder();
+
+ Random rnd = ThreadLocalRandom.current();
+
+ int size = rnd.nextInt(25) + 1;
+
+ for (int i = 0; i < size; i++)
+ sb.append(ALPHABET[rnd.nextInt(ALPHABET.length)]);
+
+ return sb.toString();
+ }
+
+ /**
+ * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
+ * new empty page memory.
+ * @return Page memory instance.
+ */
+ protected PageMemory memory(boolean clean) throws Exception {
+ DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
+
+ DataRegionConfiguration plcCfg = new DataRegionConfiguration()
+ .setMaxSize(30 * 1024 * 1024).setInitialSize(30 * 1024 * 1024);
+
+ return new PageMemoryNoStoreImpl(
+ log,
+ provider,
+ null,
+ PAGE_SIZE,
+ plcCfg,
+ new DataRegionMetricsImpl(plcCfg),
+ true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
deleted file mode 100644
index 880e37e..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.database;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
-import org.apache.ignite.internal.processors.cache.persistence.MetadataStorage;
-import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.RootPage;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class MetadataStorageSelfTest extends GridCommonAbstractTest {
- /** Make sure page is small enough to trigger multiple pages in a linked list. */
- private static final int PAGE_SIZE = 1024;
-
- /** */
- private static File allocationPath;
-
- /** */
- private static final char[] ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ".toCharArray();
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testMetaIndexAllocation() throws Exception {
- metaAllocation();
- }
-
- /**
- * @throws Exception If failed.
- */
- private void metaAllocation() throws Exception {
- PageMemory mem = memory(true);
-
- int[] cacheIds = new int[]{1, "partitioned".hashCode(), "replicated".hashCode()};
-
- Map<Integer, Map<String, RootPage>> allocatedIdxs = new HashMap<>();
-
- mem.start();
-
- try {
- final Map<Integer, MetadataStorage> storeMap = new HashMap<>();
-
- for (int i = 0; i < 1_000; i++) {
- int cacheId = cacheIds[i % cacheIds.length];
-
- Map<String, RootPage> idxMap = allocatedIdxs.get(cacheId);
-
- if (idxMap == null) {
- idxMap = new HashMap<>();
-
- allocatedIdxs.put(cacheId, idxMap);
- }
-
- String idxName;
-
- do {
- idxName = randomName();
- } while (idxMap.containsKey(idxName));
-
- MetadataStorage metaStore = storeMap.get(cacheId);
-
- if (metaStore == null) {
- metaStore = new MetadataStorage(mem, null, new AtomicLong(), cacheId,
- PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX,
- null, mem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX), true);
-
- storeMap.put(cacheId, metaStore);
- }
-
- final RootPage rootPage = metaStore.getOrAllocateForTree(idxName);
-
- assertTrue(rootPage.isAllocated());
-
- idxMap.put(idxName, rootPage);
- }
-
- for (int cacheId : cacheIds) {
- Map<String, RootPage> idxMap = allocatedIdxs.get(cacheId);
-
- for (Map.Entry<String, RootPage> entry : idxMap.entrySet()) {
- String idxName = entry.getKey();
- FullPageId rootPageId = entry.getValue().pageId();
-
- final RootPage rootPage = storeMap.get(cacheId).getOrAllocateForTree(idxName);
-
- assertEquals("Invalid root page ID restored [cacheId=" + cacheId + ", idxName=" + idxName + ']',
- rootPageId, rootPage.pageId());
-
- assertFalse("Root page already allocated [cacheId=" + cacheId + ", idxName=" + idxName + ']',
- rootPage.isAllocated());
- }
- }
- }
- finally {
- mem.stop();
- }
- }
-
- /**
- * @return Random name.
- */
- private static String randomName() {
- StringBuilder sb = new StringBuilder();
-
- Random rnd = ThreadLocalRandom.current();
-
- int size = rnd.nextInt(25) + 1;
-
- for (int i = 0; i < size; i++)
- sb.append(ALPHABET[rnd.nextInt(ALPHABET.length)]);
-
- return sb.toString();
- }
-
- /**
- * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
- * new empty page memory.
- * @return Page memory instance.
- */
- protected PageMemory memory(boolean clean) throws Exception {
- DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
-
- DataRegionConfiguration plcCfg = new DataRegionConfiguration()
- .setMaxSize(30 * 1024 * 1024).setInitialSize(30 * 1024 * 1024);
-
- return new PageMemoryNoStoreImpl(
- log,
- provider,
- null,
- PAGE_SIZE,
- plcCfg,
- new DataRegionMetricsImpl(plcCfg),
- true);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index 93cd911..ca05288 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -61,6 +61,7 @@ import org.apache.ignite.spi.IgnitePortProtocol;
import org.apache.ignite.spi.IgniteSpiContext;
import org.apache.ignite.spi.IgniteSpiException;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
@@ -541,6 +542,11 @@ public class GridSpiTestContext implements IgniteSpiContext {
}
/** {@inheritDoc} */
+ @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag discoData) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public Collection<SecuritySubject> authenticatedSubjects() {
return Collections.emptyList();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 53ca516..91a1144 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -1973,4 +1973,13 @@ public final class GridTestUtils {
public static void deleteDbFiles() throws Exception {
deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
}
+
+ /**
+ * @param node Node.
+ * @param topVer Ready exchange version to wait for before trying to merge exchanges.
+ */
+ public static void mergeExchangeWaitVersion(Ignite node, long topVer) {
+ ((IgniteEx)node).context().cache().context().exchange().mergeExchangesTestWaitVersion(
+ new AffinityTopologyVersion(topVer, 0));
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index f5e8eef..74a0de9 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -60,8 +60,8 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.affinity.GridAffinityFunctionContextImpl;
@@ -85,6 +85,9 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCach
import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.typedef.F;
@@ -106,7 +109,6 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.cache.CacheMode.LOCAL;
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheRebalanceMode.NONE;
import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
@@ -769,6 +771,25 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
}
/**
+ * Compares checksums between primary and backup partitions of specified caches.
+ * Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
+ * concurrently updated.
+ *
+ * @param ig Ignite instance.
+ * @param cacheNames Cache names (if null, all user caches will be verified).
+ * @throws IgniteCheckedException If checksum conflict has been found.
+ */
+ protected void verifyBackupPartitions(Ignite ig, Set<String> cacheNames) throws IgniteCheckedException {
+ Map<PartitionKey, List<PartitionHashRecord>> conflicts = ig.compute().execute(
+ new VerifyBackupPartitionsTask(), cacheNames);
+
+ if (!conflicts.isEmpty()) {
+ throw new IgniteCheckedException("Partition checksums are different for backups " +
+ "of the following partitions: " + conflicts.keySet());
+ }
+ }
+
+ /**
* @param top Topology.
* @param topVer Version to wait for.
* @throws Exception If failed.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
index 534b07f..075ab91 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
@@ -26,6 +26,7 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCluster;
import org.apache.ignite.IgniteCompute;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.BaselineNode;
import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
@@ -314,6 +315,31 @@ public class IgniteClusterProcessProxy implements IgniteClusterEx {
throw new UnsupportedOperationException("Operation is not supported yet.");
}
+ /** {@inheritDoc} */
+ @Override public boolean active() {
+ throw new UnsupportedOperationException("Operation is not supported yet.");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void active(boolean active) {
+ throw new UnsupportedOperationException("Operation is not supported yet.");
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Collection<BaselineNode> currentBaselineTopology() {
+ throw new UnsupportedOperationException("Operation is not supported yet.");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setBaselineTopology(Collection<? extends BaselineNode> baselineTop) {
+ throw new UnsupportedOperationException("Operation is not supported yet.");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setBaselineTopology(long topVer) {
+ throw new UnsupportedOperationException("Operation is not supported yet.");
+ }
+
/**
*
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index 2cc2047..1b4e2da 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -54,9 +54,9 @@ import org.apache.ignite.internal.processors.continuous.GridMessageListenSelfTes
import org.apache.ignite.internal.processors.database.BPlusTreeFakeReuseSelfTest;
import org.apache.ignite.internal.processors.database.BPlusTreeReuseSelfTest;
import org.apache.ignite.internal.processors.database.BPlusTreeSelfTest;
-import org.apache.ignite.internal.processors.database.FreeListImplSelfTest;
+import org.apache.ignite.internal.processors.database.CacheFreeListImplSelfTest;
+import org.apache.ignite.internal.processors.database.IndexStorageSelfTest;
import org.apache.ignite.internal.processors.database.DataRegionMetricsSelfTest;
-import org.apache.ignite.internal.processors.database.MetadataStorageSelfTest;
import org.apache.ignite.internal.processors.database.SwapPathConstructionSelfTest;
import org.apache.ignite.internal.processors.odbc.OdbcConfigurationValidationSelfTest;
import org.apache.ignite.internal.processors.odbc.OdbcEscapeSequenceSelfTest;
@@ -176,8 +176,8 @@ public class IgniteBasicTestSuite extends TestSuite {
suite.addTestSuite(BPlusTreeSelfTest.class);
suite.addTestSuite(BPlusTreeFakeReuseSelfTest.class);
suite.addTestSuite(BPlusTreeReuseSelfTest.class);
- suite.addTestSuite(MetadataStorageSelfTest.class);
- suite.addTestSuite(FreeListImplSelfTest.class);
+ suite.addTestSuite(IndexStorageSelfTest.class);
+ suite.addTestSuite(CacheFreeListImplSelfTest.class);
suite.addTestSuite(DataRegionMetricsSelfTest.class);
suite.addTestSuite(SwapPathConstructionSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
index 7357d7e..99dd828 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite.java
@@ -37,6 +37,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCa
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearRemoveFailureTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearRemoveFailureTest;
import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionDistributionTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteChangingBaselineDownCacheRemoveFailoverTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteChangingBaselineUpCacheRemoveFailoverTest;
import org.apache.ignite.testframework.GridTestUtils;
/**
@@ -74,6 +76,8 @@ public class IgniteCacheFailoverTestSuite extends TestSuite {
suite.addTestSuite(GridCacheDhtClientRemoveFailureTest.class);
suite.addTestSuite(GridCacheNearRemoveFailureTest.class);
suite.addTestSuite(GridCacheAtomicNearRemoveFailureTest.class);
+ suite.addTestSuite(IgniteChangingBaselineUpCacheRemoveFailoverTest.class);
+ suite.addTestSuite(IgniteChangingBaselineDownCacheRemoveFailoverTest.class);
suite.addTestSuite(IgniteCacheAtomicNodeJoinTest.class);
suite.addTestSuite(IgniteCacheTxNodeJoinTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite2.java
index 1150286..6668755 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFailoverTestSuite2.java
@@ -30,6 +30,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridCa
import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedFailoverSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedTxSalvageSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedFailoverSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteChangingBaselineDownCachePutAllFailoverTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteChangingBaselineUpCachePutAllFailoverTest;
/**
*
@@ -59,6 +61,8 @@ public class IgniteCacheFailoverTestSuite2 {
suite.addTestSuite(CachePutAllFailoverAtomicTest.class);
suite.addTestSuite(CachePutAllFailoverTxTest.class);
+ suite.addTestSuite(IgniteChangingBaselineDownCachePutAllFailoverTest.class);
+ suite.addTestSuite(IgniteChangingBaselineUpCachePutAllFailoverTest.class);
return suite;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index 3b071de..59425c8 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -28,7 +28,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePds
import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsEvictionTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreePageMemoryImplTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreeReuseListPageMemoryImplTest;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.MetadataStoragePageMemoryImplTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.IndexStoragePageMemoryImplTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplNoLoadTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteThrottleSmokeTest;
@@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRing
import org.apache.ignite.internal.processors.database.IgniteDbClientNearCachePutGetTest;
import org.apache.ignite.internal.processors.database.IgniteDbDynamicCacheSelfTest;
import org.apache.ignite.internal.processors.database.IgniteDbMultiNodePutGetTest;
+import org.apache.ignite.internal.processors.database.IgniteDbPutGetWithCacheStoreTest;
import org.apache.ignite.internal.processors.database.IgniteDbSingleNodePutGetTest;
import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeTinyPutGetTest;
@@ -53,7 +54,7 @@ public class IgnitePdsTestSuite extends TestSuite {
// Basic PageMemory tests.
suite.addTestSuite(PageMemoryImplNoLoadTest.class);
- suite.addTestSuite(MetadataStoragePageMemoryImplTest.class);
+ suite.addTestSuite(IndexStoragePageMemoryImplTest.class);
suite.addTestSuite(IgnitePdsEvictionTest.class);
suite.addTestSuite(PageMemoryImplTest.class);
@@ -75,6 +76,7 @@ public class IgnitePdsTestSuite extends TestSuite {
suite.addTestSuite(IgnitePdsSingleNodePutGetPersistenceTest.class);
suite.addTestSuite(IgnitePdsDynamicCacheTest.class);
suite.addTestSuite(IgnitePdsClientNearCachePutGetTest.class);
+ suite.addTestSuite(IgniteDbPutGetWithCacheStoreTest.class);
suite.addTestSuite(IgniteClusterActivateDeactivateTestWithPersistence.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 17833ac..8dc318a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -25,10 +25,14 @@ import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchange
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAllBaselineNodesOnlineFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOfflineBaselineNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionDuringPartitionClearTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
import org.apache.ignite.internal.processors.cache.persistence.db.filename.IgniteUidAsConsistentIdMigrationTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundSelfTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushDefaultSelfTest;
@@ -94,8 +98,15 @@ public class IgnitePdsTestSuite2 extends TestSuite {
suite.addTestSuite(IgniteWalSerializerVersionTest.class);
+ // BaselineTopology tests
+ suite.addTestSuite(IgniteAllBaselineNodesOnlineFullApiSelfTest.class);
+ suite.addTestSuite(IgniteOfflineBaselineNodeFullApiSelfTest.class);
+ suite.addTestSuite(IgniteOnlineNodeOutOfBaselineFullApiSelfTest.class);
+
suite.addTestSuite(WalCompactionTest.class);
+ suite.addTestSuite(IgniteCheckpointDirtyPagesForLowLoadTest.class);
+
return suite;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
index f016e39..6039ae3 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
@@ -19,6 +19,8 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheBaselineTopologyTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteBaselineAffinityTopologyActivationTest;
import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateCacheTest;
import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateDataStreamerTest;
import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateDataStructureTest;
@@ -70,6 +72,9 @@ public class IgniteStandByClusterSuite extends TestSuite {
suite.addTestSuite(IgniteChangeGlobalStateFailOverTest.class);
suite.addTestSuite(IgniteChangeGlobalStateServiceTest.class);
+ suite.addTestSuite(CacheBaselineTopologyTest.class);
+ suite.addTestSuite(IgniteBaselineAffinityTopologyActivationTest.class);
+
return suite;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index d1cf930..e524d04 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -40,6 +40,7 @@ import org.apache.ignite.spi.discovery.ClusterMetricsSnapshotSerializeSelfTest;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.thread.GridThreadPoolExecutorServiceSelfTest;
import org.apache.ignite.thread.IgniteThreadPoolSizeTest;
+import org.apache.ignite.util.GridCommandHandlerTest;
import org.apache.ignite.util.GridIntListSelfTest;
import org.apache.ignite.util.GridLongListSelfTest;
import org.apache.ignite.util.GridMessageCollectionTest;
@@ -111,6 +112,9 @@ public class IgniteUtilSelfTestSuite extends TestSuite {
//dbx
suite.addTestSuite(PageIdUtilsSelfTest.class);
+ // control.sh
+ suite.addTestSuite(GridCommandHandlerTest.class);
+
return suite;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
new file mode 100644
index 0000000..b119bfa
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.io.File;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.commandline.CommandHandler;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
+
+/**
+ * Command line handler test.
+ */
+public class GridCommandHandlerTest extends GridCommonAbstractTest {
+ /**
+ * @return Folder in work directory.
+ * @throws IgniteCheckedException If failed to resolve folder name.
+ */
+ protected File folder(String folder) throws IgniteCheckedException {
+ return U.resolveWorkDirectory(U.defaultWorkDirectory(), folder, false);
+ }
+
+ /**
+ * Delete all files created by database engine during test.
+ */
+ private void cleanupDbFiles() throws IgniteCheckedException {
+ deleteRecursively(folder("db"));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ cleanupDbFiles();
+
+ stopAllGrids();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanupDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+ DataStorageConfiguration memCfg = new DataStorageConfiguration().setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration().setMaxSize(100 * 1024 * 1024));
+
+ cfg.setDataStorageConfiguration(memCfg);
+
+ DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration();
+ dsCfg.setWalMode(WALMode.LOG_ONLY);
+ dsCfg.getDefaultDataRegionConfiguration().setPersistenceEnabled(true);
+
+ return cfg;
+ }
+
+ /**
+ * Test activation works via control.sh
+ *
+ * @throws Exception If failed.
+ */
+ public void testActivate() throws Exception {
+ Ignite ignite = startGrids(1);
+
+ assertFalse(ignite.active());
+
+ CommandHandler cmd = new CommandHandler();
+
+ assertEquals(EXIT_CODE_OK, cmd.execute("--activate"));
+
+ assertTrue(ignite.active());
+ }
+
+ /**
+ * Test deactivation works via control.sh
+ *
+ * @throws Exception If failed.
+ */
+ public void testDeactivate() throws Exception {
+ Ignite ignite = startGrids(1);
+
+ assertFalse(ignite.active());
+
+ ignite.active(true);
+
+ assertTrue(ignite.active());
+
+ CommandHandler cmd = new CommandHandler();
+
+ assertEquals(EXIT_CODE_OK, cmd.execute("--deactivate"));
+
+ assertFalse(ignite.active());
+ }
+
+ /**
+ * Test cluster active state works via control.sh
+ *
+ * @throws Exception If failed.
+ */
+ public void testState() throws Exception {
+ Ignite ignite = startGrids(1);
+
+ assertFalse(ignite.active());
+
+ CommandHandler cmd = new CommandHandler();
+
+ assertEquals(EXIT_CODE_OK, cmd.execute("--state"));
+
+ ignite.active(true);
+
+ assertEquals(EXIT_CODE_OK, cmd.execute("--state"));
+ }
+
+ /**
+ * Test baseline collect works via control.sh
+ *
+ * @throws Exception If failed.
+ */
+ public void testBaselineCollect() throws Exception {
+ Ignite ignite = startGrids(1);
+
+ assertFalse(ignite.active());
+
+ ignite.active(true);
+
+ CommandHandler cmd = new CommandHandler();
+
+ cmd.execute("--baseline");
+
+ assertEquals(1, ignite.cluster().currentBaselineTopology().size());
+ }
+
+ /**
+ * @param ignites Ignites.
+ * @return Local node consistent ID.
+ */
+ private String consistentIds(Ignite... ignites) {
+ String res = "";
+
+ for (Ignite ignite : ignites) {
+ String consistentId = ignite.cluster().localNode().consistentId().toString();
+
+ if (!F.isEmpty(res))
+ res += ", ";
+
+ res += consistentId;
+ }
+
+ return res;
+ }
+
+ /**
+ * Test baseline add items works via control.sh
+ *
+ * @throws Exception If failed.
+ */
+ public void testBaselineAdd() throws Exception {
+ Ignite ignite = startGrids(1);
+
+ assertFalse(ignite.active());
+
+ ignite.active(true);
+
+ CommandHandler cmd = new CommandHandler();
+
+ Ignite other = startGrid(2);
+
+ assertEquals(EXIT_CODE_OK, cmd.execute("--baseline", "add", consistentIds(other)));
+ assertEquals(EXIT_CODE_OK, cmd.execute("--baseline", "add", consistentIds(other)));
+
+ assertEquals(2, ignite.cluster().currentBaselineTopology().size());
+ }
+
+ /**
+ * Test baseline remove works via control.sh
+ *
+ * @throws Exception If failed.
+ */
+ public void testBaselineRemove() throws Exception {
+ Ignite ignite = startGrids(1);
+ Ignite other = startGrid("nodeToStop");
+
+ assertFalse(ignite.active());
+
+ ignite.active(true);
+
+ String offlineNodeConsId = consistentIds(other);
+
+ stopGrid("nodeToStop");
+
+ CommandHandler cmd = new CommandHandler();
+
+ assertEquals(EXIT_CODE_OK, cmd.execute("--baseline"));
+ assertEquals(EXIT_CODE_OK, cmd.execute("--baseline", "remove", offlineNodeConsId));
+
+ assertEquals(1, ignite.cluster().currentBaselineTopology().size());
+ }
+
+ /**
+ * Test baseline set works via control.sh
+ *
+ * @throws Exception If failed.
+ */
+ public void testBaselineSet() throws Exception {
+ Ignite ignite = startGrids(1);
+
+ assertFalse(ignite.active());
+
+ ignite.active(true);
+
+ Ignite other = startGrid(2);
+
+ CommandHandler cmd = new CommandHandler();
+
+ assertEquals(EXIT_CODE_OK, cmd.execute("--baseline", "set", consistentIds(ignite, other)));
+
+ assertEquals(2, ignite.cluster().currentBaselineTopology().size());
+
+ assertEquals(EXIT_CODE_UNEXPECTED_ERROR, cmd.execute("--baseline", "set", "invalidConsistentId"));
+ }
+
+ /**
+ * Test baseline set by topology version works via control.sh
+ *
+ * @throws Exception If failed.
+ */
+ public void testBaselineVersion() throws Exception {
+ Ignite ignite = startGrids(1);
+
+ assertFalse(ignite.active());
+
+ ignite.active(true);
+
+ CommandHandler cmd = new CommandHandler();
+
+ startGrid(2);
+
+ assertEquals(EXIT_CODE_OK, cmd.execute("--baseline"));
+
+ assertEquals(EXIT_CODE_OK, cmd.execute("--baseline", "version", String.valueOf(ignite.cluster().topologyVersion())));
+
+ assertEquals(2, ignite.cluster().currentBaselineTopology().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/dev-utils/pom.xml
----------------------------------------------------------------------
diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml
index a782638..e57b17d 100644
--- a/modules/dev-utils/pom.xml
+++ b/modules/dev-utils/pom.xml
@@ -58,6 +58,32 @@
<skip>true</skip>
</configuration>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4.1</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.ignite.development.utils.IgniteWalConverter</mainClass>
+ </manifest>
+ </archive>
+
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
[12/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
new file mode 100644
index 0000000..7daef3c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
@@ -0,0 +1,420 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Metadata storage.
+ */
+public class IndexStorageImpl implements IndexStorage {
+ /** Max index name length (bytes num) */
+ public static final int MAX_IDX_NAME_LEN = 255;
+
+ /** Reserved size for index name. Needed for backward compatibility. */
+ public static final int RESERVED_IDX_NAME_LEN = 768;
+
+ /** Bytes in byte. */
+ private static final int BYTE_LEN = 1;
+
+ /** Page memory. */
+ private final PageMemory pageMem;
+
+ /** Index tree. */
+ private final MetaTree metaTree;
+
+ /** Meta page reuse tree. */
+ private final ReuseList reuseList;
+
+ /** Cache group ID. */
+ private final int grpId;
+
+ /** */
+ private final int allocPartId;
+
+ /** */
+ private final byte allocSpace;
+
+ /**
+ * @param pageMem Page memory.
+ * @param wal Write ahead log manager.
+ */
+ public IndexStorageImpl(
+ final PageMemory pageMem,
+ final IgniteWriteAheadLogManager wal,
+ final AtomicLong globalRmvId,
+ final int grpId,
+ final int allocPartId,
+ final byte allocSpace,
+ final ReuseList reuseList,
+ final long rootPageId,
+ final boolean initNew
+ ) {
+ try {
+ this.pageMem = pageMem;
+ this.grpId = grpId;
+ this.allocPartId = allocPartId;
+ this.allocSpace = allocSpace;
+ this.reuseList = reuseList;
+
+ metaTree = new MetaTree(grpId, allocPartId, allocSpace, pageMem, wal, globalRmvId, rootPageId,
+ reuseList, MetaStoreInnerIO.VERSIONS, MetaStoreLeafIO.VERSIONS, initNew);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public RootPage getOrAllocateForTree(final String idxName) throws IgniteCheckedException {
+ final MetaTree tree = metaTree;
+
+ synchronized (this) {
+ byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
+
+ if (idxNameBytes.length > MAX_IDX_NAME_LEN)
+ throw new IllegalArgumentException("Too long encoded indexName [maxAllowed=" + MAX_IDX_NAME_LEN +
+ ", currentLength=" + idxNameBytes.length + ", name=" + idxName + "]");
+
+ final IndexItem row = tree.findOne(new IndexItem(idxNameBytes, 0));
+
+ if (row == null) {
+ long pageId = 0;
+
+ if (reuseList != null)
+ pageId = reuseList.takeRecycledPage();
+
+ pageId = pageId == 0 ? pageMem.allocatePage(grpId, allocPartId, allocSpace) : pageId;
+
+ tree.put(new IndexItem(idxNameBytes, pageId));
+
+ return new RootPage(new FullPageId(pageId, grpId), true);
+ }
+ else {
+ final FullPageId pageId = new FullPageId(row.pageId, grpId);
+
+ return new RootPage(pageId, false);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public RootPage dropRootPage(final String idxName)
+ throws IgniteCheckedException {
+ byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
+
+ final IndexItem row = metaTree.remove(new IndexItem(idxNameBytes, 0));
+
+ if (row != null) {
+ if (reuseList == null)
+ pageMem.freePage(grpId, row.pageId);
+ }
+
+ return row != null ? new RootPage(new FullPageId(row.pageId, grpId), false) : null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void destroy() throws IgniteCheckedException {
+ metaTree.destroy();
+ }
+
+ /**
+ *
+ */
+ private static class MetaTree extends BPlusTree<IndexItem, IndexItem> {
+ /** */
+ private final int allocPartId;
+
+ /** */
+ private final byte allocSpace;
+
+ /**
+ * @param pageMem Page memory.
+ * @param metaPageId Meta page ID.
+ * @param reuseList Reuse list.
+ * @param innerIos Inner IOs.
+ * @param leafIos Leaf IOs.
+ * @throws IgniteCheckedException If failed.
+ */
+ private MetaTree(
+ final int cacheId,
+ final int allocPartId,
+ final byte allocSpace,
+ final PageMemory pageMem,
+ final IgniteWriteAheadLogManager wal,
+ final AtomicLong globalRmvId,
+ final long metaPageId,
+ final ReuseList reuseList,
+ final IOVersions<? extends BPlusInnerIO<IndexItem>> innerIos,
+ final IOVersions<? extends BPlusLeafIO<IndexItem>> leafIos,
+ final boolean initNew
+ ) throws IgniteCheckedException {
+ super(treeName("meta", "Meta"), cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList, innerIos, leafIos);
+
+ this.allocPartId = allocPartId;
+ this.allocSpace = allocSpace;
+
+ initTree(initNew);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
+ return pageMem.allocatePage(groupId(), allocPartId, allocSpace);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int compare(final BPlusIO<IndexItem> io, final long pageAddr, final int idx,
+ final IndexItem row) throws IgniteCheckedException {
+ final int off = ((IndexIO)io).getOffset(pageAddr, idx);
+
+ int shift = 0;
+
+ // Compare index names.
+ final int len = PageUtils.getUnsignedByte(pageAddr, off + shift);
+
+ shift += BYTE_LEN;
+
+ for (int i = 0; i < len && i < row.idxName.length; i++) {
+ final int cmp = Byte.compare(PageUtils.getByte(pageAddr, off + i + shift), row.idxName[i]);
+
+ if (cmp != 0)
+ return cmp;
+ }
+
+ return Integer.compare(len, row.idxName.length);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IndexItem getRow(final BPlusIO<IndexItem> io, final long pageAddr,
+ final int idx, Object ignore) throws IgniteCheckedException {
+ return readRow(pageAddr, ((IndexIO)io).getOffset(pageAddr, idx));
+ }
+ }
+
+ /**
+ *
+ */
+ private static class IndexItem {
+ /** */
+ private byte[] idxName;
+
+ /** */
+ private long pageId;
+
+ /**
+ * @param idxName Index name.
+ * @param pageId Page ID.
+ */
+ private IndexItem(final byte[] idxName, final long pageId) {
+ this.idxName = idxName;
+ this.pageId = pageId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "I [idxName=" + new String(idxName) + ", pageId=" + U.hexLong(pageId) + ']';
+ }
+ }
+
+ /**
+ * Store row to buffer.
+ *
+ * @param pageAddr Page address.
+ * @param off Offset in buf.
+ * @param row Row to store.
+ */
+ private static void storeRow(
+ final long pageAddr,
+ int off,
+ final IndexItem row
+ ) {
+ // Index name length.
+ PageUtils.putUnsignedByte(pageAddr, off, row.idxName.length);
+ off++;
+
+ // Index name.
+ PageUtils.putBytes(pageAddr, off, row.idxName);
+ off += row.idxName.length;
+
+ // Page ID.
+ PageUtils.putLong(pageAddr, off, row.pageId);
+ }
+
+ /**
+ * Copy row data.
+ *
+ * @param dstPageAddr Destination page address.
+ * @param dstOff Destination buf offset.
+ * @param srcPageAddr Source page address.
+ * @param srcOff Src buf offset.
+ */
+ private static void storeRow(
+ final long dstPageAddr,
+ int dstOff,
+ final long srcPageAddr,
+ int srcOff
+ ) {
+ // Index name length.
+ final int len = PageUtils.getUnsignedByte(srcPageAddr, srcOff);
+ srcOff++;
+
+ PageUtils.putUnsignedByte(dstPageAddr, dstOff, len);
+ dstOff++;
+
+ PageHandler.copyMemory(srcPageAddr, srcOff, dstPageAddr, dstOff, len);
+ srcOff += len;
+ dstOff += len;
+
+ // Page ID.
+ PageUtils.putLong(dstPageAddr, dstOff, PageUtils.getLong(srcPageAddr, srcOff));
+ }
+
+ /**
+ * Read row from buffer.
+ *
+ * @param pageAddr Page address.
+ * @param off Offset.
+ * @return Read row.
+ */
+ private static IndexItem readRow(final long pageAddr, int off) {
+ // Index name length.
+ final int len = PageUtils.getUnsignedByte(pageAddr, off) & 0xFF;
+ off++;
+
+ // Index name.
+ final byte[] idxName = PageUtils.getBytes(pageAddr, off, len);
+ off += len;
+
+ // Page ID.
+ final long pageId = PageUtils.getLong(pageAddr, off);
+
+ return new IndexItem(idxName, pageId);
+ }
+
+ /**
+ *
+ */
+ private interface IndexIO {
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @return Offset in buffer according to {@code idx}.
+ */
+ int getOffset(long pageAddr, int idx);
+ }
+
+ /**
+ *
+ */
+ public static final class MetaStoreInnerIO extends BPlusInnerIO<IndexItem> implements IndexIO {
+ /** */
+ public static final IOVersions<MetaStoreInnerIO> VERSIONS = new IOVersions<>(
+ new MetaStoreInnerIO(1)
+ );
+
+ /**
+ * @param ver Version.
+ */
+ private MetaStoreInnerIO(final int ver) {
+ // name bytes and 1 byte for length, 8 bytes pageId
+ super(T_METASTORE_INNER, ver, false, RESERVED_IDX_NAME_LEN + 1 + 8);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void storeByOffset(long pageAddr, int off, IndexItem row) throws IgniteCheckedException {
+ storeRow(pageAddr, off, row);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void store(final long dstPageAddr, final int dstIdx, final BPlusIO<IndexItem> srcIo,
+ final long srcPageAddr,
+ final int srcIdx) throws IgniteCheckedException {
+ storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree, final long pageAddr,
+ final int idx) throws IgniteCheckedException {
+ return readRow(pageAddr, offset(idx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getOffset(long pageAddr, final int idx) {
+ return offset(idx);
+ }
+ }
+
+ /**
+ *
+ */
+ public static final class MetaStoreLeafIO extends BPlusLeafIO<IndexItem> implements IndexIO {
+ /** */
+ public static final IOVersions<MetaStoreLeafIO> VERSIONS = new IOVersions<>(
+ new MetaStoreLeafIO(1)
+ );
+
+ /**
+ * @param ver Version.
+ */
+ private MetaStoreLeafIO(final int ver) {
+ // 4 byte cache ID, UTF-16 symbols and 1 byte for length, 8 bytes pageId
+ super(T_METASTORE_LEAF, ver, RESERVED_IDX_NAME_LEN + 1 + 8);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void storeByOffset(long buf, int off, IndexItem row) throws IgniteCheckedException {
+ storeRow(buf, off, row);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void store(final long dstPageAddr,
+ final int dstIdx,
+ final BPlusIO<IndexItem> srcIo,
+ final long srcPageAddr,
+ final int srcIdx) throws IgniteCheckedException {
+ storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree,
+ final long pageAddr,
+ final int idx) throws IgniteCheckedException {
+ return readRow(pageAddr, offset(idx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getOffset(long pageAddr, final int idx) {
+ return offset(idx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java
deleted file mode 100644
index c09ce4e..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence;
-
-import org.apache.ignite.IgniteCheckedException;
-
-/**
- * Meta store.
- */
-public interface MetaStore {
- /**
- * Get or allocate initial page for an index.
- *
- * @param idxName Index name.
- * @return {@link RootPage} that keeps pageId, allocated flag that shows whether the page
- * was newly allocated, and rootId that is counter which increments each time new page allocated.
- * @throws IgniteCheckedException If failed.
- */
- public RootPage getOrAllocateForTree(String idxName) throws IgniteCheckedException;
-
- /**
- * Deallocate index page and remove from tree.
- *
- * @param idxName Index name.
- * @return Root ID or -1 if no page was removed.
- * @throws IgniteCheckedException If failed.
- */
- public RootPage dropRootPage(String idxName) throws IgniteCheckedException;
-
- /**
- * Destroy this meta store.
- *
- * @throws IgniteCheckedException If failed.
- */
- public void destroy() throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java
deleted file mode 100644
index e667807..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java
+++ /dev/null
@@ -1,420 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence;
-
-import java.nio.charset.StandardCharsets;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- * Metadata storage.
- */
-public class MetadataStorage implements MetaStore {
- /** Max index name length (bytes num) */
- public static final int MAX_IDX_NAME_LEN = 255;
-
- /** Reserved size for index name. Needed for backward compatibility. */
- public static final int RESERVED_IDX_NAME_LEN = 768;
-
- /** Bytes in byte. */
- private static final int BYTE_LEN = 1;
-
- /** Page memory. */
- private final PageMemory pageMem;
-
- /** Index tree. */
- private final MetaTree metaTree;
-
- /** Meta page reuse tree. */
- private final ReuseList reuseList;
-
- /** Cache group ID. */
- private final int grpId;
-
- /** */
- private final int allocPartId;
-
- /** */
- private final byte allocSpace;
-
- /**
- * @param pageMem Page memory.
- * @param wal Write ahead log manager.
- */
- public MetadataStorage(
- final PageMemory pageMem,
- final IgniteWriteAheadLogManager wal,
- final AtomicLong globalRmvId,
- final int grpId,
- final int allocPartId,
- final byte allocSpace,
- final ReuseList reuseList,
- final long rootPageId,
- final boolean initNew
- ) {
- try {
- this.pageMem = pageMem;
- this.grpId = grpId;
- this.allocPartId = allocPartId;
- this.allocSpace = allocSpace;
- this.reuseList = reuseList;
-
- metaTree = new MetaTree(grpId, allocPartId, allocSpace, pageMem, wal, globalRmvId, rootPageId,
- reuseList, MetaStoreInnerIO.VERSIONS, MetaStoreLeafIO.VERSIONS, initNew);
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public RootPage getOrAllocateForTree(final String idxName) throws IgniteCheckedException {
- final MetaTree tree = metaTree;
-
- synchronized (this) {
- byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
-
- if (idxNameBytes.length > MAX_IDX_NAME_LEN)
- throw new IllegalArgumentException("Too long encoded indexName [maxAllowed=" + MAX_IDX_NAME_LEN +
- ", currentLength=" + idxNameBytes.length + ", name=" + idxName + "]");
-
- final IndexItem row = tree.findOne(new IndexItem(idxNameBytes, 0));
-
- if (row == null) {
- long pageId = 0;
-
- if (reuseList != null)
- pageId = reuseList.takeRecycledPage();
-
- pageId = pageId == 0 ? pageMem.allocatePage(grpId, allocPartId, allocSpace) : pageId;
-
- tree.put(new IndexItem(idxNameBytes, pageId));
-
- return new RootPage(new FullPageId(pageId, grpId), true);
- }
- else {
- final FullPageId pageId = new FullPageId(row.pageId, grpId);
-
- return new RootPage(pageId, false);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public RootPage dropRootPage(final String idxName)
- throws IgniteCheckedException {
- byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
-
- final IndexItem row = metaTree.remove(new IndexItem(idxNameBytes, 0));
-
- if (row != null) {
- if (reuseList == null)
- pageMem.freePage(grpId, row.pageId);
- }
-
- return row != null ? new RootPage(new FullPageId(row.pageId, grpId), false) : null;
- }
-
- /** {@inheritDoc} */
- @Override public void destroy() throws IgniteCheckedException {
- metaTree.destroy();
- }
-
- /**
- *
- */
- private static class MetaTree extends BPlusTree<IndexItem, IndexItem> {
- /** */
- private final int allocPartId;
-
- /** */
- private final byte allocSpace;
-
- /**
- * @param pageMem Page memory.
- * @param metaPageId Meta page ID.
- * @param reuseList Reuse list.
- * @param innerIos Inner IOs.
- * @param leafIos Leaf IOs.
- * @throws IgniteCheckedException If failed.
- */
- private MetaTree(
- final int cacheId,
- final int allocPartId,
- final byte allocSpace,
- final PageMemory pageMem,
- final IgniteWriteAheadLogManager wal,
- final AtomicLong globalRmvId,
- final long metaPageId,
- final ReuseList reuseList,
- final IOVersions<? extends BPlusInnerIO<IndexItem>> innerIos,
- final IOVersions<? extends BPlusLeafIO<IndexItem>> leafIos,
- final boolean initNew
- ) throws IgniteCheckedException {
- super(treeName("meta", "Meta"), cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList, innerIos, leafIos);
-
- this.allocPartId = allocPartId;
- this.allocSpace = allocSpace;
-
- initTree(initNew);
- }
-
- /** {@inheritDoc} */
- @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
- return pageMem.allocatePage(groupId(), allocPartId, allocSpace);
- }
-
- /** {@inheritDoc} */
- @Override protected int compare(final BPlusIO<IndexItem> io, final long pageAddr, final int idx,
- final IndexItem row) throws IgniteCheckedException {
- final int off = ((IndexIO)io).getOffset(pageAddr, idx);
-
- int shift = 0;
-
- // Compare index names.
- final int len = PageUtils.getUnsignedByte(pageAddr, off + shift);
-
- shift += BYTE_LEN;
-
- for (int i = 0; i < len && i < row.idxName.length; i++) {
- final int cmp = Byte.compare(PageUtils.getByte(pageAddr, off + i + shift), row.idxName[i]);
-
- if (cmp != 0)
- return cmp;
- }
-
- return Integer.compare(len, row.idxName.length);
- }
-
- /** {@inheritDoc} */
- @Override protected IndexItem getRow(final BPlusIO<IndexItem> io, final long pageAddr,
- final int idx, Object ignore) throws IgniteCheckedException {
- return readRow(pageAddr, ((IndexIO)io).getOffset(pageAddr, idx));
- }
- }
-
- /**
- *
- */
- private static class IndexItem {
- /** */
- private byte[] idxName;
-
- /** */
- private long pageId;
-
- /**
- * @param idxName Index name.
- * @param pageId Page ID.
- */
- private IndexItem(final byte[] idxName, final long pageId) {
- this.idxName = idxName;
- this.pageId = pageId;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "I [idxName=" + new String(idxName) + ", pageId=" + U.hexLong(pageId) + ']';
- }
- }
-
- /**
- * Store row to buffer.
- *
- * @param pageAddr Page address.
- * @param off Offset in buf.
- * @param row Row to store.
- */
- private static void storeRow(
- final long pageAddr,
- int off,
- final IndexItem row
- ) {
- // Index name length.
- PageUtils.putUnsignedByte(pageAddr, off, row.idxName.length);
- off++;
-
- // Index name.
- PageUtils.putBytes(pageAddr, off, row.idxName);
- off += row.idxName.length;
-
- // Page ID.
- PageUtils.putLong(pageAddr, off, row.pageId);
- }
-
- /**
- * Copy row data.
- *
- * @param dstPageAddr Destination page address.
- * @param dstOff Destination buf offset.
- * @param srcPageAddr Source page address.
- * @param srcOff Src buf offset.
- */
- private static void storeRow(
- final long dstPageAddr,
- int dstOff,
- final long srcPageAddr,
- int srcOff
- ) {
- // Index name length.
- final int len = PageUtils.getUnsignedByte(srcPageAddr, srcOff);
- srcOff++;
-
- PageUtils.putUnsignedByte(dstPageAddr, dstOff, len);
- dstOff++;
-
- PageHandler.copyMemory(srcPageAddr, srcOff, dstPageAddr, dstOff, len);
- srcOff += len;
- dstOff += len;
-
- // Page ID.
- PageUtils.putLong(dstPageAddr, dstOff, PageUtils.getLong(srcPageAddr, srcOff));
- }
-
- /**
- * Read row from buffer.
- *
- * @param pageAddr Page address.
- * @param off Offset.
- * @return Read row.
- */
- private static IndexItem readRow(final long pageAddr, int off) {
- // Index name length.
- final int len = PageUtils.getUnsignedByte(pageAddr, off) & 0xFF;
- off++;
-
- // Index name.
- final byte[] idxName = PageUtils.getBytes(pageAddr, off, len);
- off += len;
-
- // Page ID.
- final long pageId = PageUtils.getLong(pageAddr, off);
-
- return new IndexItem(idxName, pageId);
- }
-
- /**
- *
- */
- private interface IndexIO {
- /**
- * @param pageAddr Page address.
- * @param idx Index.
- * @return Offset in buffer according to {@code idx}.
- */
- int getOffset(long pageAddr, int idx);
- }
-
- /**
- *
- */
- public static final class MetaStoreInnerIO extends BPlusInnerIO<IndexItem> implements IndexIO {
- /** */
- public static final IOVersions<MetaStoreInnerIO> VERSIONS = new IOVersions<>(
- new MetaStoreInnerIO(1)
- );
-
- /**
- * @param ver Version.
- */
- private MetaStoreInnerIO(final int ver) {
- // name bytes and 1 byte for length, 8 bytes pageId
- super(T_METASTORE_INNER, ver, false, RESERVED_IDX_NAME_LEN + 1 + 8);
- }
-
- /** {@inheritDoc} */
- @Override public void storeByOffset(long pageAddr, int off, IndexItem row) throws IgniteCheckedException {
- storeRow(pageAddr, off, row);
- }
-
- /** {@inheritDoc} */
- @Override public void store(final long dstPageAddr, final int dstIdx, final BPlusIO<IndexItem> srcIo,
- final long srcPageAddr,
- final int srcIdx) throws IgniteCheckedException {
- storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
- }
-
- /** {@inheritDoc} */
- @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree, final long pageAddr,
- final int idx) throws IgniteCheckedException {
- return readRow(pageAddr, offset(idx));
- }
-
- /** {@inheritDoc} */
- @Override public int getOffset(long pageAddr, final int idx) {
- return offset(idx);
- }
- }
-
- /**
- *
- */
- public static final class MetaStoreLeafIO extends BPlusLeafIO<IndexItem> implements IndexIO {
- /** */
- public static final IOVersions<MetaStoreLeafIO> VERSIONS = new IOVersions<>(
- new MetaStoreLeafIO(1)
- );
-
- /**
- * @param ver Version.
- */
- private MetaStoreLeafIO(final int ver) {
- // 4 byte cache ID, UTF-16 symbols and 1 byte for length, 8 bytes pageId
- super(T_METASTORE_LEAF, ver, RESERVED_IDX_NAME_LEN + 1 + 8);
- }
-
- /** {@inheritDoc} */
- @Override public void storeByOffset(long buf, int off, IndexItem row) throws IgniteCheckedException {
- storeRow(buf, off, row);
- }
-
- /** {@inheritDoc} */
- @Override public void store(final long dstPageAddr,
- final int dstIdx,
- final BPlusIO<IndexItem> srcIo,
- final long srcPageAddr,
- final int srcIdx) throws IgniteCheckedException {
- storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
- }
-
- /** {@inheritDoc} */
- @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree,
- final long pageAddr,
- final int idx) throws IgniteCheckedException {
- return readRow(pageAddr, offset(idx));
- }
-
- /** {@inheritDoc} */
- @Override public int getOffset(long pageAddr, final int idx) {
- return offset(idx);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RecoveryDebug.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RecoveryDebug.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RecoveryDebug.java
new file mode 100644
index 0000000..54017b4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RecoveryDebug.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.TimeZone;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+/**
+ *
+ */
+public class RecoveryDebug implements AutoCloseable {
+ /** */
+ private static final ThreadLocal<SimpleDateFormat> sdf = new ThreadLocal<SimpleDateFormat>() {
+ /** {@inheritDoc} */
+ @Override protected SimpleDateFormat initialValue() {
+ SimpleDateFormat f = new SimpleDateFormat("dd-MM-yyyy-HH-mm-ss-SSS");
+
+ f.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ return f;
+ }
+ };
+
+ /** */
+ @Nullable private final IgniteLogger log;
+
+ /** */
+ @Nullable private FileChannel fc;
+
+ /**
+ * @param constId Consistent ID.
+ */
+ public RecoveryDebug(Object constId, long time) {
+ this(constId, time, null);
+ }
+
+ /**
+ * @param constId Consistent ID.
+ * @param log Logger.
+ */
+ public RecoveryDebug(Object constId,long time, @Nullable IgniteLogger log) {
+ this.log = log;
+
+ try {
+ String workDir = U.defaultWorkDirectory();
+
+ File tmpDir = new File(workDir, "tmp");
+
+ if (!tmpDir.exists())
+ if (!tmpDir.mkdir())
+ return;
+
+ File f = new File(tmpDir, "recovery-" +
+ sdf.get().format(new Date(time)) + "-" + constId +".log");
+
+ f.createNewFile();
+
+ fc = FileChannel.open(Paths.get(f.getPath()), EnumSet.of(CREATE, READ, WRITE));
+ }
+ catch (IgniteCheckedException | IOException e) {
+ U.error(log, "Fail create recovery debug file.", e);
+
+ fc = null;
+ }
+ }
+
+ /**
+ * @param rec TX record to append.
+ * @return {@code this} for convenience.
+ */
+ public RecoveryDebug append(TxRecord rec) {
+ GridCacheVersion txVer = rec.nearXidVersion();
+
+ return fc == null ? this : appendFile(
+ "Tx record " + rec.state() + " " + rec.nearXidVersion() + " timestamp " + rec.timestamp()
+ );
+ }
+
+ /**
+ * @param rec Data record to append.
+ * @param unwrapKeyValue unwrap key and value flag.
+ * @return {@code this} for convenience.
+ */
+ public RecoveryDebug append(DataRecord rec, boolean unwrapKeyValue) {
+ if (fc == null)
+ return this;
+
+ append("Data record\n");
+
+ for (DataEntry dataEntry : rec.writeEntries())
+ append("\t" + dataEntry.op() + " " + dataEntry.nearXidVersion() +
+ (unwrapKeyValue ? " " + dataEntry.key() + " " + dataEntry.value() : "") + "\n"
+ );
+
+ return this;
+ }
+
+ /**
+ * @param st Statement to append.
+ * @return {@code this} for convenience.
+ */
+ public RecoveryDebug append(Object st) {
+ return fc == null ? this : appendFile(st);
+ }
+
+ /**
+ * @param st Statement to append.
+ * @return {@code this} for convenience.
+ */
+ private RecoveryDebug appendFile(Object st) {
+ try {
+ fc.write(ByteBuffer.wrap(st.toString().getBytes()));
+ }
+ catch (IOException e) {
+ U.error(null, "Fail write to recovery dump file.", e);
+ }
+
+ return this;
+ }
+
+ /**
+ * Closes this debug insrance.
+ */
+ @Override public void close() {
+ if (fc != null) {
+ try {
+ fc.force(true);
+
+ fc.close();
+ }
+ catch (IOException e) {
+ U.error(null, "Fail close recovery dump file.", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/Storable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/Storable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/Storable.java
new file mode 100644
index 0000000..ae200df
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/Storable.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+/**
+ * Simple interface for data, store in some RowStore.
+ */
+public interface Storable {
+ /**
+ * @param link Link for this row.
+ */
+ public void link(long link);
+
+ /**
+ * @return Link for this row.
+ */
+ public long link();
+
+ /**
+ * @return Partition.
+ */
+ public int partition();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloader.java
new file mode 100644
index 0000000..3c54a49
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloader.java
@@ -0,0 +1,189 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Part of direct node to node file downloading
+ */
+public class FileDownloader {
+ /** */
+ private final IgniteLogger log;
+
+ /** */
+ private static final int CHUNK_SIZE = 1024 * 1024;
+
+ /** */
+ private final Path path;
+
+ /** */
+ private final AtomicLong size = new AtomicLong(-1);
+
+ /** */
+ private ServerSocketChannel serverChannel;
+
+ /** */
+ private volatile GridFutureAdapter<?> finishFut;
+
+ /** */
+ private final GridFutureAdapter<?> initFut = new GridFutureAdapter<>();
+
+ /**
+ *
+ */
+ public FileDownloader(IgniteLogger log, Path path) {
+ this.log = log;
+ this.path = path;
+ }
+
+ /**
+ *
+ */
+ public InetSocketAddress start() throws IgniteCheckedException {
+ try {
+ ServerSocketChannel ch = ServerSocketChannel.open();
+
+ ch.bind(null);
+
+ serverChannel = ch;
+
+ return (InetSocketAddress)ch.getLocalAddress();
+ }
+ catch (Exception ex) {
+ throw new IgniteCheckedException(ex);
+ }
+ }
+
+ /**
+ *
+ */
+ public void download(GridFutureAdapter<?> fut){
+ this.finishFut = fut;
+
+ final ServerSocketChannel ch = serverChannel;
+
+ fut.listen(new IgniteInClosureX<IgniteInternalFuture<?>>() {
+ @Override public void applyx(IgniteInternalFuture<?> future) throws IgniteCheckedException {
+ try {
+
+ if (log != null && log.isInfoEnabled())
+ log.info("Server socket closed " + ch.getLocalAddress());
+
+ ch.close();
+ }
+ catch (Exception ex) {
+ U.error(log, "Fail close socket.", ex);
+
+ throw new IgniteCheckedException(ex);
+ }
+ }
+ });
+
+ FileChannel writeChannel = null;
+ SocketChannel readChannel = null;
+
+ try {
+ File f = new File(path.toUri().getPath());
+
+ if (f.exists())
+ f.delete();
+
+ File cacheWorkDir = f.getParentFile();
+
+ if (!cacheWorkDir.exists())
+ cacheWorkDir.mkdir();
+
+ writeChannel = FileChannel.open(path, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
+
+ initFut.onDone();
+
+ readChannel = serverChannel.accept();
+
+ long pos = 0;
+
+ long size = this.size.get();
+
+ while (size == -1 || pos < size) {
+ pos += writeChannel.transferFrom(readChannel, pos, CHUNK_SIZE);
+
+ if (size == -1)
+ size = this.size.get();
+ }
+ }
+ catch (IOException ex) {
+ initFut.onDone(ex);
+
+ fut.onDone(ex);
+ }
+ finally {
+ try {
+ if (writeChannel != null)
+ writeChannel.close();
+ }
+ catch (IOException ex) {
+ throw new IgniteException("Could not close file: " + path);
+ }
+
+ try {
+ if (readChannel != null)
+ readChannel.close();
+ }
+ catch (IOException ex) {
+ throw new IgniteException("Could not close socket");
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ public void download(long size, Throwable th) {
+ try {
+ initFut.get();
+
+ if (th != null)
+ finishFut.onDone(th);
+ else {
+ if (!this.size.compareAndSet(-1, size))
+ finishFut.onDone(new IgniteException("Size mismatch: " + this.size.get() + " != " + size));
+ else
+ finishFut.onDone();
+ }
+
+ }
+ catch (IgniteCheckedException e) {
+ finishFut.onDone(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 47f1d4d..053ab2b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -292,8 +292,10 @@ public class FilePageStore implements PageStore {
lock.writeLock().lock();
try {
+ // Since we always have a meta-page in the store, never revert allocated counter to a value smaller than
+ // header + page.
if (inited)
- allocated.set(fileIO.size());
+ allocated.set(Math.max(headerSize() + pageSize, fileIO.size()));
recover = false;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 66af0dd..d406df6 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -48,11 +48,13 @@ import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
import org.apache.ignite.internal.processors.cache.StoredCacheData;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.marshaller.Marshaller;
import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
@@ -196,8 +198,23 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
}
/** {@inheritDoc} */
+ @Override public void initializeForMetastorage()
+ throws IgniteCheckedException {
+ int grpId = MetaStorage.METASTORAGE_CACHE_ID;
+
+ if (!idxCacheStores.containsKey(grpId)) {
+ CacheStoreHolder holder = initDir(new File(storeWorkDir, "metastorage"), grpId, 1);
+
+ CacheStoreHolder old = idxCacheStores.put(grpId, holder);
+
+ assert old == null : "Non-null old store holder for metastorage";
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException {
- File cacheWorkDir = cacheWorkDirectory(cacheData.config());
+ File cacheWorkDir = cacheWorkDir(cacheData.config());
+
File file;
checkAndInitCacheWorkDir(cacheWorkDir);
@@ -321,18 +338,10 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
}
/**
- * @param ccfg Cache configuration.
- * @return Cache work directory.
+ *
*/
- private File cacheWorkDirectory(CacheConfiguration ccfg) {
- String dirName;
-
- if (ccfg.getGroupName() != null)
- dirName = CACHE_GRP_DIR_PREFIX + ccfg.getGroupName();
- else
- dirName = CACHE_DIR_PREFIX + ccfg.getName();
-
- return new File(storeWorkDir, dirName);
+ public Path getPath(boolean isSharedGroup, String cacheOrGroupName, int partId) {
+ return getPartitionFile(cacheWorkDir(isSharedGroup, cacheOrGroupName), partId).toPath();
}
/**
@@ -344,25 +353,36 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
private CacheStoreHolder initForCache(CacheGroupDescriptor grpDesc, CacheConfiguration ccfg) throws IgniteCheckedException {
assert !grpDesc.sharedGroup() || ccfg.getGroupName() != null : ccfg.getName();
- File cacheWorkDir = cacheWorkDirectory(ccfg);
+ File cacheWorkDir = cacheWorkDir(ccfg);
+
+ return initDir(cacheWorkDir, grpDesc.groupId(), grpDesc.config().getAffinity().partitions());
+ }
+ /**
+ * @param cacheWorkDir Work directory.
+ * @param grpId Group ID.
+ * @param partitions Number of partitions.
+ * @return Cache store holder.
+ * @throws IgniteCheckedException If failed.
+ */
+ private CacheStoreHolder initDir(File cacheWorkDir, int grpId, int partitions) throws IgniteCheckedException {
boolean dirExisted = checkAndInitCacheWorkDir(cacheWorkDir);
File idxFile = new File(cacheWorkDir, INDEX_FILE_NAME);
if (dirExisted && !idxFile.exists())
- grpsWithoutIdx.add(grpDesc.groupId());
+ grpsWithoutIdx.add(grpId);
FileVersionCheckingFactory pageStoreFactory = new FileVersionCheckingFactory(
dsCfg.getFileIOFactory(), igniteCfg.getDataStorageConfiguration());
FilePageStore idxStore = pageStoreFactory.createPageStore(PageMemory.FLAG_IDX, idxFile);
- FilePageStore[] partStores = new FilePageStore[grpDesc.config().getAffinity().partitions()];
+ FilePageStore[] partStores = new FilePageStore[partitions];
for (int partId = 0; partId < partStores.length; partId++) {
FilePageStore partStore = pageStoreFactory.createPageStore(
- PageMemory.FLAG_DATA, new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)));
+ PageMemory.FLAG_DATA, getPartitionFile(cacheWorkDir, partId));
partStores[partId] = partStore;
}
@@ -372,6 +392,14 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
/**
* @param cacheWorkDir Cache work directory.
+ * @param partId Partition id.
+ */
+ @NotNull private File getPartitionFile(File cacheWorkDir, int partId) {
+ return new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId));
+ }
+
+ /**
+ * @param cacheWorkDir Cache work directory.
*/
private boolean checkAndInitCacheWorkDir(File cacheWorkDir) throws IgniteCheckedException {
boolean dirExisted = false;
@@ -558,8 +586,21 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
* @return Store dir for given cache.
*/
public File cacheWorkDir(CacheConfiguration ccfg) {
- String dirName = ccfg.getGroupName() == null ?
- CACHE_DIR_PREFIX + ccfg.getName() : CACHE_GRP_DIR_PREFIX + ccfg.getGroupName();
+ boolean isSharedGrp = ccfg.getGroupName() != null;
+
+ return cacheWorkDir(isSharedGrp, isSharedGrp ? ccfg.getGroupName() : ccfg.getName());
+ }
+
+ /**
+ *
+ */
+ public File cacheWorkDir(boolean isSharedGroup, String cacheOrGroupName) {
+ String dirName;
+
+ if (isSharedGroup)
+ dirName = CACHE_GRP_DIR_PREFIX + cacheOrGroupName;
+ else
+ dirName = CACHE_DIR_PREFIX + cacheOrGroupName;
return new File(storeWorkDir, dirName);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileUploader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileUploader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileUploader.java
new file mode 100644
index 0000000..ba21ae9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileUploader.java
@@ -0,0 +1,100 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+
+/**
+ * Part of direct node to node file downloading
+ */
+public class FileUploader {
+ /** */
+ private static final int CHUNK_SIZE = 1024 * 1024;
+
+ /** */
+ private final Path path;
+
+ /**
+ *
+ */
+ public FileUploader(Path path) {
+ this.path = path;
+ }
+
+ /**
+ *
+ */
+ public void upload(SocketChannel writeChannel, GridFutureAdapter<Long> finishFut) {
+ FileChannel readChannel = null;
+
+ try {
+ File file = new File(path.toUri().getPath());
+
+ if (!file.exists()) {
+ finishFut.onDone(
+ new IgniteCheckedException(
+ new FileNotFoundException(file.getAbsolutePath())
+ )
+ );
+
+ return;
+ }
+
+ readChannel = FileChannel.open(path, StandardOpenOption.READ);
+
+ long written = 0;
+
+ long size = readChannel.size();
+
+ while (written < size)
+ written += readChannel.transferTo(written, CHUNK_SIZE, writeChannel);
+
+ finishFut.onDone(written);
+ }
+ catch (IOException ex) {
+ finishFut.onDone(ex);
+ }
+ finally {
+ //FIXME: when an error occurs on writeChannel.close() no attempt to close readChannel will happen. Need to be fixed.
+ try {
+ if (writeChannel != null)
+ writeChannel.close();
+ }
+ catch (IOException ex) {
+ throw new IgniteException("Could not close socket.");
+ }
+
+ try {
+ if (readChannel != null)
+ readChannel.close();
+ }
+ catch (IOException ex) {
+ throw new IgniteException("Could not close file: " + path);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
new file mode 100644
index 0000000..83ff91b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
@@ -0,0 +1,141 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+/**
+ * {@link FileIO} that allows to work with ZIP compressed file.
+ * Doesn't allow random access and setting {@link FileIO#position()} backwards.
+ * Allows sequential reads including setting {@link FileIO#position()} forward.
+ */
+public class UnzipFileIO implements FileIO {
+ /** Zip input stream. */
+ private final ZipInputStream zis;
+
+ /** Byte array for draining data. */
+ private final byte[] arr = new byte[128 * 1024];
+
+ /** Size of uncompressed data. */
+ private final long size;
+
+ /** Total bytes read counter. */
+ private long totalBytesRead = 0;
+
+ /**
+ * @param zip Compressed file.
+ */
+ public UnzipFileIO(File zip) throws IOException {
+ zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
+
+ ZipEntry entry = zis.getNextEntry();
+ size = entry.getSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long position() throws IOException {
+ return totalBytesRead;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void position(long newPosition) throws IOException {
+ if (newPosition == totalBytesRead)
+ return;
+
+ if (newPosition < totalBytesRead)
+ throw new UnsupportedOperationException("Seeking backwards is not supported.");
+
+ long bytesRemaining = newPosition - totalBytesRead;
+
+ while (bytesRemaining > 0) {
+ int bytesToRead = bytesRemaining > arr.length ? arr.length : (int)bytesRemaining;
+
+ bytesRemaining -= zis.read(arr, 0, bytesToRead);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(ByteBuffer dstBuf) throws IOException {
+ int bytesRead = zis.read(arr, 0, Math.min(dstBuf.remaining(), arr.length));
+
+ if (bytesRead == -1)
+ return -1;
+
+ dstBuf.put(arr, 0, bytesRead);
+
+ totalBytesRead += bytesRead;
+
+ return bytesRead;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(ByteBuffer dstBuf, long position) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(byte[] buf, int off, int len) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int write(ByteBuffer srcBuf) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int write(ByteBuffer srcBuf, long position) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(byte[] buf, int off, int len) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void force() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long size() throws IOException {
+ return size;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void clear() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ zis.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
index e7a7e63..ba6d822 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
@@ -161,6 +161,9 @@ public class PdsConsistentIdProcessor extends GridProcessorAdapter implements Pd
if (!CU.isPersistenceEnabled(cfg))
return compatibleResolve(pstStoreBasePath, consistentId);
+ if (ctx.clientNode())
+ return new PdsFolderSettings(pstStoreBasePath, UUID.randomUUID());
+
if (getBoolean(IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID, false))
return compatibleResolve(pstStoreBasePath, consistentId);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
new file mode 100644
index 0000000..0847ca6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
@@ -0,0 +1,605 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.freelist;
+
+import java.util.concurrent.atomic.AtomicReferenceArray;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.Storable;
+import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ */
+public abstract class AbstractFreeList<T extends Storable> extends PagesList implements FreeList<T>, ReuseList {
+ /** */
+ private static final int BUCKETS = 256; // Must be power of 2.
+
+ /** */
+ private static final int REUSE_BUCKET = BUCKETS - 1;
+
+ /** */
+ private static final Integer COMPLETE = Integer.MAX_VALUE;
+
+ /** */
+ private static final Integer FAIL_I = Integer.MIN_VALUE;
+
+ /** */
+ private static final Long FAIL_L = Long.MAX_VALUE;
+
+ /** */
+ private static final int MIN_PAGE_FREE_SPACE = 8;
+
+ /** */
+ private final int shift;
+
+ /** */
+ private final AtomicReferenceArray<Stripe[]> buckets = new AtomicReferenceArray<>(BUCKETS);
+
+ /** */
+ private final int MIN_SIZE_FOR_DATA_PAGE;
+
+ /** */
+ private final int emptyDataPagesBucket;
+
+ /** */
+ private final PageHandler<T, Boolean> updateRow = new UpdateRowHandler();
+
+ /** */
+ private final DataRegionMetricsImpl memMetrics;
+
+ /** */
+ private final PageEvictionTracker evictionTracker;
+
+ /**
+ *
+ */
+ private final class UpdateRowHandler extends PageHandler<T, Boolean> {
+ @Override public Boolean run(
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageIO iox,
+ Boolean walPlc,
+ T row,
+ int itemId)
+ throws IgniteCheckedException {
+ AbstractDataPageIO<T> io = (AbstractDataPageIO<T>)iox;
+
+ int rowSize = io.getRowSize(row);
+
+ boolean updated = io.updateRow(pageAddr, itemId, pageSize(), null, row, rowSize);
+
+ evictionTracker.touchPage(pageId);
+
+ if (updated && needWalDeltaRecord(pageId, page, walPlc)) {
+ // TODO This record must contain only a reference to a logical WAL record with the actual data.
+ byte[] payload = new byte[rowSize];
+
+ DataPagePayload data = io.readPayload(pageAddr, itemId, pageSize());
+
+ assert data.payloadSize() == rowSize;
+
+ PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
+
+ wal.log(new DataPageUpdateRecord(
+ cacheId,
+ pageId,
+ itemId,
+ payload));
+ }
+
+ return updated;
+ }
+ }
+
+ /** */
+ private final PageHandler<T, Integer> writeRow = new WriteRowHandler();
+
+ /**
+ *
+ */
+ private final class WriteRowHandler extends PageHandler<T, Integer> {
+ @Override public Integer run(
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageIO iox,
+ Boolean walPlc,
+ T row,
+ int written)
+ throws IgniteCheckedException {
+ AbstractDataPageIO<T> io = (AbstractDataPageIO<T>)iox;
+
+ int rowSize = io.getRowSize(row);
+ int oldFreeSpace = io.getFreeSpace(pageAddr);
+
+ assert oldFreeSpace > 0 : oldFreeSpace;
+
+ // If the full row does not fit into this page write only a fragment.
+ written = (written == 0 && oldFreeSpace >= rowSize) ? addRow(pageId, page, pageAddr, io, row, rowSize) :
+ addRowFragment(pageId, page, pageAddr, io, row, written, rowSize);
+
+ // Reread free space after update.
+ int newFreeSpace = io.getFreeSpace(pageAddr);
+
+ if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
+ int bucket = bucket(newFreeSpace, false);
+
+ put(null, pageId, page, pageAddr, bucket);
+ }
+
+ if (written == rowSize)
+ evictionTracker.touchPage(pageId);
+
+ // Avoid boxing with garbage generation for usual case.
+ return written == rowSize ? COMPLETE : written;
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param io IO.
+ * @param row Row.
+ * @param rowSize Row size.
+ * @return Written size which is always equal to row size here.
+ * @throws IgniteCheckedException If failed.
+ */
+ private int addRow(
+ long pageId,
+ long page,
+ long pageAddr,
+ AbstractDataPageIO<T> io,
+ T row,
+ int rowSize
+ ) throws IgniteCheckedException {
+ io.addRow(pageId, pageAddr, row, rowSize, pageSize());
+
+ if (needWalDeltaRecord(pageId, page, null)) {
+ // TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data.
+ byte[] payload = new byte[rowSize];
+
+ DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
+
+ assert data.payloadSize() == rowSize;
+
+ PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
+
+ wal.log(new DataPageInsertRecord(
+ grpId,
+ pageId,
+ payload));
+ }
+
+ return rowSize;
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param io IO.
+ * @param row Row.
+ * @param written Written size.
+ * @param rowSize Row size.
+ * @return Updated written size.
+ * @throws IgniteCheckedException If failed.
+ */
+ private int addRowFragment(
+ long pageId,
+ long page,
+ long pageAddr,
+ AbstractDataPageIO<T> io,
+ T row,
+ int written,
+ int rowSize
+ ) throws IgniteCheckedException {
+ // Read last link before the fragment write, because it will be updated there.
+ long lastLink = row.link();
+
+ int payloadSize = io.addRowFragment(pageMem, pageAddr, row, written, rowSize, pageSize());
+
+ assert payloadSize > 0 : payloadSize;
+
+ if (needWalDeltaRecord(pageId, page, null)) {
+ // TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data.
+ byte[] payload = new byte[payloadSize];
+
+ DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
+
+ PageUtils.getBytes(pageAddr, data.offset(), payload, 0, payloadSize);
+
+ wal.log(new DataPageInsertFragmentRecord(grpId, pageId, payload, lastLink));
+ }
+
+ return written + payloadSize;
+ }
+ }
+
+ /** */
+ private final PageHandler<Void, Long> rmvRow;
+
+ /**
+ *
+ */
+ private final class RemoveRowHandler extends PageHandler<Void, Long> {
+ /** Indicates whether partition ID should be masked from page ID. */
+ private final boolean maskPartId;
+
+ /** */
+ RemoveRowHandler(boolean maskPartId) {
+ this.maskPartId = maskPartId;
+ }
+
+ @Override public Long run(
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageIO iox,
+ Boolean walPlc,
+ Void ignored,
+ int itemId)
+ throws IgniteCheckedException {
+ AbstractDataPageIO<T> io = (AbstractDataPageIO<T>)iox;
+
+ int oldFreeSpace = io.getFreeSpace(pageAddr);
+
+ assert oldFreeSpace >= 0 : oldFreeSpace;
+
+ long nextLink = io.removeRow(pageAddr, itemId, pageSize());
+
+ if (needWalDeltaRecord(pageId, page, walPlc))
+ wal.log(new DataPageRemoveRecord(cacheId, pageId, itemId));
+
+ int newFreeSpace = io.getFreeSpace(pageAddr);
+
+ if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
+ int newBucket = bucket(newFreeSpace, false);
+
+ if (oldFreeSpace > MIN_PAGE_FREE_SPACE) {
+ int oldBucket = bucket(oldFreeSpace, false);
+
+ if (oldBucket != newBucket) {
+ // It is possible that page was concurrently taken for put, in this case put will handle bucket change.
+ pageId = maskPartId ? PageIdUtils.maskPartitionId(pageId) : pageId;
+ if (removeDataPage(pageId, page, pageAddr, io, oldBucket))
+ put(null, pageId, page, pageAddr, newBucket);
+ }
+ }
+ else
+ put(null, pageId, page, pageAddr, newBucket);
+
+ if (io.isEmpty(pageAddr))
+ evictionTracker.forgetPage(pageId);
+ }
+
+ // For common case boxed 0L will be cached inside of Long, so no garbage will be produced.
+ return nextLink;
+ }
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param name Name (for debug purpose).
+ * @param memMetrics Memory metrics.
+ * @param memPlc Data region.
+ * @param reuseList Reuse list or {@code null} if this free list will be a reuse list for itself.
+ * @param wal Write ahead log manager.
+ * @param metaPageId Metadata page ID.
+ * @param initNew {@code True} if new metadata should be initialized.
+ * @throws IgniteCheckedException If failed.
+ */
+ public AbstractFreeList(
+ int cacheId,
+ String name,
+ DataRegionMetricsImpl memMetrics,
+ DataRegion memPlc,
+ ReuseList reuseList,
+ IgniteWriteAheadLogManager wal,
+ long metaPageId,
+ boolean initNew) throws IgniteCheckedException {
+ super(cacheId, name, memPlc.pageMemory(), BUCKETS, wal, metaPageId);
+
+ rmvRow = new RemoveRowHandler(cacheId == 0);
+
+ this.evictionTracker = memPlc.evictionTracker();
+ this.reuseList = reuseList == null ? this : reuseList;
+ int pageSize = pageMem.pageSize();
+
+ assert U.isPow2(pageSize) : "Page size must be a power of 2: " + pageSize;
+ assert U.isPow2(BUCKETS);
+ assert BUCKETS <= pageSize : pageSize;
+
+ // TODO this constant is used because currently we cannot reuse data pages as index pages
+ // TODO and vice-versa. It should be removed when data storage format is finalized.
+ MIN_SIZE_FOR_DATA_PAGE = pageSize - AbstractDataPageIO.MIN_DATA_PAGE_OVERHEAD;
+
+ int shift = 0;
+
+ while (pageSize > BUCKETS) {
+ shift++;
+ pageSize >>>= 1;
+ }
+
+ this.shift = shift;
+
+ this.memMetrics = memMetrics;
+
+ emptyDataPagesBucket = bucket(MIN_SIZE_FOR_DATA_PAGE, false);
+
+ init(metaPageId, initNew);
+ }
+
+ /**
+ * Calculates average fill factor over FreeListImpl instance.
+ *
+ * @return Tuple (numenator, denominator).
+ */
+ public T2<Long, Long> fillFactor() {
+ long pageSize = pageSize();
+
+ long totalSize = 0;
+ long loadSize = 0;
+
+ for (int b = BUCKETS - 2; b > 0; b--) {
+ long bsize = pageSize - ((REUSE_BUCKET - b) << shift);
+
+ long pages = bucketsSize[b].longValue();
+
+ loadSize += pages * (pageSize - bsize);
+
+ totalSize += pages * pageSize;
+ }
+
+ return totalSize == 0 ? new T2<>(0L, 0L) : new T2<>(loadSize, totalSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void dumpStatistics(IgniteLogger log) {
+ long dataPages = 0;
+
+ final boolean dumpBucketsInfo = false;
+
+ for (int b = 0; b < BUCKETS; b++) {
+ long size = bucketsSize[b].longValue();
+
+ if (!isReuseBucket(b))
+ dataPages += size;
+
+ if (dumpBucketsInfo) {
+ Stripe[] stripes = getBucket(b);
+
+ boolean empty = true;
+
+ if (stripes != null) {
+ for (Stripe stripe : stripes) {
+ if (!stripe.empty) {
+ empty = false;
+
+ break;
+ }
+ }
+ }
+
+ if (log.isInfoEnabled())
+ log.info("Bucket [b=" + b +
+ ", size=" + size +
+ ", stripes=" + (stripes != null ? stripes.length : 0) +
+ ", stripesEmpty=" + empty + ']');
+ }
+ }
+
+ if (dataPages > 0) {
+ if (log.isInfoEnabled())
+ log.info("FreeList [name=" + name +
+ ", buckets=" + BUCKETS +
+ ", dataPages=" + dataPages +
+ ", reusePages=" + bucketsSize[REUSE_BUCKET].longValue() + "]");
+ }
+ }
+
+ /**
+ * @param freeSpace Page free space.
+ * @param allowReuse {@code True} if it is allowed to get reuse bucket.
+ * @return Bucket.
+ */
+ private int bucket(int freeSpace, boolean allowReuse) {
+ assert freeSpace > 0 : freeSpace;
+
+ int bucket = freeSpace >>> shift;
+
+ assert bucket >= 0 && bucket < BUCKETS : bucket;
+
+ if (!allowReuse && isReuseBucket(bucket))
+ bucket--;
+
+ return bucket;
+ }
+
+ /**
+ * @param part Partition.
+ * @return Page ID.
+ * @throws IgniteCheckedException If failed.
+ */
+ private long allocateDataPage(int part) throws IgniteCheckedException {
+ assert part <= PageIdAllocator.MAX_PARTITION_ID;
+ assert part != PageIdAllocator.INDEX_PARTITION;
+
+ return pageMem.allocatePage(grpId, part, PageIdAllocator.FLAG_DATA);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void insertDataRow(T row) throws IgniteCheckedException {
+ int rowSize = ioVersions().latest().getRowSize(row);
+
+ int written = 0;
+
+ do {
+ if (written != 0)
+ memMetrics.incrementLargeEntriesPages();
+
+ int freeSpace = Math.min(MIN_SIZE_FOR_DATA_PAGE, rowSize - written);
+
+ long pageId = 0L;
+
+ if (freeSpace == MIN_SIZE_FOR_DATA_PAGE)
+ pageId = takeEmptyPage(emptyDataPagesBucket, ioVersions());
+
+ boolean reuseBucket = false;
+
+ // TODO: properly handle reuse bucket.
+ if (pageId == 0L) {
+ for (int b = bucket(freeSpace, false) + 1; b < BUCKETS - 1; b++) {
+ pageId = takeEmptyPage(b, ioVersions());
+
+ if (pageId != 0L) {
+ reuseBucket = isReuseBucket(b);
+
+ break;
+ }
+ }
+ }
+
+ boolean allocated = pageId == 0L;
+
+ if (allocated)
+ pageId = allocateDataPage(row.partition());
+ else
+ pageId = PageIdUtils.changePartitionId(pageId, (row.partition()));
+
+ AbstractDataPageIO<T> init = reuseBucket || allocated ? ioVersions().latest() : null;
+
+ written = write(pageId, writeRow, init, row, written, FAIL_I);
+
+ assert written != FAIL_I; // We can't fail here.
+ }
+ while (written != COMPLETE);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean updateDataRow(long link, T row) throws IgniteCheckedException {
+ assert link != 0;
+
+ long pageId = PageIdUtils.pageId(link);
+ int itemId = PageIdUtils.itemId(link);
+
+ Boolean updated = write(pageId, updateRow, row, itemId, null);
+
+ assert updated != null; // Can't fail here.
+
+ return updated;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void removeDataRowByLink(long link) throws IgniteCheckedException {
+ assert link != 0;
+
+ long pageId = PageIdUtils.pageId(link);
+ int itemId = PageIdUtils.itemId(link);
+
+ long nextLink = write(pageId, rmvRow, itemId, FAIL_L);
+
+ assert nextLink != FAIL_L; // Can't fail here.
+
+ while (nextLink != 0L) {
+ memMetrics.decrementLargeEntriesPages();
+
+ itemId = PageIdUtils.itemId(nextLink);
+ pageId = PageIdUtils.pageId(nextLink);
+
+ nextLink = write(pageId, rmvRow, itemId, FAIL_L);
+
+ assert nextLink != FAIL_L; // Can't fail here.
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Stripe[] getBucket(int bucket) {
+ return buckets.get(bucket);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean casBucket(int bucket, Stripe[] exp, Stripe[] upd) {
+ return buckets.compareAndSet(bucket, exp, upd);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean isReuseBucket(int bucket) {
+ return bucket == REUSE_BUCKET;
+ }
+
+ /**
+ * @return Number of empty data pages in free list.
+ */
+ public int emptyDataPages() {
+ return bucketsSize[emptyDataPagesBucket].intValue();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void addForRecycle(ReuseBag bag) throws IgniteCheckedException {
+ assert reuseList == this : "not allowed to be a reuse list";
+
+ put(bag, 0, 0, 0L, REUSE_BUCKET);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long takeRecycledPage() throws IgniteCheckedException {
+ assert reuseList == this : "not allowed to be a reuse list";
+
+ return takeEmptyPage(REUSE_BUCKET, null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long recycledPagesCount() throws IgniteCheckedException {
+ assert reuseList == this : "not allowed to be a reuse list";
+
+ return storedPagesCount(REUSE_BUCKET);
+ }
+
+ /**
+ * @return IOVersions.
+ */
+ public abstract IOVersions<? extends AbstractDataPageIO<T>> ioVersions();
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "FreeList [name=" + name + ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
new file mode 100644
index 0000000..dc0c92e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.freelist;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+
+/**
+ * FreeList implementation for cache.
+ */
+public class CacheFreeListImpl extends AbstractFreeList<CacheDataRow> {
+ /**
+ * @param cacheId Cache id.
+ * @param name Name.
+ * @param regionMetrics Region metrics.
+ * @param dataRegion Data region.
+ * @param reuseList Reuse list.
+ * @param wal Wal.
+ * @param metaPageId Meta page id.
+ * @param initNew Initialize new.
+ */
+ public CacheFreeListImpl(int cacheId, String name, DataRegionMetricsImpl regionMetrics, DataRegion dataRegion,
+ ReuseList reuseList,
+ IgniteWriteAheadLogManager wal, long metaPageId, boolean initNew) throws IgniteCheckedException {
+ super(cacheId, name, regionMetrics, dataRegion, reuseList, wal, metaPageId, initNew);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IOVersions<? extends AbstractDataPageIO<CacheDataRow>> ioVersions() {
+ return DataPageIO.VERSIONS;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "FreeList [name=" + name + ']';
+ }
+}
[17/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
IEP-4 Baseline topology for persistent caches (Phase 1)
Contributed by:
Dmitriy Govorukhin <dm...@gmail.com>
Dmitry Pavlov <dp...@gmail.com>
Eduard Shangareev <ed...@gmail.com>
Ilya Lantukh <il...@gridgain.com>
Ivan Rakov <iv...@gmail.com>
Sergey Chugunov <se...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6f7aba85
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6f7aba85
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6f7aba85
Branch: refs/heads/master
Commit: 6f7aba8526ff01a159b289aff932324a3604c1d8
Parents: 43a62cd
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Wed Jan 17 13:25:22 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Jan 17 13:30:12 2018 +0300
----------------------------------------------------------------------
RELEASE_NOTES.txt | 8 +
.../DummyPersistenceCompatibilityTest.java | 25 +-
.../testframework/util/MavenUtils.java | 26 +-
.../src/main/java/org/apache/ignite/Ignite.java | 4 +
.../java/org/apache/ignite/IgniteCluster.java | 39 +
.../apache/ignite/IgniteSystemProperties.java | 17 +
.../org/apache/ignite/cluster/BaselineNode.java | 73 +
.../org/apache/ignite/cluster/ClusterNode.java | 8 +-
.../configuration/AtomicConfiguration.java | 2 +-
.../configuration/DataRegionConfiguration.java | 6 +
.../configuration/DataStorageConfiguration.java | 8 +-
.../configuration/IgniteConfiguration.java | 37 +
.../apache/ignite/internal/GridComponent.java | 3 +
.../ignite/internal/GridKernalContext.java | 8 +-
.../ignite/internal/GridKernalContextImpl.java | 11 +
.../internal/GridNodeOrderComparator.java | 45 -
.../ignite/internal/GridPluginComponent.java | 5 +
.../apache/ignite/internal/IgniteKernal.java | 50 +-
.../internal/cluster/DetachedClusterNode.java | 121 ++
.../cluster/IgniteClusterAsyncImpl.java | 26 +
.../internal/cluster/IgniteClusterImpl.java | 172 +++
.../internal/cluster/NodeOrderComparator.java | 61 +
.../cluster/NodeOrderLegacyComparator.java | 46 +
.../internal/commandline/CommandHandler.java | 644 ++++++++-
.../internal/managers/GridManagerAdapter.java | 19 +
.../managers/discovery/ConsistentIdMapper.java | 96 +-
.../internal/managers/discovery/DiscoCache.java | 105 +-
.../discovery/GridDiscoveryManager.java | 124 +-
.../mem/unsafe/UnsafeMemoryProvider.java | 12 +-
.../pagemem/store/IgnitePageStoreManager.java | 5 +
.../pagemem/wal/IgniteWriteAheadLogManager.java | 5 +-
.../wal/record/BaselineTopologyRecord.java | 76 ++
.../internal/pagemem/wal/record/CacheState.java | 24 +
.../internal/pagemem/wal/record/DataEntry.java | 4 +-
.../internal/pagemem/wal/record/DataRecord.java | 6 +-
.../pagemem/wal/record/ExchangeRecord.java | 86 ++
.../pagemem/wal/record/FilteredRecord.java | 3 +
.../pagemem/wal/record/MetastoreDataRecord.java | 62 +
.../pagemem/wal/record/PageSnapshot.java | 6 +-
.../pagemem/wal/record/SnapshotRecord.java | 12 +
.../pagemem/wal/record/TimeStampRecord.java | 8 +
.../internal/pagemem/wal/record/TxRecord.java | 50 +-
.../internal/pagemem/wal/record/WALRecord.java | 18 +-
.../delta/DataPageInsertFragmentRecord.java | 5 +-
.../wal/record/delta/DataPageInsertRecord.java | 5 +-
.../wal/record/delta/DataPageRemoveRecord.java | 5 +-
.../delta/DataPageSetFreeListPageRecord.java | 5 +-
.../wal/record/delta/DataPageUpdateRecord.java | 5 +-
.../processors/GridProcessorAdapter.java | 5 +
.../affinity/GridAffinityAssignmentCache.java | 100 +-
.../cache/CacheAffinitySharedManager.java | 90 +-
.../cache/CacheJoinNodeDiscoveryData.java | 2 +-
.../processors/cache/ClusterCachesInfo.java | 59 +-
.../processors/cache/ExchangeActions.java | 31 +-
.../cache/ExchangeDiscoveryEvents.java | 29 +-
.../processors/cache/GridCacheAdapter.java | 20 +
.../processors/cache/GridCacheContext.java | 70 +-
.../processors/cache/GridCacheMapEntry.java | 63 +-
.../GridCachePartitionExchangeManager.java | 63 +-
.../processors/cache/GridCacheProcessor.java | 172 ++-
.../cache/GridCacheUpdateTxResult.java | 21 +-
.../processors/cache/GridCacheUtils.java | 18 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 7 +-
.../cache/LocalJoinCachesContext.java | 127 ++
.../processors/cache/StateChangeRequest.java | 42 +
.../cache/binary/BinaryMetadataFileStore.java | 4 +-
.../binary/CacheObjectBinaryProcessorImpl.java | 117 +-
.../GridDistributedTxRemoteAdapter.java | 17 +-
.../dht/GridDhtAssignmentFetchFuture.java | 4 +-
.../distributed/dht/GridDhtCacheAdapter.java | 4 +
.../distributed/dht/GridDhtLocalPartition.java | 3 +
.../dht/GridDhtPartitionTopologyImpl.java | 79 +-
.../dht/GridDhtTopologyFutureAdapter.java | 2 +-
.../distributed/dht/GridDhtTxLocalAdapter.java | 8 +-
.../distributed/dht/GridDhtTxPrepareFuture.java | 5 +
.../dht/GridPartitionedGetFuture.java | 13 +-
.../dht/GridPartitionedSingleGetFuture.java | 18 +-
.../atomic/GridNearAtomicFullUpdateRequest.java | 1 +
.../colocated/GridDhtDetachedCacheEntry.java | 8 +
.../CachePartitionPartialCountersMap.java | 17 +
.../dht/preloader/GridDhtPartitionDemander.java | 2 +-
.../GridDhtPartitionsExchangeFuture.java | 231 +++-
.../IgniteDhtPartitionsToReloadMap.java | 7 +
...StopCachesOnClientReconnectExchangeTask.java | 59 +
.../distributed/near/GridNearCacheEntry.java | 7 +
.../distributed/near/GridNearGetFuture.java | 13 +-
.../near/GridNearOptimisticTxPrepareFuture.java | 61 +-
...ridNearOptimisticTxPrepareFutureAdapter.java | 2 +-
.../near/GridNearTxPrepareFutureAdapter.java | 2 +-
.../cache/persistence/CacheDataRow.java | 2 +-
.../persistence/DataRegionMetricsImpl.java | 2 +-
.../cache/persistence/DataStructure.java | 3 +-
.../GridCacheDatabaseSharedManager.java | 668 +++++++++-
.../persistence/GridCacheOffheapManager.java | 416 +++---
.../IgniteCacheDatabaseSharedManager.java | 73 +-
.../cache/persistence/IndexStorage.java | 51 +
.../cache/persistence/IndexStorageImpl.java | 420 ++++++
.../processors/cache/persistence/MetaStore.java | 51 -
.../cache/persistence/MetadataStorage.java | 420 ------
.../cache/persistence/RecoveryDebug.java | 170 +++
.../processors/cache/persistence/Storable.java | 38 +
.../cache/persistence/file/FileDownloader.java | 189 +++
.../cache/persistence/file/FilePageStore.java | 4 +-
.../persistence/file/FilePageStoreManager.java | 77 +-
.../cache/persistence/file/FileUploader.java | 100 ++
.../cache/persistence/file/UnzipFileIO.java | 141 ++
.../filename/PdsConsistentIdProcessor.java | 3 +
.../persistence/freelist/AbstractFreeList.java | 605 +++++++++
.../persistence/freelist/CacheFreeListImpl.java | 59 +
.../cache/persistence/freelist/FreeList.java | 8 +-
.../persistence/freelist/FreeListImpl.java | 619 ---------
.../cache/persistence/freelist/PagesList.java | 11 +-
.../persistence/metastorage/MetaStorage.java | 450 +++++++
.../metastorage/MetastorageDataRow.java | 91 ++
.../MetastorageLifecycleListener.java | 47 +
.../metastorage/MetastorageRowStore.java | 96 ++
.../metastorage/MetastorageSearchRow.java | 38 +
.../metastorage/MetastorageTree.java | 266 ++++
.../metastorage/MetsatorageSearchRowImpl.java | 55 +
.../metastorage/ReadOnlyMetastorage.java | 28 +
.../metastorage/ReadWriteMetastorage.java | 32 +
.../cache/persistence/pagemem/PageMemoryEx.java | 28 +-
.../persistence/pagemem/PageMemoryImpl.java | 57 +-
.../partstate/PagesAllocationRange.java | 7 +-
.../partstate/PartitionAllocationMap.java | 36 +-
.../snapshot/IgniteCacheSnapshotManager.java | 14 +
.../snapshot/SnapshotDiscoveryMessage.java | 7 +-
.../cache/persistence/tree/BPlusTree.java | 1 +
.../persistence/tree/io/AbstractDataPageIO.java | 1256 ++++++++++++++++++
.../cache/persistence/tree/io/DataPageIO.java | 1214 +----------------
.../cache/persistence/tree/io/PageIO.java | 26 +-
.../tree/io/PagePartitionCountersIO.java | 34 +-
.../tree/io/PagePartitionMetaIO.java | 17 +-
.../persistence/tree/io/SimpleDataPageIO.java | 127 ++
.../wal/AbstractWalRecordsIterator.java | 20 +-
.../wal/FileWriteAheadLogManager.java | 230 +++-
.../wal/reader/IgniteWalIteratorFactory.java | 40 +-
.../wal/reader/StandaloneGridKernalContext.java | 6 +
.../reader/StandaloneWalRecordsIterator.java | 65 +-
.../BaselineTopologyRecordSerializer.java | 168 +++
.../wal/serializer/RecordDataV1Serializer.java | 80 +-
.../wal/serializer/RecordDataV2Serializer.java | 224 +++-
.../wal/serializer/RecordV1Serializer.java | 6 +-
.../wal/serializer/RecordV2Serializer.java | 11 +-
.../wal/serializer/TxRecordSerializer.java | 152 +--
.../store/GridCacheStoreManagerAdapter.java | 6 +
.../cache/transactions/IgniteTxAdapter.java | 73 +-
.../transactions/IgniteTxLocalAdapter.java | 36 +-
.../cache/transactions/IgniteTxManager.java | 5 +
.../CollectConflictPartitionKeysTask.java | 235 ++++
.../cache/verify/PartitionEntryHashRecord.java | 237 ++++
.../cache/verify/PartitionHashRecord.java | 130 ++
.../processors/cache/verify/PartitionKey.java | 103 ++
.../RetrieveConflictPartitionValuesTask.java | 204 +++
.../verify/VerifyBackupPartitionsTask.java | 280 ++++
.../processors/cluster/BaselineTopology.java | 484 +++++++
.../cluster/BaselineTopologyHistory.java | 127 ++
.../cluster/BaselineTopologyHistoryItem.java | 79 ++
.../processors/cluster/BranchingPointType.java | 48 +
.../cluster/ChangeGlobalStateFinishMessage.java | 19 +-
.../cluster/ChangeGlobalStateMessage.java | 38 +-
.../cluster/DiscoveryDataClusterState.java | 83 +-
.../cluster/GridClusterStateProcessor.java | 770 +++++++++--
.../cluster/IGridClusterStateProcessor.java | 122 ++
.../AtomicDataStructureProxy.java | 4 +-
.../datastructures/DataStructuresProcessor.java | 50 +-
.../processors/query/GridQueryProcessor.java | 2 +-
.../cluster/GridChangeStateCommandHandler.java | 4 +-
.../service/GridServiceProcessor.java | 7 +-
.../GridInternalSubscriptionProcessor.java | 57 +
.../util/lang/GridFilteredClosableIterator.java | 37 +
.../util/lang/GridFilteredIterator.java | 2 +-
.../ignite/internal/util/lang/GridFunc.java | 24 +
.../visor/baseline/VisorBaselineNode.java | 89 ++
.../visor/baseline/VisorBaselineOperation.java | 49 +
.../visor/baseline/VisorBaselineTask.java | 224 ++++
.../visor/baseline/VisorBaselineTaskArg.java | 99 ++
.../visor/baseline/VisorBaselineTaskResult.java | 144 ++
.../visor/compute/VisorGatewayTask.java | 6 +-
.../org/apache/ignite/spi/IgniteSpiAdapter.java | 6 +
.../org/apache/ignite/spi/IgniteSpiContext.java | 7 +
.../ignite/spi/discovery/tcp/ServerImpl.java | 19 +-
.../spi/discovery/tcp/TcpDiscoverySpi.java | 20 +-
.../tcp/internal/DiscoveryDataPacket.java | 14 +-
.../tcp/internal/TcpDiscoveryNode.java | 7 +-
.../resources/META-INF/classnames.properties | 74 +-
.../IgniteClientReconnectCacheTest.java | 6 +-
.../GridCacheAbstractRemoveFailureTest.java | 76 +-
.../cache/GridCacheAbstractSelfTest.java | 3 +-
.../processors/cache/GridCacheTestEntryEx.java | 4 +-
.../IgniteClusterActivateDeactivateTest.java | 87 +-
...erActivateDeactivateTestWithPersistence.java | 75 +-
.../distributed/CacheBaselineTopologyTest.java | 711 ++++++++++
.../distributed/CacheExchangeMergeTest.java | 10 +-
.../CacheLateAffinityAssignmentTest.java | 5 +-
.../CachePutAllFailoverAbstractTest.java | 35 +-
.../IgniteOptimisticTxSuspendResumeTest.java | 19 +
...eBaselineAffinityTopologyActivationTest.java | 993 ++++++++++++++
...tePdsBinaryMetadataOnClusterRestartTest.java | 270 +++-
.../IgnitePdsCacheRebalancingAbstractTest.java | 7 +-
...IgnitePersistentStoreDataStructuresTest.java | 2 +
...teAllBaselineNodesOnlineFullApiSelfTest.java | 37 +
.../IgniteBaselineAbstractFullApiSelfTest.java | 50 +
...gingBaselineDownCachePutAllFailoverTest.java | 127 ++
...gingBaselineDownCacheRemoveFailoverTest.java | 157 +++
...angingBaselineUpCachePutAllFailoverTest.java | 122 ++
...angingBaselineUpCacheRemoveFailoverTest.java | 154 +++
...gniteOfflineBaselineNodeFullApiSelfTest.java | 41 +
...eOnlineNodeOutOfBaselineFullApiSelfTest.java | 39 +
...teStableBaselineCachePutAllFailoverTest.java | 104 ++
...teStableBaselineCacheRemoveFailoverTest.java | 130 ++
...gniteCheckpointDirtyPagesForLowLoadTest.java | 175 +++
...ckpointSimulationWithRealCpDisabledTest.java | 72 +-
.../db/wal/IgniteWalFlushFailoverTest.java | 21 +-
...lFlushMultiNodeFailoverAbstractSelfTest.java | 8 +-
.../db/wal/IgniteWalRecoveryTest.java | 284 +++-
.../db/wal/IgniteWalSerializerVersionTest.java | 38 +-
.../persistence/db/wal/WalCompactionTest.java | 8 +-
.../db/wal/WalRecoveryTxLogicalRecordsTest.java | 11 +-
.../db/wal/reader/IgniteWalReaderTest.java | 17 +-
.../persistence/file/FileDownloaderTest.java | 90 ++
.../pagemem/IndexStoragePageMemoryImplTest.java | 104 ++
.../MetadataStoragePageMemoryImplTest.java | 104 --
.../pagemem/NoOpPageStoreManager.java | 5 +
.../persistence/pagemem/NoOpWALManager.java | 2 +-
.../IgniteChangeGlobalStateAbstractTest.java | 1 +
.../IgniteChangeGlobalStateFailOverTest.java | 6 +-
...gniteAbstractStandByClientReconnectTest.java | 2 +
...eStandByClientReconnectToNewClusterTest.java | 6 +-
.../database/CacheFreeListImplSelfTest.java | 565 ++++++++
.../database/FreeListImplSelfTest.java | 565 --------
.../IgniteDbPutGetWithCacheStoreTest.java | 196 +++
.../database/IndexStorageSelfTest.java | 171 +++
.../database/MetadataStorageSelfTest.java | 171 ---
.../testframework/GridSpiTestContext.java | 6 +
.../ignite/testframework/GridTestUtils.java | 9 +
.../junits/common/GridCommonAbstractTest.java | 25 +-
.../multijvm/IgniteClusterProcessProxy.java | 26 +
.../ignite/testsuites/IgniteBasicTestSuite.java | 8 +-
.../IgniteCacheFailoverTestSuite.java | 4 +
.../IgniteCacheFailoverTestSuite2.java | 4 +
.../ignite/testsuites/IgnitePdsTestSuite.java | 6 +-
.../ignite/testsuites/IgnitePdsTestSuite2.java | 11 +
.../testsuites/IgniteStandByClusterSuite.java | 5 +
.../testsuites/IgniteUtilSelfTestSuite.java | 6 +-
.../ignite/util/GridCommandHandlerTest.java | 274 ++++
modules/dev-utils/pom.xml | 26 +
.../development/utils/IgniteWalConverter.java | 32 +-
.../development/utils/RecordSizeCountStat.java | 83 ++
.../ignite/development/utils/TxWalStat.java | 318 +++++
.../ignite/development/utils/WalStat.java | 327 +++++
.../processors/query/h2/opt/GridH2Table.java | 2 -
.../IgniteBinaryObjectFieldsQuerySelfTest.java | 7 +-
...eLockPartitionOnAffinityRunAbstractTest.java | 2 +-
...PartitionOnAffinityRunAtomicCacheOpTest.java | 18 +-
.../IgniteCacheQueryNodeRestartSelfTest.java | 56 +-
...ckPartitionOnAffinityRunAtomicCacheTest.java | 96 ++
...neLockPartitionOnAffinityRunTxCacheTest.java | 87 ++
...ngBaselineCacheQueryNodeRestartSelfTest.java | 125 ++
...StableBaselineBinObjFieldsQuerySelfTest.java | 78 ++
...eBaselineCacheQueryNodeRestartsSelfTest.java | 68 +
.../IgniteCacheAffinityRunTestSuite.java | 4 +
.../IgniteCacheQuerySelfTestSuite.java | 2 +
.../IgniteCacheQuerySelfTestSuite2.java | 4 +
.../ApiParity/ClusterParityTest.cs | 6 +-
.../ApiParity/IgniteConfigurationParityTest.cs | 3 +-
266 files changed, 19765 insertions(+), 4726 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/RELEASE_NOTES.txt
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 27c5767..7af132b 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,6 +1,14 @@
Apache Ignite Release Notes
===========================
+Apache Ignite In-Memory Data Fabric 2.4
+---------------------------------------
+Ignite:
+* Introduced Baseline Affinity Topology
+* Added ability to convert WAL to human-readable form
+* Added support for uninterruptible writes for WAL and storage
+* Local scan performance was improved up to 2 times in certain use-cases
+
Apache Ignite In-Memory Data Fabric 2.3
---------------------------------------
Ignite:
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
index 64a1cbf..b34c921 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
@@ -46,6 +46,9 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
/** */
protected static final String TEST_CACHE_NAME = DummyPersistenceCompatibilityTest.class.getSimpleName();
+ /** */
+ protected volatile boolean compactFooter;
+
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
super.beforeTest();
@@ -59,11 +62,17 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
cfg.setPeerClassLoadingEnabled(false);
- DataStorageConfiguration memCfg = new DataStorageConfiguration()
- .setDefaultDataRegionConfiguration(
- new DataRegionConfiguration().setPersistenceEnabled(true));
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ ));
- cfg.setDataStorageConfiguration(memCfg);
+ cfg.setBinaryConfiguration(
+ new BinaryConfiguration()
+ .setCompactFooter(compactFooter)
+ );
return cfg;
}
@@ -102,7 +111,11 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
* @throws Exception If failed.
*/
protected void doTestStartupWithOldVersion(String igniteVer, boolean compactFooter) throws Exception {
+ boolean prev = this.compactFooter;
+
try {
+ this.compactFooter = compactFooter;
+
startGrid(1, igniteVer, new ConfigurationClosure(compactFooter), new PostStartupClosure());
stopAllGrids();
@@ -113,10 +126,12 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
ignite.active(true);
- validateResultingCacheData(ignite.getOrCreateCache(TEST_CACHE_NAME));
+ validateResultingCacheData(ignite.cache(TEST_CACHE_NAME));
}
finally {
stopAllGrids();
+
+ this.compactFooter = prev;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
----------------------------------------------------------------------
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
index b2c798d..14250c9 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java
@@ -39,6 +39,12 @@ public class MavenUtils {
/** Path to Maven local repository. For caching. */
private static String locRepPath = null;
+ /** */
+ private static final String GG_MVN_REPO = "http://www.gridgainsystems.com/nexus/content/repositories/external";
+
+ /** Set this flag to true if running PDS compatibility tests locally. */
+ private static boolean useGgRepo;
+
/**
* Gets a path to an artifact with given version and groupId=org.apache.ignite and artifactId={@code artifactName}.
* <br>
@@ -123,7 +129,7 @@ public class MavenUtils {
* @throws Exception In case of an error.
*/
private static String defineMavenLocalRepositoryPath() throws Exception {
- String output = exec("mvn help:effective-settings");
+ String output = exec(buildMvnCommand() + " help:effective-settings");
int endTagPos = output.indexOf("</localRepository>");
@@ -141,7 +147,8 @@ public class MavenUtils {
private static void downloadArtifact(String artifact) throws Exception {
X.println("Downloading artifact... Identifier: " + artifact);
- exec("mvn dependency:get -Dartifact=" + artifact);
+ exec(buildMvnCommand() + " dependency:get -Dartifact=" + artifact +
+ (useGgRepo ? " -DremoteRepositories=" + GG_MVN_REPO : ""));
X.println("Download is finished");
}
@@ -193,4 +200,19 @@ public class MavenUtils {
throw e;
}
}
+
+ /**
+ * @return Maven executable command.
+ */
+ private static String buildMvnCommand() {
+ String m2Home = System.getenv("M2_HOME");
+
+ if (m2Home == null)
+ m2Home = System.getProperty("M2_HOME");
+
+ if (m2Home == null)
+ return "mvn";
+
+ return m2Home + "/bin/mvn" ;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/Ignite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java
index c8de43b..4e05463 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignite.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java
@@ -659,7 +659,9 @@ public interface Ignite extends AutoCloseable {
* Checks Ignite grid is active or not active.
*
* @return {@code True} if grid is active. {@code False} If grid is not active.
+ * @deprecated Use {@link IgniteCluster#active()} instead.
*/
+ @Deprecated
public boolean active();
/**
@@ -667,7 +669,9 @@ public interface Ignite extends AutoCloseable {
*
* @param active If {@code True} start activation process. If {@code False} start deactivation process.
* @throws IgniteException If there is an already started transaction or lock in the same thread.
+ * @deprecated Use {@link IgniteCluster#active(boolean)} instead.
*/
+ @Deprecated
public void active(boolean active);
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java b/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
index cb2484f..b80de0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.cluster.BaselineNode;
import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.cluster.ClusterStartNodeResult;
@@ -427,6 +428,44 @@ public interface IgniteCluster extends ClusterGroup, IgniteAsyncSupport {
*/
@Nullable public IgniteFuture<?> clientReconnectFuture();
+ /**
+ * Checks Ignite grid is active or not active.
+ *
+ * @return {@code True} if grid is active. {@code False} If grid is not active.
+ */
+ public boolean active();
+
+ /**
+ * Changes Ignite grid state to active or inactive.
+ *
+ * @param active If {@code True} start activation process. If {@code False} start deactivation process.
+ * @throws IgniteException If there is an already started transaction or lock in the same thread.
+ */
+ public void active(boolean active);
+
+ /**
+ * Gets current baseline topology. If baseline topology was not set, will return {@code null}.
+ *
+ * @return Collection of nodes included to the current baseline topology.
+ */
+ @Nullable public Collection<BaselineNode> currentBaselineTopology();
+
+ /**
+ * Sets baseline topology. The cluster must be activated for this method to be called.
+ *
+ * @param baselineTop A collection of nodes to be included to the baseline topology.
+ */
+ public void setBaselineTopology(Collection<? extends BaselineNode> baselineTop);
+
+ /**
+ * Sets baseline topology constructed from the cluster topology of the given version (the method succeeds
+ * only if the cluster topology has not changed). All client and daemon nodes will be filtered out of the
+ * resulting baseline.
+ *
+ * @param topVer Topology version to set.
+ */
+ public void setBaselineTopology(long topVer);
+
/** {@inheritDoc} */
@Deprecated
@Override public IgniteCluster withAsync();
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 7ffc13d..1bbaacd 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -760,6 +760,13 @@ public final class IgniteSystemProperties {
public static final String IGNITE_WAL_SERIALIZER_VERSION = "IGNITE_WAL_SERIALIZER_VERSION";
/**
+ * If the property is set Ignite will use legacy node comparator (based on node order) inste
+ *
+ * Default value is {@code false}.
+ */
+ public static final String IGNITE_USE_LEGACY_NODE_COMPARATOR = "IGNITE_USE_LEGACY_NODE_COMPARATOR";
+
+ /**
* Property that indicates should be mapped byte buffer used or not.
* Possible values: {@code true} and {@code false}.
*/
@@ -773,6 +780,16 @@ public final class IgniteSystemProperties {
public static final String IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID = "IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID";
/**
+ * Default value is {@code false}.
+ */
+ public static final String IGNITE_WAL_DEBUG_LOG_ON_RECOVERY = "IGNITE_WAL_DEBUG_LOG_ON_RECOVERY";
+
+ /**
+ * Number of checkpoint history entries held in memory.
+ */
+ public static final String IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE = "IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE";
+
+ /**
* If this property is set to {@code true} enable logging in {@link GridClient}.
*/
public static final String IGNITE_GRID_CLIENT_LOG_ENABLED = "IGNITE_GRID_CLIENT_LOG_ENABLED";
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/cluster/BaselineNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cluster/BaselineNode.java b/modules/core/src/main/java/org/apache/ignite/cluster/BaselineNode.java
new file mode 100644
index 0000000..9564aa0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/cluster/BaselineNode.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cluster;
+
+import java.util.Map;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Interface representing a single node from baseline topology.
+ */
+public interface BaselineNode {
+ /**
+ * Gets consistent globally unique node ID. This method returns consistent node ID which
+ * survives node restarts.
+ *
+ * @return Consistent globally unique node ID.
+ */
+ public Object consistentId();
+
+ /**
+ * Gets a node attribute. Attributes are assigned to nodes at startup
+ * via {@link IgniteConfiguration#getUserAttributes()} method.
+ * <p>
+ * The system adds the following attributes automatically:
+ * <ul>
+ * <li>{@code {@link System#getProperties()}} - All system properties.</li>
+ * <li>{@code {@link System#getenv(String)}} - All environment properties.</li>
+ * <li>All attributes defined in {@link org.apache.ignite.internal.IgniteNodeAttributes}</li>
+ * </ul>
+ * <p>
+ * Note that attributes cannot be changed at runtime.
+ *
+ * @param <T> Attribute Type.
+ * @param name Attribute name. <b>Note</b> that attribute names starting with
+ * {@code org.apache.ignite} are reserved for internal use.
+ * @return Attribute value or {@code null}.
+ */
+ @Nullable
+ public <T> T attribute(String name);
+
+ /**
+ * Gets all node attributes. Attributes are assigned to nodes at startup
+ * via {@link IgniteConfiguration#getUserAttributes()} method.
+ * <p>
+ * The system adds the following attributes automatically:
+ * <ul>
+ * <li>{@code {@link System#getProperties()}} - All system properties.</li>
+ * <li>{@code {@link System#getenv(String)}} - All environment properties.</li>
+ * <li>All attributes defined in {@link org.apache.ignite.internal.IgniteNodeAttributes}</li>
+ * </ul>
+ * <p>
+ * Note that attributes cannot be changed at runtime.
+ *
+ * @return All node attributes.
+ */
+ public Map<String, Object> attributes();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java b/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
index e122ff6..97b3165 100644
--- a/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
@@ -101,7 +101,7 @@ import org.jetbrains.annotations.Nullable;
* that comes with JDK as it also provides ability to view any node parameter
* as a graph.
*/
-public interface ClusterNode {
+public interface ClusterNode extends BaselineNode {
/**
* Gets globally unique node ID. A new ID is generated every time a node restarts.
*
@@ -115,7 +115,7 @@ public interface ClusterNode {
*
* @return Consistent globally unique node ID.
*/
- public Object consistentId();
+ @Override public Object consistentId();
/**
* Gets a node attribute. Attributes are assigned to nodes at startup
@@ -135,7 +135,7 @@ public interface ClusterNode {
* {@code org.apache.ignite} are reserved for internal use.
* @return Attribute value or {@code null}.
*/
- @Nullable public <T> T attribute(String name);
+ @Override @Nullable public <T> T attribute(String name);
/**
* Gets metrics snapshot for this node. Note that node metrics are constantly updated
@@ -167,7 +167,7 @@ public interface ClusterNode {
*
* @return All node attributes.
*/
- public Map<String, Object> attributes();
+ @Override public Map<String, Object> attributes();
/**
* Gets collection of addresses this node is known by.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java
index 24cc173..b017fdf 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java
@@ -29,7 +29,7 @@ import static org.apache.ignite.cache.CacheMode.PARTITIONED;
*/
public class AtomicConfiguration {
/** */
- public static final int DFLT_BACKUPS = 0;
+ public static final int DFLT_BACKUPS = 1;
/** */
public static final CacheMode DFLT_CACHE_MODE = PARTITIONED;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java
index 4ae87e3..2950e9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java
@@ -19,6 +19,7 @@ package org.apache.ignite.configuration;
import java.io.Serializable;
import org.apache.ignite.DataRegionMetrics;
import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.mxbean.DataRegionMetricsMXBean;
import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
@@ -431,4 +432,9 @@ public final class DataRegionConfiguration implements Serializable {
return this;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DataRegionConfiguration.class, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
index 381c315..f1439d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
@@ -23,6 +23,7 @@ import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOF
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
/**
@@ -209,7 +210,7 @@ public class DataStorageConfiguration implements Serializable {
private int walTlbSize = DFLT_TLB_SIZE;
/** WAl buffer size. */
- private int walBuffSize/* = DFLT_WAL_BUFF_SIZE*/;
+ private int walBuffSize;
/** Wal flush frequency in milliseconds. */
private long walFlushFreq = DFLT_WAL_FLUSH_FREQ;
@@ -902,4 +903,9 @@ public class DataStorageConfiguration implements Serializable {
return this;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DataStorageConfiguration.class, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index fc1fb6b..7dc4711 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -199,6 +199,9 @@ public class IgniteConfiguration {
/** Default value for active on start flag. */
public static final boolean DFLT_ACTIVE_ON_START = true;
+ /** Default value for auto-activation flag. */
+ public static final boolean DFLT_AUTO_ACTIVATION = true;
+
/** Default failure detection timeout in millis. */
@SuppressWarnings("UnnecessaryBoxing")
public static final Long DFLT_FAILURE_DETECTION_TIMEOUT = new Long(10_000);
@@ -470,6 +473,9 @@ public class IgniteConfiguration {
/** Active on start flag. */
private boolean activeOnStart = DFLT_ACTIVE_ON_START;
+ /** Auto-activation flag. */
+ private boolean autoActivation = DFLT_AUTO_ACTIVATION;
+
/** */
private long longQryWarnTimeout = DFLT_LONG_QRY_WARN_TIMEOUT;
@@ -514,6 +520,7 @@ public class IgniteConfiguration {
addrRslvr = cfg.getAddressResolver();
allResolversPassReq = cfg.isAllSegmentationResolversPassRequired();
atomicCfg = cfg.getAtomicConfiguration();
+ autoActivation = cfg.isAutoActivationEnabled();
binaryCfg = cfg.getBinaryConfiguration();
dsCfg = cfg.getDataStorageConfiguration();
memCfg = cfg.getMemoryConfiguration();
@@ -2275,6 +2282,36 @@ public class IgniteConfiguration {
}
/**
+ * Get the flag indicating that cluster is enabled to activate automatically.
+ *
+ * If it is set to {@code true} and BaselineTopology is set as well than cluster activates automatically
+ * when all nodes from the BaselineTopology join the cluster.
+ *
+ * <p>
+ * Default value is {@link #DFLT_AUTO_ACTIVATION}.
+ * <p>
+ *
+ * @return Auto activation enabled flag value.
+ */
+ public boolean isAutoActivationEnabled() {
+ return autoActivation;
+ }
+
+ /**
+ * Sets flag indicating whether the cluster is enabled to activate automatically.
+ * This value should be the same on all nodes in the cluster.
+ *
+ * @param autoActivation Auto activation enabled flag value.
+ * @return {@code this} instance.
+ * @see #isAutoActivationEnabled()
+ */
+ public IgniteConfiguration setAutoActivationEnabled(boolean autoActivation) {
+ this.autoActivation = autoActivation;
+
+ return this;
+ }
+
+ /**
* Gets flag indicating whether cache sanity check is enabled. If enabled, then Ignite
* will perform the following checks and throw an exception if check fails:
* <ul>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java b/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java
index 93ffe95..1e32e4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridComponent.java
@@ -146,6 +146,9 @@ public interface GridComponent {
*/
@Nullable public IgniteNodeValidationResult validateNode(ClusterNode node);
+ /** */
+ @Nullable public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag.JoiningNodeDiscoveryData discoData);
+
/**
* Gets unique component type to distinguish components providing discovery data. Must return non-null value
* if component implements any of methods {@link #collectJoiningNodeData(DiscoveryDataBag)}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 210b401..ce12b61 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -42,8 +42,8 @@ import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
-import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
import org.apache.ignite.internal.processors.hadoop.HadoopHelper;
+import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
import org.apache.ignite.internal.processors.igfs.IgfsHelper;
import org.apache.ignite.internal.processors.igfs.IgfsProcessorAdapter;
import org.apache.ignite.internal.processors.job.GridJobProcessor;
@@ -62,6 +62,7 @@ import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
import org.apache.ignite.internal.processors.segmentation.GridSegmentationProcessor;
import org.apache.ignite.internal.processors.service.GridServiceProcessor;
import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
+import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
import org.apache.ignite.internal.processors.task.GridTaskProcessor;
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
@@ -649,4 +650,9 @@ public interface GridKernalContext extends Iterable<GridComponent> {
* @return PDS mode folder name resolver, also generates consistent ID in case new folder naming is used
*/
public PdsFoldersResolver pdsFolderResolver();
+
+ /**
+ * @return subscription processor to manage internal-only (strict node-local) subscriptions between components.
+ */
+ public GridInternalSubscriptionProcessor internalSubscriptionProcessor();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 1f0292c..36c6231 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -79,6 +79,7 @@ import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
import org.apache.ignite.internal.processors.segmentation.GridSegmentationProcessor;
import org.apache.ignite.internal.processors.service.GridServiceProcessor;
import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
+import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
import org.apache.ignite.internal.processors.task.GridTaskProcessor;
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
@@ -381,6 +382,9 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
/** PDS mode folder name resolver, also generates consistent ID in case new folder naming is used */
private PdsFoldersResolver pdsFolderRslvr;
+ /** */
+ private GridInternalSubscriptionProcessor internalSubscriptionProc;
+
/**
* No-arg constructor is required by externalization.
*/
@@ -585,6 +589,8 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
mappingProc = (GridMarshallerMappingProcessor)comp;
else if (comp instanceof PdsFoldersResolver)
pdsFolderRslvr = (PdsFoldersResolver)comp;
+ else if (comp instanceof GridInternalSubscriptionProcessor)
+ internalSubscriptionProc = (GridInternalSubscriptionProcessor)comp;
else if (!(comp instanceof DiscoveryNodeValidationProcessor
|| comp instanceof PlatformPluginProcessor))
assert (comp instanceof GridPluginComponent) : "Unknown manager class: " + comp.getClass();
@@ -1068,6 +1074,11 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
return platformProc;
}
+ /** {@inheritDoc} */
+ @Override public GridInternalSubscriptionProcessor internalSubscriptionProcessor() {
+ return internalSubscriptionProc;
+ }
+
/**
* @param disconnected Disconnected flag.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/GridNodeOrderComparator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridNodeOrderComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/GridNodeOrderComparator.java
deleted file mode 100644
index 1c60e07..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridNodeOrderComparator.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal;
-
-import java.io.Serializable;
-import java.util.Comparator;
-import org.apache.ignite.cluster.ClusterNode;
-
-/**
- * Node order comparator.
- */
-public class GridNodeOrderComparator implements Comparator<ClusterNode>, Serializable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- public static final Comparator<ClusterNode> INSTANCE = new GridNodeOrderComparator();
-
- /**
- * Private constructor. Don't create this class, use {@link #INSTANCE}.
- */
- private GridNodeOrderComparator() {
-
- }
-
- /** {@inheritDoc} */
- @Override public int compare(ClusterNode n1, ClusterNode n2) {
- return n1.order() < n2.order() ? -1 : n1.order() > n2.order() ? 1 : n1.id().compareTo(n2.id());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java b/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java
index fd59d24..1e49e5c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridPluginComponent.java
@@ -118,6 +118,11 @@ public class GridPluginComponent implements GridComponent {
}
/** {@inheritDoc} */
+ @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, JoiningNodeDiscoveryData discoData) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public void printMemoryStats() {
// No-op.
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index c62d59c..3bd2a15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -83,6 +83,7 @@ import org.apache.ignite.Ignition;
import org.apache.ignite.MemoryMetrics;
import org.apache.ignite.PersistenceMetrics;
import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cluster.BaselineNode;
import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
@@ -125,6 +126,7 @@ import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsi
import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
import org.apache.ignite.internal.processors.closure.GridClosureProcessor;
import org.apache.ignite.internal.processors.cluster.ClusterProcessor;
+import org.apache.ignite.internal.processors.cluster.IGridClusterStateProcessor;
import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
@@ -152,6 +154,7 @@ import org.apache.ignite.internal.processors.security.GridSecurityProcessor;
import org.apache.ignite.internal.processors.segmentation.GridSegmentationProcessor;
import org.apache.ignite.internal.processors.service.GridServiceProcessor;
import org.apache.ignite.internal.processors.session.GridTaskSessionProcessor;
+import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
import org.apache.ignite.internal.processors.task.GridTaskProcessor;
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
@@ -771,6 +774,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
// Ack various information.
ackAsciiLogo();
ackConfigUrl();
+ ackConfiguration(cfg);
ackDaemon();
ackOsInfo();
ackLanguageRuntime();
@@ -835,6 +839,10 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
cfg.getMarshaller().setContext(ctx.marshallerContext());
+ GridInternalSubscriptionProcessor subscriptionProc = new GridInternalSubscriptionProcessor(ctx);
+
+ startProcessor(subscriptionProc);
+
ClusterProcessor clusterProc = new ClusterProcessor(ctx);
startProcessor(clusterProc);
@@ -916,7 +924,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
startProcessor(new GridAffinityProcessor(ctx));
startProcessor(createComponent(GridSegmentationProcessor.class, ctx));
startProcessor(createComponent(IgniteCacheObjectProcessor.class, ctx));
- startProcessor(new GridClusterStateProcessor(ctx));
+ startProcessor(createComponent(IGridClusterStateProcessor.class, ctx));
startProcessor(new GridCacheProcessor(ctx));
startProcessor(new GridQueryProcessor(ctx));
startProcessor(new ClientListenerProcessor(ctx));
@@ -1763,6 +1771,17 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
log.info("Config URL: " + System.getProperty(IGNITE_CONFIG_URL, "n/a"));
}
+
+ /**
+ * Acks configuration.
+ */
+ private void ackConfiguration(IgniteConfiguration cfg) {
+ assert log != null;
+
+ if (log.isInfoEnabled())
+ log.info(cfg.toString());
+ }
+
/**
* Acks Logger configuration.
*/
@@ -3300,7 +3319,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
guard();
try {
- return context().state().publicApiActiveState();
+ return context().state().publicApiActiveState(true);
}
finally {
unguard();
@@ -3309,17 +3328,19 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
/** {@inheritDoc} */
@Override public void active(boolean active) {
- guard();
+ cluster().active(active);
+ }
- try {
- context().state().changeGlobalState(active).get();
- }
- catch (IgniteCheckedException e) {
- throw U.convertException(e);
- }
- finally {
- unguard();
- }
+ /** */
+ private Collection<BaselineNode> baselineNodes() {
+ Collection<ClusterNode> srvNodes = cluster().forServers().nodes();
+
+ ArrayList baselineNodes = new ArrayList(srvNodes.size());
+
+ for (ClusterNode clN : srvNodes)
+ baselineNodes.add(clN);
+
+ return baselineNodes;
}
/** {@inheritDoc} */
@@ -3614,7 +3635,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
* @throws IgniteException if cluster in inActive state
*/
private void checkClusterState() throws IgniteException {
- if (!ctx.state().publicApiActiveState()) {
+ if (!ctx.state().publicApiActiveState(true)) {
throw new IgniteException("Can not perform the operation because the cluster is inactive. Note, that " +
"the cluster is considered inactive by default if Ignite Persistent Store is used to let all the nodes " +
"join the cluster. To activate the cluster call Ignite.active(true).");
@@ -3792,6 +3813,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
if (cls.equals(DiscoveryNodeValidationProcessor.class))
return (T)new OsDiscoveryNodeValidationProcessor(ctx);
+ if (cls.equals(IGridClusterStateProcessor.class))
+ return (T)new GridClusterStateProcessor(ctx);
+
Class<T> implCls = null;
try {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/cluster/DetachedClusterNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/DetachedClusterNode.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/DetachedClusterNode.java
new file mode 100644
index 0000000..2c72bb0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/DetachedClusterNode.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.cluster;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Representation of cluster node that isn't currently present in cluster.
+ */
+public class DetachedClusterNode implements ClusterNode {
+ /** */
+ @GridToStringExclude
+ private final UUID uuid = UUID.randomUUID();
+
+ /** Consistent ID. */
+ @GridToStringInclude
+ private final Object consistentId;
+
+ /** Node attributes. */
+ @GridToStringInclude
+ private final Map<String, Object> attributes;
+
+ /**
+ * @param consistentId Consistent ID.
+ * @param attributes Node attributes.
+ */
+ public DetachedClusterNode(Object consistentId, Map<String, Object> attributes) {
+ this.consistentId = consistentId;
+ this.attributes = attributes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public UUID id() {
+ return uuid;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object consistentId() {
+ return consistentId;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public <T> T attribute(String name) {
+ return (T)attributes.get(name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public ClusterMetrics metrics() {
+ throw new UnsupportedOperationException("Operation is not supported on DetachedClusterNode");
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, Object> attributes() {
+ return attributes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<String> addresses() {
+ throw new UnsupportedOperationException("Operation is not supported on DetachedClusterNode");
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<String> hostNames() {
+ throw new UnsupportedOperationException("Operation is not supported on DetachedClusterNode");
+ }
+
+ /** {@inheritDoc} */
+ @Override public long order() {
+ throw new UnsupportedOperationException("Operation is not supported on DetachedClusterNode");
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteProductVersion version() {
+ throw new UnsupportedOperationException("Operation is not supported on DetachedClusterNode");
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isLocal() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDaemon() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isClient() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DetachedClusterNode.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
index f2c14eb..e238cb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
@@ -30,6 +30,7 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteCluster;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.BaselineNode;
import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
@@ -306,6 +307,31 @@ public class IgniteClusterAsyncImpl extends AsyncSupportAdapter<IgniteCluster>
}
/** {@inheritDoc} */
+ @Override public boolean active() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void active(boolean active) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Collection<BaselineNode> currentBaselineTopology() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setBaselineTopology(Collection<? extends BaselineNode> baselineTop) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setBaselineTopology(long topVer) {
+
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public IgniteFuture<?> clientReconnectFuture() {
return cluster.clientReconnectFuture();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
index 0c95dc2..392b43c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
@@ -30,6 +30,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
@@ -37,6 +38,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteCluster;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.BaselineNode;
import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.cluster.ClusterGroupEmptyException;
import org.apache.ignite.cluster.ClusterNode;
@@ -45,6 +47,7 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteComponentType;
import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.IgniteFutureImpl;
@@ -284,6 +287,175 @@ public class IgniteClusterImpl extends ClusterGroupAdapter implements IgniteClus
}
/** {@inheritDoc} */
+ @Override public boolean active() {
+ guard();
+
+ try {
+ return ctx.state().publicApiActiveState(true);
+ }
+ finally {
+ unguard();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void active(boolean active) {
+ guard();
+
+ try {
+ ctx.state().changeGlobalState(active, baselineNodes(), false).get();
+ }
+ catch (IgniteCheckedException e) {
+ throw U.convertException(e);
+ }
+ finally {
+ unguard();
+ }
+ }
+
+ /** */
+ private Collection<BaselineNode> baselineNodes() {
+ Collection<ClusterNode> srvNodes = ctx.cluster().get().forServers().nodes();
+
+ ArrayList baselineNodes = new ArrayList(srvNodes.size());
+
+ for (ClusterNode clN : srvNodes)
+ baselineNodes.add(clN);
+
+ return baselineNodes;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Collection<BaselineNode> currentBaselineTopology() {
+ guard();
+
+ try {
+ BaselineTopology blt = ctx.state().clusterState().baselineTopology();
+
+ return blt != null ? blt.currentBaseline() : null;
+ }
+ finally {
+ unguard();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setBaselineTopology(Collection<? extends BaselineNode> baselineTop) {
+ guard();
+
+ try {
+ if (isInMemoryMode())
+ return;
+
+ validateBeforeBaselineChange(baselineTop);
+
+ ctx.state().changeGlobalState(true, baselineTop, true).get();
+ }
+ catch (IgniteCheckedException e) {
+ throw U.convertException(e);
+ }
+ finally {
+ unguard();
+ }
+ }
+
+ /** */
+ private boolean isInMemoryMode() {
+ return !CU.isPersistenceEnabled(cfg);
+ }
+
+ /**
+ * Executes validation checks of cluster state and BaselineTopology before changing BaselineTopology to new one.
+ */
+ private void validateBeforeBaselineChange(Collection<? extends BaselineNode> baselineTop) {
+ if (!ctx.state().clusterState().active())
+ throw new IgniteException("Changing BaselineTopology on inactive cluster is not allowed.");
+
+ if (baselineTop != null) {
+ if (baselineTop.isEmpty())
+ throw new IgniteException("BaselineTopology must contain at least one node.");
+
+ Collection<Object> onlineNodes = onlineBaselineNodesRequestedForRemoval(baselineTop);
+
+ if (onlineNodes != null) {
+ if (!onlineNodes.isEmpty())
+ throw new IgniteException("Removing online nodes from BaselineTopology is not supported: " + onlineNodes);
+ }
+ else
+ //should never happen, actually: if cluster was activated, we expect it to have a BaselineTopology.
+ throw new IgniteException("Previous BaselineTopology was not found.");
+ }
+ }
+
+ /** */
+ @Nullable private Collection<Object> onlineBaselineNodesRequestedForRemoval(Collection<? extends BaselineNode> newBlt) {
+ BaselineTopology blt = ctx.state().clusterState().baselineTopology();
+ Set<Object> bltConsIds;
+
+ if (blt == null)
+ return null;
+ else
+ bltConsIds = blt.consistentIds();
+
+ ArrayList<Object> onlineNodesRequestedForRemoval = new ArrayList<>();
+
+ Collection<Object> aliveNodesConsIds = getConsistentIds(ctx.discovery().aliveServerNodes());
+
+ Collection<Object> newBltConsIds = getConsistentIds(newBlt);
+
+ for (Object oldBltConsId : bltConsIds) {
+ if (aliveNodesConsIds.contains(oldBltConsId)) {
+ if (!newBltConsIds.contains(oldBltConsId))
+ onlineNodesRequestedForRemoval.add(oldBltConsId);
+ }
+ }
+
+ return onlineNodesRequestedForRemoval;
+ }
+
+ /** */
+ private Collection<Object> getConsistentIds(Collection<? extends BaselineNode> nodes) {
+ ArrayList<Object> res = new ArrayList<>(nodes.size());
+
+ for (BaselineNode n : nodes)
+ res.add(n.consistentId());
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setBaselineTopology(long topVer) {
+ guard();
+
+ try {
+ if (isInMemoryMode())
+ return;
+
+ Collection<ClusterNode> top = topology(topVer);
+
+ if (top == null)
+ throw new IgniteException("Topology version does not exist: " + topVer);
+
+ Collection<BaselineNode> target = new ArrayList<>(top.size());
+
+ for (ClusterNode node : top) {
+ if (!node.isClient())
+ target.add(node);
+ }
+
+ validateBeforeBaselineChange(target);
+
+ ctx.state().changeGlobalState(true, target, true).get();
+ }
+ catch (IgniteCheckedException e) {
+ throw U.convertException(e);
+ }
+ finally {
+ unguard();
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public void enableStatistics(Collection<String> caches, boolean enabled) {
guard();
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderComparator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderComparator.java
new file mode 100644
index 0000000..a493850
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderComparator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.cluster;
+
+import java.io.Serializable;
+import java.util.Comparator;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ * Node order comparator.
+ */
+public class NodeOrderComparator implements Comparator<ClusterNode>, Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private static final Comparator<ClusterNode> INSTANCE = new NodeOrderComparator();
+
+ /**
+ * @return Node comparator.
+ */
+ public static Comparator<ClusterNode> getInstance() {
+ return IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_LEGACY_NODE_COMPARATOR) ?
+ NodeOrderLegacyComparator.INSTANCE : INSTANCE;
+ }
+
+ /**
+ * Private constructor. Don't create this class, use {@link #getInstance()}.
+ */
+ private NodeOrderComparator() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compare(ClusterNode n1, ClusterNode n2) {
+ Object id1 = n1.consistentId();
+ Object id2 = n2.consistentId();
+
+ if (id1 instanceof Comparable && id2 instanceof Comparable && id1.getClass().equals(id2.getClass()))
+ return ((Comparable)id1).compareTo(id2);
+
+ return id1.toString().compareTo(id2.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderLegacyComparator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderLegacyComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderLegacyComparator.java
new file mode 100644
index 0000000..18a6ac3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/NodeOrderLegacyComparator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.cluster;
+
+import java.io.Serializable;
+import java.util.Comparator;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ * Node order comparator.
+ */
+public class NodeOrderLegacyComparator implements Comparator<ClusterNode>, Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ public static final Comparator<ClusterNode> INSTANCE = new NodeOrderLegacyComparator();
+
+ /**
+ * Private constructor. Don't create this class, use {@link #INSTANCE}.
+ */
+ private NodeOrderLegacyComparator() {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compare(ClusterNode n1, ClusterNode n2) {
+ return n1.order() < n2.order() ? -1 : n1.order() > n2.order() ? 1 : n1.id().compareTo(n2.id());
+ }
+}
\ No newline at end of file
[15/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
index 8ea2981..ed469a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
@@ -19,7 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -68,7 +69,7 @@ public class DataPageUpdateRecord extends PageDeltaRecord {
@Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
assert payload != null;
- DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+ AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
io.updateRow(pageAddr, itemId, pageMem.pageSize(), payload, null, 0);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java
index d6f78ab..cda6850 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/GridProcessorAdapter.java
@@ -152,6 +152,11 @@ public abstract class GridProcessorAdapter implements GridProcessor {
}
/** {@inheritDoc} */
+ @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag.JoiningNodeDiscoveryData discoData) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return S.toString(GridProcessorAdapter.class, this);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index c451b77..dd46246 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.affinity;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -38,9 +39,10 @@ import org.apache.ignite.cache.affinity.AffinityFunction;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.GridNodeOrderComparator;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -48,7 +50,6 @@ import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgnitePredicate;
import org.jetbrains.annotations.Nullable;
-import org.jsr166.ConcurrentHashMap8;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_AFFINITY_HISTORY_SIZE;
import static org.apache.ignite.IgniteSystemProperties.getInteger;
@@ -85,6 +86,12 @@ public class GridAffinityAssignmentCache {
/** */
private List<List<ClusterNode>> idealAssignment;
+ /** */
+ private BaselineTopology baselineTopology;
+
+ /** */
+ private List<List<ClusterNode>> baselineAssignment;
+
/** Cache item corresponding to the head topology version. */
private final AtomicReference<GridAffinityAssignment> head;
@@ -260,8 +267,8 @@ public class GridAffinityAssignmentCache {
* @return Affinity assignments.
*/
@SuppressWarnings("IfMayBeConditional")
- public List<List<ClusterNode>> calculate(AffinityTopologyVersion topVer, DiscoveryEvent discoEvt,
- DiscoCache discoCache) {
+ public List<List<ClusterNode>> calculate(AffinityTopologyVersion topVer, @Nullable DiscoveryEvent discoEvt,
+ @Nullable DiscoCache discoCache) {
if (log.isDebugEnabled())
log.debug("Calculating affinity [topVer=" + topVer + ", locNodeId=" + ctx.localNodeId() +
", discoEvt=" + discoEvt + ']');
@@ -274,11 +281,20 @@ public class GridAffinityAssignmentCache {
if (!locCache) {
sorted = new ArrayList<>(discoCache.cacheGroupAffinityNodes(groupId()));
- Collections.sort(sorted, GridNodeOrderComparator.INSTANCE);
+ Collections.sort(sorted, NodeOrderComparator.getInstance());
}
else
sorted = Collections.singletonList(ctx.discovery().localNode());
+ boolean hasBaseline = false;
+ boolean changedBaseline = false;
+
+ if (discoCache != null) {
+ hasBaseline = discoCache.state().baselineTopology() != null;
+ changedBaseline = !hasBaseline ? baselineTopology != null :
+ !discoCache.state().baselineTopology().equals(baselineTopology);
+ }
+
List<List<ClusterNode>> assignment;
if (prevAssignment != null && discoEvt != null) {
@@ -286,18 +302,51 @@ public class GridAffinityAssignmentCache {
if (!affNode)
assignment = prevAssignment;
+ else if (hasBaseline && !changedBaseline) {
+ if (baselineAssignment == null)
+ baselineAssignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(
+ discoCache.state().baselineTopology().createBaselineView(sorted, nodeFilter),
+ prevAssignment, discoEvt, topVer, backups));
+
+ assignment = currentBaselineAssignment(topVer);
+ }
+ else if (hasBaseline && changedBaseline) {
+ baselineAssignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(
+ discoCache.state().baselineTopology().createBaselineView(sorted, nodeFilter),
+ prevAssignment, discoEvt, topVer, backups));
+
+ assignment = currentBaselineAssignment(topVer);
+ }
+ else
+ assignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(sorted, prevAssignment,
+ discoEvt, topVer, backups));
+ }
+ else {
+ if (hasBaseline) {
+ baselineAssignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(
+ discoCache.state().baselineTopology().createBaselineView(sorted, nodeFilter),
+ prevAssignment, discoEvt, topVer, backups));
+
+ assignment = currentBaselineAssignment(topVer);
+ }
else
assignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(sorted, prevAssignment,
discoEvt, topVer, backups));
}
- else
- assignment = aff.assignPartitions(new GridAffinityFunctionContextImpl(sorted, prevAssignment, discoEvt,
- topVer, backups));
assert assignment != null;
idealAssignment = assignment;
+ if (hasBaseline) {
+ baselineTopology = discoCache.state().baselineTopology();
+ assert baselineAssignment != null;
+ }
+ else {
+ baselineTopology = null;
+ baselineAssignment = null;
+ }
+
if (locCache)
initialize(topVer, assignment);
@@ -305,6 +354,41 @@ public class GridAffinityAssignmentCache {
}
/**
+ * @param topVer Topology version.
+ * @return Baseline assignment with filtered out offline nodes.
+ */
+ private List<List<ClusterNode>> currentBaselineAssignment(AffinityTopologyVersion topVer) {
+ Map<Object, ClusterNode> alives = new HashMap<>();
+
+ for (ClusterNode node : ctx.discovery().nodes(topVer)) {
+ if (!node.isClient() && !node.isDaemon())
+ alives.put(node.consistentId(), node);
+ }
+
+ List<List<ClusterNode>> result = new ArrayList<>(baselineAssignment.size());
+
+ for (int p = 0; p < baselineAssignment.size(); p++) {
+ List<ClusterNode> baselineMapping = baselineAssignment.get(p);
+ List<ClusterNode> currentMapping = null;
+
+ for (ClusterNode node : baselineMapping) {
+ ClusterNode aliveNode = alives.get(node.consistentId());
+
+ if (aliveNode != null) {
+ if (currentMapping == null)
+ currentMapping = new ArrayList<>();
+
+ currentMapping.add(aliveNode);
+ }
+ }
+
+ result.add(p, currentMapping != null ? currentMapping : Collections.<ClusterNode>emptyList());
+ }
+
+ return result;
+ }
+
+ /**
* Copies previous affinity assignment when discovery event does not cause affinity assignment changes
* (e.g. client node joins on leaves).
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 6347d03..63aba98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -126,9 +126,6 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
/** */
private final ThreadLocal<ClientCacheChangeDiscoveryMessage> clientCacheChanges = new ThreadLocal<>();
- /** Caches initialized flag (initialized when join activate cluster or after activation. */
- private boolean cachesInitialized;
-
/** Discovery listener. */
private final GridLocalEventListener discoLsnr = new GridLocalEventListener() {
@Override public void onEvent(Event evt) {
@@ -167,24 +164,9 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
if (state.transition() || !state.active())
return;
- if (type == EVT_NODE_JOINED && node.isLocal()) {
- // Clean-up in case of client reconnect.
- caches.clear();
-
+ if (type == EVT_NODE_JOINED && node.isLocal())
lastAffVer = null;
- caches.init(cctx.cache().cacheGroupDescriptors(), cctx.cache().cacheDescriptors());
-
- cachesInitialized = true;
- }
- else if (customMsg instanceof ChangeGlobalStateFinishMessage) {
- if (!cachesInitialized && ((ChangeGlobalStateFinishMessage)customMsg).clusterActive()) {
- caches.init(cctx.cache().cacheGroupDescriptors(), cctx.cache().cacheDescriptors());
-
- cachesInitialized = true;
- }
- }
-
if (!CU.clientNode(node) && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) {
synchronized (mux) {
assert lastAffVer == null || topVer.compareTo(lastAffVer) > 0;
@@ -195,6 +177,19 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
}
/**
+ * Must be called from exchange thread.
+ */
+ public void initCachesOnLocalJoin(
+ Map<Integer, CacheGroupDescriptor> cacheGroupDescriptors,
+ Map<String, DynamicCacheDescriptor> cacheDescriptors
+ ) {
+ // Clean-up in case of client reconnect.
+ caches.clear();
+
+ caches.init(cacheGroupDescriptors, cacheDescriptors);
+ }
+
+ /**
* Callback invoked from discovery thread when discovery custom message is received.
*
* @param msg Customer message.
@@ -319,6 +314,18 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
}
/**
+ * @return Group IDs.
+ */
+ public Set<Integer> waitGroups() {
+ synchronized (mux) {
+ if (waitInfo == null || !waitInfo.topVer.equals(lastAffVer) )
+ return Collections.emptySet();
+
+ return new HashSet<>(waitInfo.waitGrps.keySet());
+ }
+ }
+
+ /**
* @param waitInfo Cache rebalance information.
* @return Message.
*/
@@ -706,6 +713,20 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
}
/**
+ * @param cctx Stopped cache context.
+ */
+ public void stopCacheOnReconnect(GridCacheContext cctx) {
+ caches.registeredCaches.remove(cctx.cacheId());
+ }
+
+ /**
+ * @param grpCtx Stopped cache group context.
+ */
+ public void stopCacheGroupOnReconnect(CacheGroupContext grpCtx) {
+ caches.registeredGrps.remove(grpCtx.groupId());
+ }
+
+ /**
* Called on exchange initiated for cache start/stop request.
*
* @param fut Exchange future.
@@ -1412,7 +1433,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
* @param crd Coordinator flag.
* @throws IgniteCheckedException If failed.
*/
- public void onServerJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException {
+ public void onServerJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd)
+ throws IgniteCheckedException {
assert !fut.firstEvent().eventNode().isClient();
boolean locJoin = fut.firstEvent().eventNode().isLocal();
@@ -1450,6 +1472,28 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
}
/**
+ * @param fut Exchange future
+ * @param crd Coordinator flag.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void onBaselineTopologyChanged(final GridDhtPartitionsExchangeFuture fut, boolean crd) throws IgniteCheckedException {
+ assert !fut.firstEvent().eventNode().isClient();
+
+ WaitRebalanceInfo waitRebalanceInfo = initAffinityOnNodeJoin(fut, crd);
+
+ this.waitInfo = waitRebalanceInfo != null && !waitRebalanceInfo.empty() ? waitRebalanceInfo : null;
+
+ WaitRebalanceInfo info = this.waitInfo;
+
+ if (crd) {
+ if (log.isDebugEnabled()) {
+ log.debug("Computed new affinity after node join [topVer=" + fut.initialVersion() +
+ ", waitGrps=" + (info != null ? groupNames(info.waitGrps.keySet()) : null) + ']');
+ }
+ }
+ }
+
+ /**
* @param grpIds Cache group IDs.
* @return Cache names.
*/
@@ -1892,7 +1936,6 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
* @param rebalanceInfo Rebalance information.
* @param latePrimary If {@code true} delays primary assignment if it is not owner.
* @param affCache Already calculated assignments (to reduce data stored in history).
- * @throws IgniteCheckedException If failed.
*/
private void initAffinityOnNodeJoin(
ExchangeDiscoveryEvents evts,
@@ -1900,9 +1943,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
GridAffinityAssignmentCache aff,
WaitRebalanceInfo rebalanceInfo,
boolean latePrimary,
- Map<Object, List<List<ClusterNode>>> affCache)
- throws IgniteCheckedException
- {
+ Map<Object, List<List<ClusterNode>>> affCache
+ ) {
if (addedOnExchnage) {
if (!aff.lastVersion().equals(evts.topologyVersion()))
calculateAndInit(evts, aff, evts.topologyVersion());
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
index ecfa8bc..6d2688c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
@@ -51,7 +51,7 @@ public class CacheJoinNodeDiscoveryData implements Serializable {
* @param templates Templates.
* @param startCaches {@code True} if required to start all caches on joining node.
*/
- public CacheJoinNodeDiscoveryData(
+ public CacheJoinNodeDiscoveryData(
IgniteUuid cacheDeploymentId,
Map<String, CacheJoinNodeDiscoveryData.CacheInfo> caches,
Map<String, CacheJoinNodeDiscoveryData.CacheInfo> templates,
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 0c4fc68..3b5a3a5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -99,7 +99,7 @@ class ClusterCachesInfo {
private GridData gridData;
/** */
- private List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> locJoinStartCaches = Collections.emptyList();
+ private LocalJoinCachesContext locJoinCachesCtx;
/** */
private Map<String, T2<CacheConfiguration, NearCacheConfiguration>> locCfgsForActivation = Collections.emptyMap();
@@ -805,13 +805,13 @@ class ClusterCachesInfo {
*
* @return Caches to be started when this node starts.
*/
- @NotNull public List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> cachesToStartOnLocalJoin() {
+ @Nullable public LocalJoinCachesContext localJoinCachesContext() {
if (ctx.isDaemon())
- return Collections.emptyList();
+ return null;
- List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> result = locJoinStartCaches;
+ LocalJoinCachesContext result = locJoinCachesCtx;
- locJoinStartCaches = Collections.emptyList();
+ locJoinCachesCtx = null;
return result;
}
@@ -888,7 +888,7 @@ class ClusterCachesInfo {
if (gridData == null) { // First node starts.
assert joinDiscoData != null;
- initStartCachesForLocalJoin(true);
+ initStartCachesForLocalJoin(true, false);
}
}
}
@@ -1090,10 +1090,8 @@ class ClusterCachesInfo {
gridData = new GridData(joinDiscoData, cachesData, conflictErr);
- if (!disconnectedState())
- initStartCachesForLocalJoin(false);
- else
- locJoinStartCaches = Collections.emptyList();
+ if (cachesOnDisconnect == null || cachesOnDisconnect.clusterActive())
+ initStartCachesForLocalJoin(false, disconnectedState());
}
/**
@@ -1102,8 +1100,8 @@ class ClusterCachesInfo {
*
* @param firstNode {@code True} if first node in cluster starts.
*/
- private void initStartCachesForLocalJoin(boolean firstNode) {
- assert F.isEmpty(locJoinStartCaches) : locJoinStartCaches;
+ private void initStartCachesForLocalJoin(boolean firstNode, boolean reconnect) {
+ assert locJoinCachesCtx == null : locJoinCachesCtx;
if (ctx.state().clusterState().transition()) {
joinOnTransition = true;
@@ -1112,7 +1110,7 @@ class ClusterCachesInfo {
}
if (joinDiscoData != null) {
- locJoinStartCaches = new ArrayList<>();
+ List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> locJoinStartCaches = new ArrayList<>();
locCfgsForActivation = new HashMap<>();
boolean active = ctx.state().clusterState().active();
@@ -1123,6 +1121,9 @@ class ClusterCachesInfo {
CacheConfiguration<?, ?> cfg = desc.cacheConfiguration();
+ if (reconnect && surviveReconnect(cfg.getName()) && cachesOnDisconnect.state.active() && active)
+ continue;
+
CacheJoinNodeDiscoveryData.CacheInfo locCfg = joinDiscoData.caches().get(cfg.getName());
NearCacheConfiguration nearCfg = null;
@@ -1159,6 +1160,11 @@ class ClusterCachesInfo {
locCfgsForActivation.put(desc.cacheName(), new T2<>(desc.cacheConfiguration(), nearCfg));
}
}
+
+ locJoinCachesCtx = new LocalJoinCachesContext(
+ locJoinStartCaches,
+ new HashMap<>(registeredCacheGrps),
+ new HashMap<>(registeredCaches));
}
}
@@ -1167,7 +1173,7 @@ class ClusterCachesInfo {
*/
public void onStateChangeFinish(ChangeGlobalStateFinishMessage msg) {
if (joinOnTransition) {
- initStartCachesForLocalJoin(false);
+ initStartCachesForLocalJoin(false, false);
joinOnTransition = false;
}
@@ -1179,10 +1185,13 @@ class ClusterCachesInfo {
* @return Exchange action.
* @throws IgniteCheckedException If configuration validation failed.
*/
- public ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopologyVersion topVer)
+ public ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopologyVersion topVer, DiscoveryDataClusterState curState)
throws IgniteCheckedException {
ExchangeActions exchangeActions = new ExchangeActions();
+ if (msg.activate() == curState.active())
+ return exchangeActions;
+
if (msg.activate()) {
for (DynamicCacheDescriptor desc : orderedCaches(CacheComparators.DIRECT)) {
desc.startTopologyVersion(topVer);
@@ -1657,6 +1666,9 @@ class ClusterCachesInfo {
Set<String> stoppedCaches = new HashSet<>();
Set<Integer> stoppedCacheGrps = new HashSet<>();
+ Set<String> survivedCaches = new HashSet<>();
+ Set<Integer> survivedCacheGrps = new HashSet<>();
+
if (!active) {
joinOnTransition = transition;
@@ -1698,8 +1710,11 @@ class ClusterCachesInfo {
if (stopped)
stoppedCacheGrps.add(locDesc.groupId());
- else
+ else {
assert locDesc.groupId() == desc.groupId();
+
+ survivedCacheGrps.add(locDesc.groupId());
+ }
}
for (Map.Entry<String, DynamicCacheDescriptor> e : cachesOnDisconnect.caches.entrySet()) {
@@ -1719,10 +1734,20 @@ class ClusterCachesInfo {
if (stopped)
stoppedCaches.add(cacheName);
+ else
+ survivedCaches.add(cacheName);
+ }
+
+ if (locJoinCachesCtx != null) {
+ locJoinCachesCtx.removeSurvivedCacheGroups(survivedCacheGrps);
+ locJoinCachesCtx.removeSurvivedCaches(survivedCaches);
+
+ if (locJoinCachesCtx.isEmpty())
+ locJoinCachesCtx = null;
}
if (!cachesOnDisconnect.clusterActive())
- initStartCachesForLocalJoin(false);
+ initStartCachesForLocalJoin(false, true);
}
if (clientReconnectReqs != null) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
index 91ad003..d84f8d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
@@ -50,6 +50,9 @@ public class ExchangeActions {
private Map<String, CacheActionData> cachesToResetLostParts;
/** */
+ private LocalJoinCachesContext locJoinCtx;
+
+ /** */
private StateChangeRequest stateChangeReq;
/**
@@ -194,14 +197,21 @@ public class ExchangeActions {
* @return {@code True} if has deactivate request.
*/
public boolean deactivate() {
- return stateChangeReq != null && !stateChangeReq.activate();
+ return stateChangeReq != null && stateChangeReq.activeChanged() && !stateChangeReq.activate();
}
/**
* @return {@code True} if has activate request.
*/
public boolean activate() {
- return stateChangeReq != null && stateChangeReq.activate();
+ return stateChangeReq != null && stateChangeReq.activeChanged() && stateChangeReq.activate();
+ }
+
+ /**
+ * @return {@code True} if has baseline topology change request.
+ */
+ public boolean changedBaseline() {
+ return stateChangeReq != null && !stateChangeReq.activeChanged();
}
/**
@@ -341,7 +351,22 @@ public class ExchangeActions {
F.isEmpty(cacheGrpsToStart) &&
F.isEmpty(cacheGrpsToStop) &&
F.isEmpty(cachesToResetLostParts) &&
- stateChangeReq == null;
+ stateChangeReq == null &&
+ locJoinCtx == null;
+ }
+
+ /**
+ * @param locJoinCtx Caches local join context.
+ */
+ public void localJoinContext(LocalJoinCachesContext locJoinCtx) {
+ this.locJoinCtx = locJoinCtx;
+ }
+
+ /**
+ * @return Caches local join context.
+ */
+ public LocalJoinCachesContext localJoinContext() {
+ return locJoinCtx;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
index d4fbe60..0e7e01c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
@@ -125,22 +125,17 @@ public class ExchangeDiscoveryEvents {
this.lastEvt = evt;
this.discoCache = cache;
- if (evt.type() != EVT_DISCOVERY_CUSTOM_EVT) {
- ClusterNode node = evt.eventNode();
+ ClusterNode node = evt.eventNode();
- if (!CU.clientNode(node)) {
- lastSrvEvt = evt;
+ if (!CU.clientNode(node)) {
+ lastSrvEvt = evt;
- srvEvtTopVer = new AffinityTopologyVersion(evt.topologyVersion(), 0);
+ srvEvtTopVer = new AffinityTopologyVersion(evt.topologyVersion(), 0);
- if (evt.type()== EVT_NODE_JOINED)
- srvJoin = true;
- else {
- assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED : evt;
-
- srvLeft = !CU.clientNode(node);
- }
- }
+ if (evt.type()== EVT_NODE_JOINED)
+ srvJoin = true;
+ else if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
+ srvLeft = !CU.clientNode(node);
}
}
@@ -160,6 +155,14 @@ public class ExchangeDiscoveryEvents {
}
/**
+ * @param evt Event.
+ * @return {@code True} if given event is {@link EventType#EVT_NODE_JOINED}.
+ */
+ public static boolean serverJoinEvent(DiscoveryEvent evt) {
+ return (evt.type() == EVT_NODE_JOINED && !CU.clientNode(evt.eventNode()));
+ }
+
+ /**
* @return Discovery data cache for last event.
*/
public DiscoCache discoveryCache() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 4370e75..50b3909 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -826,6 +826,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
if (e != null) {
+ ctx.shared().database().checkpointReadLock();
+
try {
cacheVal = e.peek(modes.heap, modes.offheap, topVer, plc);
}
@@ -837,6 +839,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
finally {
ctx0.evicts().touch(e, null);
+
+ ctx.shared().database().checkpointReadUnlock();
}
}
@@ -2052,7 +2056,15 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
try {
ctx.shared().database().ensureFreeSpace(ctx.dataRegion());
+ }
+ catch (IgniteCheckedException e) {
+ // Wrap errors (will be unwrapped).
+ throw new GridClosureException(e);
+ }
+ ctx.shared().database().checkpointReadLock();
+
+ try {
entry = entryEx(key);
entry.unswap();
@@ -2097,6 +2109,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
// Wrap errors (will be unwrapped).
throw new GridClosureException(e);
}
+ finally {
+ ctx.shared().database().checkpointReadUnlock();
+ }
}
}
});
@@ -4454,6 +4469,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
GridCacheVersion obsoleteVer = ctx.versions().next();
+ ctx.shared().database().checkpointReadLock();
+
try {
KeyCacheObject cacheKey = ctx.toCacheKeyObject(key);
@@ -4468,6 +4485,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
catch (IgniteCheckedException ex) {
U.error(log, "Failed to clearLocally entry for key: " + key, ex);
}
+ finally {
+ ctx.shared().database().checkpointReadUnlock();
+ }
return false;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 0cdf669..103a976 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -62,6 +62,7 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTransactionalCacheAdapter;
@@ -2032,24 +2033,48 @@ public class GridCacheContext<K, V> implements Externalizable {
}
/**
+ * Checks if local reads are allowed for the given partition and reserves the partition when needed. If this
+ * method returns {@code true}, then {@link #releaseForFastLocalGet(int, AffinityTopologyVersion)} method
+ * must be called after the read is completed.
+ *
* @param part Partition.
- * @param affNodes Affinity nodes.
* @param topVer Topology version.
* @return {@code True} if cache 'get' operation is allowed to get entry locally.
*/
- public boolean allowFastLocalRead(int part, List<ClusterNode> affNodes, AffinityTopologyVersion topVer) {
- boolean result = affinityNode() && rebalanceEnabled() && hasPartition(part, affNodes, topVer);
+ public boolean reserveForFastLocalGet(int part, AffinityTopologyVersion topVer) {
+ boolean result = affinityNode() && rebalanceEnabled() && checkAndReservePartition(part, topVer);
// When persistence is enabled, only reading from partitions with OWNING state is allowed.
assert !result || !group().persistenceEnabled() ||
topology().partitionState(localNodeId(), part) == OWNING :
- "result = " + result + ", persistenceEnabled = " + group().persistenceEnabled() +
- ", partitionState = " + topology().partitionState(localNodeId(), part);
+ "result=" + result + ", persistenceEnabled=" + group().persistenceEnabled() +
+ ", partitionState=" + topology().partitionState(localNodeId(), part) +
+ ", replicated=" + isReplicated();
return result;
}
/**
+ * Releases the partition that was reserved by a call to
+ * {@link #reserveForFastLocalGet(int, AffinityTopologyVersion)}.
+ *
+ * @param part Partition to release.
+ * @param topVer Topology version.
+ */
+ public void releaseForFastLocalGet(int part, AffinityTopologyVersion topVer) {
+ assert affinityNode();
+
+ if (!isReplicated() || group().persistenceEnabled()) {
+ GridDhtLocalPartition locPart = topology().localPartition(part, topVer, false);
+
+ assert locPart != null && locPart.state() == OWNING : "partition evicted after reserveForFastLocalGet " +
+ "[part=" + part + ", locPart=" + locPart + ", topVer=" + topVer + ']';
+
+ locPart.release();
+ }
+ }
+
+ /**
* Checks if it is possible to directly read data memory without entry creation (this
* is optimization to avoid unnecessary blocking synchronization on cache entry).
*
@@ -2063,17 +2088,44 @@ public class GridCacheContext<K, V> implements Externalizable {
/**
* @param part Partition.
- * @param affNodes Affinity nodes.
* @param topVer Topology version.
* @return {@code True} if partition is available locally.
*/
- private boolean hasPartition(int part, List<ClusterNode> affNodes, AffinityTopologyVersion topVer) {
+ private boolean checkAndReservePartition(int part, AffinityTopologyVersion topVer) {
assert affinityNode();
GridDhtPartitionTopology top = topology();
- return (top.rebalanceFinished(topVer) && (isReplicated() || affNodes.contains(locNode)))
- || (top.partitionState(localNodeId(), part) == OWNING);
+ if (isReplicated() && !group().persistenceEnabled()) {
+ boolean rebFinished = top.rebalanceFinished(topVer);
+
+ if (rebFinished)
+ return true;
+
+ GridDhtLocalPartition locPart = top.localPartition(part, topVer, false, false);
+
+ // No need to reserve a partition for REPLICATED cache because this partition cannot be evicted.
+ return locPart != null && locPart.state() == OWNING;
+ }
+ else {
+ GridDhtLocalPartition locPart = top.localPartition(part, topVer, false, false);
+
+ if (locPart != null && locPart.reserve()) {
+ boolean canRead = true;
+
+ try {
+ canRead = locPart.state() == OWNING;
+
+ return canRead;
+ }
+ finally {
+ if (!canRead)
+ locPart.release();
+ }
+ }
+ else
+ return false;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 345c2e4..cb36acd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -35,6 +35,7 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.eviction.EvictableEntry;
import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -943,7 +944,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
// Lock should be held by now.
if (!cctx.isAll(this, filter))
- return new GridCacheUpdateTxResult(false, null);
+ return new GridCacheUpdateTxResult(false, null, null);
final GridCacheVersion newVer;
@@ -951,6 +952,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
Object key0 = null;
Object val0 = null;
+ WALPointer logPtr = null;
long updateCntr0;
@@ -966,7 +968,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
// It is possible that 'get' could load more recent value.
if (!((GridNearCacheEntry)this).recordDhtVersion(dhtVer))
- return new GridCacheUpdateTxResult(false, null);
+ return new GridCacheUpdateTxResult(false, null, logPtr);
}
assert tx == null || (!tx.local() && tx.onePhaseCommit()) || tx.ownsLock(this) :
@@ -1002,7 +1004,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
key0 = e.key();
if (interceptorVal == null)
- return new GridCacheUpdateTxResult(false, (CacheObject)cctx.unwrapTemporary(old));
+ return new GridCacheUpdateTxResult(false, (CacheObject)cctx.unwrapTemporary(old), logPtr);
else if (interceptorVal != val0)
val0 = cctx.unwrapTemporary(interceptorVal);
@@ -1044,6 +1046,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (updateCntr != null && updateCntr != 0)
updateCntr0 = updateCntr;
+ if (tx != null && cctx.group().persistenceEnabled())
+ logPtr = logTxUpdate(tx, val, expireTime, updateCntr0);
+
update(val, expireTime, ttl, newVer, true);
drReplicate(drType, val, newVer, topVer);
@@ -1104,8 +1109,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (intercept)
cctx.config().getInterceptor().onAfterPut(new CacheLazyEntry(cctx, key, key0, val, val0, keepBinary, updateCntr0));
- return valid ? new GridCacheUpdateTxResult(true, retval ? old : null, updateCntr0) :
- new GridCacheUpdateTxResult(false, null);
+ return valid ? new GridCacheUpdateTxResult(true, retval ? old : null, updateCntr0, logPtr) :
+ new GridCacheUpdateTxResult(false, null, logPtr);
}
/**
@@ -1146,7 +1151,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
// Lock should be held by now.
if (!cctx.isAll(this, filter))
- return new GridCacheUpdateTxResult(false, null);
+ return new GridCacheUpdateTxResult(false, null, null);
GridCacheVersion obsoleteVer = null;
@@ -1158,6 +1163,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
long updateCntr0;
+ WALPointer logPtr = null;
+
boolean deferred;
boolean marked = false;
@@ -1172,7 +1179,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
// It is possible that 'get' could load more recent value.
if (!((GridNearCacheEntry)this).recordDhtVersion(dhtVer))
- return new GridCacheUpdateTxResult(false, null);
+ return new GridCacheUpdateTxResult(false, null, logPtr);
}
assert tx == null || (!tx.local() && tx.onePhaseCommit()) || tx.ownsLock(this) :
@@ -1200,7 +1207,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (cctx.cancelRemove(interceptRes)) {
CacheObject ret = cctx.toCacheObject(cctx.unwrapTemporary(interceptRes.get2()));
- return new GridCacheUpdateTxResult(false, ret);
+ return new GridCacheUpdateTxResult(false, ret, logPtr);
}
}
@@ -1228,6 +1235,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (updateCntr != null && updateCntr != 0)
updateCntr0 = updateCntr;
+ if (tx != null && cctx.group().persistenceEnabled())
+ logPtr = logTxUpdate(tx, null, 0, updateCntr0);
+
drReplicate(drType, null, newVer, topVer);
if (metrics && cctx.statisticsEnabled())
@@ -1325,10 +1335,10 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
else
ret = old;
- return new GridCacheUpdateTxResult(true, ret, updateCntr0);
+ return new GridCacheUpdateTxResult(true, ret, updateCntr0, logPtr);
}
else
- return new GridCacheUpdateTxResult(false, null);
+ return new GridCacheUpdateTxResult(false, null, logPtr);
}
/**
@@ -3549,6 +3559,39 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
/**
+ * @param tx Transaction.
+ * @param val Value.
+ * @param expireTime Expire time (or 0 if not applicable).
+ * @param updCntr Update counter.
+ * @throws IgniteCheckedException In case of log failure.
+ */
+ protected WALPointer logTxUpdate(IgniteInternalTx tx, CacheObject val, long expireTime, long updCntr)
+ throws IgniteCheckedException {
+ assert cctx.transactional();
+
+ if (tx.local()) { // For remote tx we log all updates in batch: GridDistributedTxRemoteAdapter.commitIfLocked()
+ GridCacheOperation op;
+ if (val == null)
+ op = GridCacheOperation.DELETE;
+ else
+ op = this.val == null ? GridCacheOperation.CREATE : GridCacheOperation.UPDATE;
+
+ return cctx.shared().wal().log(new DataRecord(new DataEntry(
+ cctx.cacheId(),
+ key,
+ val,
+ op,
+ tx.nearXidVersion(),
+ tx.writeVersion(),
+ expireTime,
+ key.partition(),
+ updCntr)));
+ }
+ else
+ return null;
+ }
+
+ /**
* Removes value from offheap.
*
* @throws IgniteCheckedException If failed.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index fe9ed29..7107125 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -81,6 +81,8 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.Gri
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionHistorySuppliersMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionsToReloadMap;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.RebalanceReassignExchangeTask;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
@@ -254,7 +256,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
if (cache.state().transition()) {
if (log.isDebugEnabled())
- log.debug("Add pending event: " + evt);
+ log.debug("Adding pending event: " + evt);
pendingEvts.add(new PendingDiscoveryEvent(evt, cache));
}
@@ -290,6 +292,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
* @param cache Discovery data cache.
*/
private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) {
+ // Clean local join caches context.
+ cctx.cache().localJoinCachesContext();
+
if (log.isDebugEnabled())
log.debug("Ignore event, cluster is inactive: " + evt);
}
@@ -417,7 +422,19 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
- exchFut = exchangeFuture(exchId, evt, cache, null, null);
+ ExchangeActions exchActs = null;
+
+ if (evt.type() == EVT_NODE_JOINED && evt.eventNode().isLocal()) {
+ LocalJoinCachesContext locJoinCtx = cctx.cache().localJoinCachesContext();
+
+ if (locJoinCtx != null) {
+ exchActs = new ExchangeActions();
+
+ exchActs.localJoinContext(locJoinCtx);
+ }
+ }
+
+ exchFut = exchangeFuture(exchId, evt, cache, exchActs, null);
}
else {
DiscoveryCustomMessage customMsg = ((DiscoveryCustomEvent)evt).customMessage();
@@ -555,7 +572,12 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
GridDhtPartitionExchangeId exchId = initialExchangeId();
- fut = exchangeFuture(exchId, discoEvt, discoCache, null, null);
+ fut = exchangeFuture(
+ exchId,
+ reconnect ? null : discoEvt,
+ reconnect ? null : discoCache,
+ null,
+ null);
}
else if (reconnect)
reconnectExchangeFut.onDone();
@@ -907,6 +929,16 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
}
/**
+ * @param caches Caches to stop.
+ * @return Future that will be completed when caches are stopped from the exchange thread.
+ */
+ public IgniteInternalFuture<Void> deferStopCachesOnClientReconnect(Collection<GridCacheAdapter> caches) {
+ assert cctx.discovery().localNode().isClient();
+
+ return exchWorker.deferStopCachesOnClientReconnect(caches);
+ }
+
+ /**
* Schedules next full partitions update.
*/
public void scheduleResendPartitions() {
@@ -926,6 +958,10 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
* for non coordinator - {@link GridDhtPartitionsSingleMessage SingleMessages} send
*/
private void refreshPartitions() {
+ // TODO https://issues.apache.org/jira/browse/IGNITE-6857
+ if (cctx.snapshot().snapshotOperationInProgress())
+ return;
+
ClusterNode oldest = cctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE);
if (oldest == null) {
@@ -2042,6 +2078,17 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
}
/**
+ * @param caches Caches to stop.
+ */
+ IgniteInternalFuture<Void> deferStopCachesOnClientReconnect(Collection<GridCacheAdapter> caches) {
+ StopCachesOnClientReconnectExchangeTask task = new StopCachesOnClientReconnectExchangeTask(caches);
+
+ futQ.add(task);
+
+ return task;
+ }
+
+ /**
* @param exchFut Exchange future.
*/
void addExchangeFuture(GridDhtPartitionsExchangeFuture exchFut) {
@@ -2169,7 +2216,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
int cnt = 0;
for (CachePartitionExchangeWorkerTask task : futQ) {
- if (isExchangeTask(task)) {
+ if (task instanceof GridDhtPartitionsExchangeFuture) {
U.warn(log, ">>> " + ((GridDhtPartitionsExchangeFuture)task).shortInfo());
if (++cnt == 10)
@@ -2344,13 +2391,17 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
if (!cctx.kernalContext().clientNode()) {
assignsMap = new HashMap<>();
- for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+ IgniteCacheSnapshotManager snp = cctx.snapshot();
+
+ for (final CacheGroupContext grp : cctx.cache().cacheGroups()) {
long delay = grp.config().getRebalanceDelay();
+ boolean disableRebalance = snp.partitionsAreFrozen(grp);
+
GridDhtPreloaderAssignments assigns = null;
// Don't delay for dummy reassigns to avoid infinite recursion.
- if (delay == 0 || forcePreload)
+ if ((delay == 0 || forcePreload) && !disableRebalance)
assigns = grp.preloader().assign(exchId, exchFut);
assignsMap.put(grp.groupId(), assigns);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index e635e62..f234ecc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -81,6 +81,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalP
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache;
import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearAtomicCache;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTransactionalCache;
import org.apache.ignite.internal.processors.cache.dr.GridCacheDrManager;
@@ -92,6 +93,7 @@ import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabase
import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
@@ -360,6 +362,13 @@ public class GridCacheProcessor extends GridProcessorAdapter {
processStatisticsModeChange(task0.message());
}
+ else if (task instanceof StopCachesOnClientReconnectExchangeTask) {
+ StopCachesOnClientReconnectExchangeTask task0 = (StopCachesOnClientReconnectExchangeTask)task;
+
+ stopCachesOnClientReconnect(task0.stoppedCaches());
+
+ task0.onDone();
+ }
else
U.warn(log, "Unsupported custom exchange task: " + task);
}
@@ -1085,25 +1094,10 @@ public class GridCacheProcessor extends GridProcessorAdapter {
for (GridCacheAdapter cache : reconnected)
cache.context().gate().reconnected(false);
- IgniteInternalFuture<?> stopFut = null;
-
- if (!stoppedCaches.isEmpty()) {
- stopFut = ctx.closure().runLocalSafe(new Runnable() {
- @Override public void run() {
- for (GridCacheAdapter cache : stoppedCaches) {
- CacheGroupContext grp = cache.context().group();
-
- onKernalStop(cache, true);
- stopCache(cache, true, false);
-
- if (!grp.hasCaches())
- stopCacheGroup(grp);
- }
- }
- });
- }
+ if (!stoppedCaches.isEmpty())
+ return sharedCtx.exchange().deferStopCachesOnClientReconnect(stoppedCaches);
- return stopFut;
+ return null;
}
/**
@@ -1160,12 +1154,12 @@ public class GridCacheProcessor extends GridProcessorAdapter {
if (log.isInfoEnabled()) {
log.info("Started cache [name=" + cfg.getName() +
- ", id=" + cacheCtx.cacheId() +
+ ", id="+cacheCtx.cacheId() +
(cfg.getGroupName() != null ? ", group=" + cfg.getGroupName() : "") +
", memoryPolicyName=" + memPlcName +
", mode=" + cfg.getCacheMode() +
", atomicity=" + cfg.getAtomicityMode() +
- ", backups=" + cfg.getBackups() + ']');
+ ", backups=" + cfg.getBackups() +']');
}
}
@@ -1178,69 +1172,71 @@ public class GridCacheProcessor extends GridProcessorAdapter {
private void stopCache(GridCacheAdapter<?, ?> cache, boolean cancel, boolean destroy) {
GridCacheContext ctx = cache.context();
- if (!cache.isNear() && ctx.shared().wal() != null) {
- try {
- ctx.shared().wal().fsync(null);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to flush write-ahead log on cache stop " +
- "[cache=" + ctx.name() + "]", e);
+ try {
+ if (!cache.isNear() && ctx.shared().wal() != null) {
+ try {
+ ctx.shared().wal().fsync(null);
+ } catch (IgniteCheckedException e) {
+ U.error(log, "Failed to flush write-ahead log on cache stop " +
+ "[cache=" + ctx.name() + "]", e);
+ }
}
- }
- sharedCtx.removeCacheContext(ctx);
+ sharedCtx.removeCacheContext(ctx);
- cache.stop();
+ cache.stop();
- ctx.kernalContext().query().onCacheStop(ctx, !cache.context().group().persistenceEnabled() || destroy);
+ ctx.kernalContext().query().onCacheStop(ctx, !cache.context().group().persistenceEnabled() || destroy);
- if (isNearEnabled(ctx)) {
- GridDhtCacheAdapter dht = ctx.near().dht();
+ if (isNearEnabled(ctx)) {
+ GridDhtCacheAdapter dht = ctx.near().dht();
- // Check whether dht cache has been started.
- if (dht != null) {
- dht.stop();
+ // Check whether dht cache has been started.
+ if (dht != null) {
+ dht.stop();
- GridCacheContext<?, ?> dhtCtx = dht.context();
+ GridCacheContext<?, ?> dhtCtx = dht.context();
- List<GridCacheManager> dhtMgrs = dhtManagers(dhtCtx);
+ List<GridCacheManager> dhtMgrs = dhtManagers(dhtCtx);
- for (ListIterator<GridCacheManager> it = dhtMgrs.listIterator(dhtMgrs.size()); it.hasPrevious(); ) {
- GridCacheManager mgr = it.previous();
+ for (ListIterator<GridCacheManager> it = dhtMgrs.listIterator(dhtMgrs.size()); it.hasPrevious(); ) {
+ GridCacheManager mgr = it.previous();
- mgr.stop(cancel, destroy);
+ mgr.stop(cancel, destroy);
+ }
}
}
- }
- List<GridCacheManager> mgrs = ctx.managers();
+ List<GridCacheManager> mgrs = ctx.managers();
- Collection<GridCacheManager> excludes = dhtExcludes(ctx);
+ Collection<GridCacheManager> excludes = dhtExcludes(ctx);
- // Reverse order.
- for (ListIterator<GridCacheManager> it = mgrs.listIterator(mgrs.size()); it.hasPrevious(); ) {
- GridCacheManager mgr = it.previous();
+ // Reverse order.
+ for (ListIterator<GridCacheManager> it = mgrs.listIterator(mgrs.size()); it.hasPrevious(); ) {
+ GridCacheManager mgr = it.previous();
- if (!excludes.contains(mgr))
- mgr.stop(cancel, destroy);
- }
+ if (!excludes.contains(mgr))
+ mgr.stop(cancel, destroy);
+ }
- ctx.kernalContext().continuous().onCacheStop(ctx);
+ ctx.kernalContext().continuous().onCacheStop(ctx);
- ctx.kernalContext().cache().context().snapshot().onCacheStop(ctx);
+ ctx.kernalContext().cache().context().snapshot().onCacheStop(ctx);
- ctx.group().stopCache(ctx, destroy);
+ ctx.group().stopCache(ctx, destroy);
- U.stopLifecycleAware(log, lifecycleAwares(ctx.group(), cache.configuration(), ctx.store().configuredStore()));
+ U.stopLifecycleAware(log, lifecycleAwares(ctx.group(), cache.configuration(), ctx.store().configuredStore()));
- if (log.isInfoEnabled()) {
- if (ctx.group().sharedGroup())
- log.info("Stopped cache [cacheName=" + cache.name() + ", group=" + ctx.group().name() + ']');
- else
- log.info("Stopped cache [cacheName=" + cache.name() + ']');
+ if (log.isInfoEnabled()) {
+ if (ctx.group().sharedGroup())
+ log.info("Stopped cache [cacheName=" + cache.name() + ", group=" + ctx.group().name() + ']');
+ else
+ log.info("Stopped cache [cacheName=" + cache.name() + ']');
+ }
+ }
+ finally {
+ cleanup(ctx);
}
-
- cleanup(ctx);
}
/**
@@ -1733,20 +1729,24 @@ public class GridCacheProcessor extends GridProcessorAdapter {
/**
* @return Caches to be started when this node starts.
*/
- @NotNull public List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> cachesToStartOnLocalJoin() {
- return cachesInfo.cachesToStartOnLocalJoin();
+ @Nullable public LocalJoinCachesContext localJoinCachesContext() {
+ return cachesInfo.localJoinCachesContext();
}
/**
- * @param caches Caches to start.
+ * @param locJoinCtx Local join cache context.
* @param exchTopVer Current exchange version.
* @throws IgniteCheckedException If failed.
*/
- public void startCachesOnLocalJoin(List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches,
- AffinityTopologyVersion exchTopVer)
- throws IgniteCheckedException {
- if (!F.isEmpty(caches)) {
- for (T2<DynamicCacheDescriptor, NearCacheConfiguration> t : caches) {
+ public void startCachesOnLocalJoin(
+ LocalJoinCachesContext locJoinCtx,
+ AffinityTopologyVersion exchTopVer
+ ) throws IgniteCheckedException {
+ if (locJoinCtx != null) {
+ sharedCtx.affinity().initCachesOnLocalJoin(
+ locJoinCtx.cacheGroupDescriptors(), locJoinCtx.cacheDescriptors());
+
+ for (T2<DynamicCacheDescriptor, NearCacheConfiguration> t : locJoinCtx.caches()) {
DynamicCacheDescriptor desc = t.get1();
prepareCacheStart(
@@ -2402,12 +2402,16 @@ public class GridCacheProcessor extends GridProcessorAdapter {
/**
* @param msg Message.
* @param topVer Current topology version.
+ * @param curState Current cluster state.
* @throws IgniteCheckedException If configuration validation failed.
* @return Exchange actions.
*/
- public ExchangeActions onStateChangeRequest(ChangeGlobalStateMessage msg, AffinityTopologyVersion topVer)
- throws IgniteCheckedException {
- return cachesInfo.onStateChangeRequest(msg, topVer);
+ public ExchangeActions onStateChangeRequest(
+ ChangeGlobalStateMessage msg,
+ AffinityTopologyVersion topVer,
+ DiscoveryDataClusterState curState
+ ) throws IgniteCheckedException {
+ return cachesInfo.onStateChangeRequest(msg, topVer, curState);
}
/**
@@ -2476,6 +2480,28 @@ public class GridCacheProcessor extends GridProcessorAdapter {
}
/**
+ * @param stoppedCaches Stopped caches.
+ */
+ private void stopCachesOnClientReconnect(Collection<GridCacheAdapter> stoppedCaches) {
+ assert ctx.discovery().localNode().isClient();
+
+ for (GridCacheAdapter cache : stoppedCaches) {
+ CacheGroupContext grp = cache.context().group();
+
+ onKernalStop(cache, true);
+ stopCache(cache, true, false);
+
+ sharedCtx.affinity().stopCacheOnReconnect(cache.context());
+
+ if (!grp.hasCaches()) {
+ stopCacheGroup(grp);
+
+ sharedCtx.affinity().stopCacheGroupOnReconnect(grp);
+ }
+ }
+ }
+
+ /**
* @return {@code True} if need locally start all existing caches on client node start.
*/
private boolean startAllCachesOnClientStart() {
@@ -4039,7 +4065,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
if (ccfg != null) {
cloneCheckSerializable(ccfg);
- if (desc != null) {
+ if (desc != null || MetaStorage.METASTORAGE_CACHE_NAME.equals(cacheName)) {
if (failIfExists) {
throw new CacheExistsException("Failed to start cache " +
"(a cache with the same name is already started): " + cacheName);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
index 461baa7..92af83b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
@@ -35,27 +36,34 @@ public class GridCacheUpdateTxResult {
/** Partition idx. */
private long updateCntr;
+ /** */
+ private WALPointer logPtr;
+
/**
* Constructor.
*
* @param success Success flag.
* @param oldVal Old value (if any),
+ * @param logPtr Logger WAL pointer for the update.
*/
- GridCacheUpdateTxResult(boolean success, @Nullable CacheObject oldVal) {
+ GridCacheUpdateTxResult(boolean success, @Nullable CacheObject oldVal, WALPointer logPtr) {
this.success = success;
this.oldVal = oldVal;
+ this.logPtr = logPtr;
}
/**
* Constructor.
*
* @param success Success flag.
- * @param oldVal Old value (if any),
+ * @param oldVal Old value (if any).
+ * @param logPtr Logger WAL pointer for the update.
*/
- GridCacheUpdateTxResult(boolean success, @Nullable CacheObject oldVal, long updateCntr) {
+ GridCacheUpdateTxResult(boolean success, @Nullable CacheObject oldVal, long updateCntr, WALPointer logPtr) {
this.success = success;
this.oldVal = oldVal;
this.updateCntr = updateCntr;
+ this.logPtr = logPtr;
}
/**
@@ -73,6 +81,13 @@ public class GridCacheUpdateTxResult {
}
/**
+ * @return Logged WAL pointer for the update if persistence is enabled.
+ */
+ public WALPointer loggedPointer() {
+ return logPtr;
+ }
+
+ /**
* @return Old value.
*/
@Nullable public CacheObject oldValue() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 17d599b..0d63fc0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -73,6 +73,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLo
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
import org.apache.ignite.internal.processors.dr.GridDrType;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.processors.query.QueryUtils;
@@ -1375,6 +1376,15 @@ public class GridCacheUtils {
}
/**
+ * @param node Node.
+ * @param discoveryDataClusterState Discovery data cluster state.
+ * @return {@code True} if node is included in BaselineTopology.
+ */
+ public static boolean baselineNode(ClusterNode node, DiscoveryDataClusterState discoveryDataClusterState) {
+ return discoveryDataClusterState.baselineTopology().consistentIds().contains(node.consistentId());
+ }
+
+ /**
* Creates and starts store session listeners.
*
* @param ctx Kernal context.
@@ -1707,8 +1717,8 @@ public class GridCacheUtils {
final AffinityTopologyVersion topVer,
final IgniteLogger log,
final GridCacheContext cctx,
- final @Nullable KeyCacheObject key,
- final @Nullable IgniteCacheExpiryPolicy expiryPlc,
+ @Nullable final KeyCacheObject key,
+ @Nullable final IgniteCacheExpiryPolicy expiryPlc,
boolean readThrough,
boolean skipVals
) {
@@ -1721,6 +1731,8 @@ public class GridCacheUtils {
while (true) {
GridCacheEntryEx entry = null;
+ cctx.shared().database().checkpointReadLock();
+
try {
entry = colocated.entryEx(key, topVer);
@@ -1750,6 +1762,8 @@ public class GridCacheUtils {
finally {
if (entry != null)
cctx.evicts().touch(entry, topVer);
+
+ cctx.shared().database().checkpointReadUnlock();
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 8793729..d6f178d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -41,8 +41,8 @@ import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapt
import org.apache.ignite.internal.processors.cache.persistence.CacheSearchRow;
import org.apache.ignite.internal.processors.cache.persistence.RootPage;
import org.apache.ignite.internal.processors.cache.persistence.RowStore;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
import org.apache.ignite.internal.processors.cache.tree.CacheDataRowStore;
@@ -1175,12 +1175,12 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
// Use grp.sharedGroup() flag since it is possible cacheId is not yet set here.
boolean sizeWithCacheId = grp.sharedGroup();
- int oldLen = FreeListImpl.getRowSize(oldRow, sizeWithCacheId);
+ int oldLen = DataPageIO.getRowSize(oldRow, sizeWithCacheId);
if (oldLen > updateValSizeThreshold)
return false;
- int newLen = FreeListImpl.getRowSize(dataRow, sizeWithCacheId);
+ int newLen = DataPageIO.getRowSize(dataRow, sizeWithCacheId);
return oldLen == newLen;
}
@@ -1565,6 +1565,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
@Override public void init(long size, long updCntr, @Nullable Map<Integer, Long> cacheSizes) {
initCntr = updCntr;
storageSize.set(size);
+
cntr.set(updCntr);
if (cacheSizes != null) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java
new file mode 100644
index 0000000..db829b2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/LocalJoinCachesContext.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Context to capture caches state for a node joining to an active cluster. Since registered caches is updated in
+ * discovery thread and caches info are updated in exchange thread, we must capture the state in discovery thread
+ * somehow and pass it to the exchange. This class holds the required context.
+ */
+public class LocalJoinCachesContext {
+ /** */
+ @GridToStringInclude
+ private List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> locJoinStartCaches = Collections.emptyList();
+
+ /** */
+ @GridToStringInclude
+ private Map<Integer, CacheGroupDescriptor> cacheGrpDescs;
+
+ /** */
+ @GridToStringInclude
+ private Map<String, DynamicCacheDescriptor> cacheDescs;
+
+ /**
+ * @param locJoinStartCaches Local caches to start on join.
+ * @param cacheGrpDescs Cache group descriptors captured during join.
+ * @param cacheDescs Cache descriptors captured during join.
+ */
+ public LocalJoinCachesContext(
+ List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> locJoinStartCaches,
+ Map<Integer, CacheGroupDescriptor> cacheGrpDescs,
+ Map<String, DynamicCacheDescriptor> cacheDescs
+ ) {
+ this.locJoinStartCaches = locJoinStartCaches;
+ this.cacheGrpDescs = cacheGrpDescs;
+ this.cacheDescs = cacheDescs;
+ }
+
+ /**
+ * @return Caches to start.
+ */
+ public List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches() {
+ return locJoinStartCaches;
+ }
+
+ /**
+ * @return Group descriptors.
+ */
+ public Map<Integer, CacheGroupDescriptor> cacheGroupDescriptors() {
+ return cacheGrpDescs;
+ }
+
+ /**
+ * @return Cache descriptors.
+ */
+ public Map<String, DynamicCacheDescriptor> cacheDescriptors() {
+ return cacheDescs;
+ }
+
+ /**
+ * @param cacheNames Survived caches to clean.
+ */
+ public void removeSurvivedCaches(Set<String> cacheNames) {
+ if (cacheDescs != null) {
+ for (String cacheName : cacheNames)
+ cacheDescs.remove(cacheName);
+ }
+
+ Iterator<T2<DynamicCacheDescriptor, NearCacheConfiguration>> it = locJoinStartCaches.iterator();
+
+ for (; it.hasNext();) {
+ T2<DynamicCacheDescriptor, NearCacheConfiguration> entry = it.next();
+
+ DynamicCacheDescriptor desc = entry.get1();
+
+ if (cacheNames.contains(desc.cacheName()))
+ it.remove();
+ }
+ }
+
+ /**
+ * @param cacheGrps Survived caches groups to clean.
+ */
+ public void removeSurvivedCacheGroups(Set<Integer> cacheGrps) {
+ if (cacheGrpDescs != null) {
+ for (Integer grpId : cacheGrps)
+ cacheGrpDescs.remove(grpId);
+ }
+ }
+
+ /**
+ * @return {@code True} if the context is empty.
+ */
+ public boolean isEmpty() {
+ return F.isEmpty(locJoinStartCaches) && F.isEmpty(cacheGrpDescs) && F.isEmpty(cacheDescs);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(LocalJoinCachesContext.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java
index 2d35e81..30a42bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StateChangeRequest.java
@@ -19,8 +19,12 @@ package org.apache.ignite.internal.processors.cache;
import java.util.UUID;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
+import org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem;
import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
/**
*
@@ -30,6 +34,12 @@ public class StateChangeRequest {
private final ChangeGlobalStateMessage msg;
/** */
+ private final BaselineTopologyHistoryItem prevBltHistItem;
+
+ /** */
+ private final boolean activeChanged;
+
+ /** */
private final AffinityTopologyVersion topVer;
/**
@@ -37,8 +47,12 @@ public class StateChangeRequest {
* @param topVer State change topology versoin.
*/
public StateChangeRequest(ChangeGlobalStateMessage msg,
+ BaselineTopologyHistoryItem bltHistItem,
+ boolean activeChanged,
AffinityTopologyVersion topVer) {
this.msg = msg;
+ prevBltHistItem = bltHistItem;
+ this.activeChanged = activeChanged;
this.topVer = topVer;
}
@@ -50,6 +64,13 @@ public class StateChangeRequest {
}
/**
+ * @return State change message ID.
+ */
+ public IgniteUuid id() {
+ return msg.id();
+ }
+
+ /**
* @return State change request ID.
*/
public UUID requestId() {
@@ -64,6 +85,27 @@ public class StateChangeRequest {
}
/**
+ * @return {@code True} if active state was changed.
+ */
+ public boolean activeChanged() {
+ return activeChanged;
+ }
+
+ /**
+ * @return Previous baseline topology.
+ */
+ @Nullable public BaselineTopologyHistoryItem prevBaselineTopologyHistoryItem() {
+ return prevBltHistItem;
+ }
+
+ /**
+ * @return Baseline topology.
+ */
+ @Nullable public BaselineTopology baselineTopology() {
+ return msg.baselineTopology();
+ }
+
+ /**
* @return Node initiated state change process.
*/
public UUID initiatorNodeId() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
index a58918b..662839c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataFileStore.java
@@ -58,8 +58,8 @@ class BinaryMetadataFileStore {
final ConcurrentMap<Integer, BinaryMetadataHolder> metadataLocCache,
final GridKernalContext ctx,
final IgniteLogger log,
- @Nullable final File binaryMetadataFileStoreDir) throws IgniteCheckedException {
-
+ @Nullable final File binaryMetadataFileStoreDir
+ ) throws IgniteCheckedException {
this.metadataLocCache = metadataLocCache;
this.ctx = ctx;
this.log = log;
[07/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKey.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKey.java
new file mode 100644
index 0000000..7c9371e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKey.java
@@ -0,0 +1,103 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Partition key - pair of cache group ID and partition ID.
+ */
+public class PartitionKey implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Group id. */
+ private final int grpId;
+
+ /** Group name. Optional field, used only for output. */
+ private volatile String grpName;
+
+ /** Partition id. */
+ private final int partId;
+
+ /**
+ * @param grpId Group id.
+ * @param partId Partition id.
+ * @param grpName Group name.
+ */
+ public PartitionKey(int grpId, int partId, String grpName) {
+ this.grpId = grpId;
+ this.partId = partId;
+ this.grpName = grpName;
+ }
+
+ /**
+ * @return Group id.
+ */
+ public int groupId() {
+ return grpId;
+ }
+
+ /**
+ * @return Partition id.
+ */
+ public int partitionId() {
+ return partId;
+ }
+
+ /**
+ * @return Group name.
+ */
+ public String groupName() {
+ return grpName;
+ }
+
+ /**
+ * @param grpName Group name.
+ */
+ public void groupName(String grpName) {
+ this.grpName = grpName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ PartitionKey key = (PartitionKey)o;
+
+ return grpId == key.grpId && partId == key.partId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = grpId;
+
+ res = 31 * res + partId;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(PartitionKey.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java
new file mode 100644
index 0000000..d2ce882
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java
@@ -0,0 +1,204 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
+import org.apache.ignite.compute.ComputeTaskAdapter;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+@GridInternal
+public class RetrieveConflictPartitionValuesTask extends ComputeTaskAdapter<Map<PartitionHashRecord, List<PartitionEntryHashRecord>>,
+ Map<PartitionHashRecord, List<PartitionEntryHashRecord>>> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Injected logger. */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
+ List<ClusterNode> subgrid,
+ Map<PartitionHashRecord, List<PartitionEntryHashRecord>> collectTaskRes
+ ) throws IgniteException {
+ Map<ComputeJob, ClusterNode> jobs = new HashMap<>();
+
+ Map<Object, ClusterNode> consIdToNode = new HashMap<>();
+
+ for (ClusterNode node : subgrid)
+ consIdToNode.put(node.consistentId(), node);
+
+ for (Map.Entry<PartitionHashRecord, List<PartitionEntryHashRecord>> e : collectTaskRes.entrySet())
+ jobs.put(new RetrieveConflictValuesJob(new T2<>(e.getKey(), e.getValue())), consIdToNode.get(e.getKey().consistentId()));
+
+ return jobs;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Map<PartitionHashRecord, List<PartitionEntryHashRecord>> reduce(List<ComputeJobResult> results)
+ throws IgniteException {
+ Map<PartitionHashRecord, List<PartitionEntryHashRecord>> totalRes = new HashMap<>();
+
+ for (ComputeJobResult res : results) {
+ T2<PartitionHashRecord, List<PartitionEntryHashRecord>> nodeRes = res.getData();
+
+ totalRes.put(nodeRes.get1(), nodeRes.get2());
+ }
+
+ return totalRes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
+ ComputeJobResultPolicy superRes = super.result(res, rcvd);
+
+ // Deny failover.
+ if (superRes == ComputeJobResultPolicy.FAILOVER) {
+ superRes = ComputeJobResultPolicy.WAIT;
+
+ log.warning("RetrieveConflictValuesJob failed on node " +
+ "[consistentId=" + res.getNode().consistentId() + "]", res.getException());
+ }
+
+ return superRes;
+ }
+
+ /**
+ *
+ */
+ public static class RetrieveConflictValuesJob extends ComputeJobAdapter {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Ignite instance. */
+ @IgniteInstanceResource
+ private IgniteEx ignite;
+
+ /** Injected logger. */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** Partition hash record. */
+ private PartitionHashRecord partHashRecord;
+
+ /** Entry hash records. */
+ private List<PartitionEntryHashRecord> entryHashRecords;
+
+ /** Partition key. */
+ private PartitionKey partKey;
+
+ /**
+ * @param arg Partition key.
+ */
+ private RetrieveConflictValuesJob(T2<PartitionHashRecord, List<PartitionEntryHashRecord>> arg) {
+ partHashRecord = arg.get1();
+ entryHashRecords = arg.get2();
+ partKey = partHashRecord.partitionKey();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<PartitionHashRecord, List<PartitionEntryHashRecord>> execute() throws IgniteException {
+ CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(partKey.groupId());
+
+ if (grpCtx == null)
+ return Collections.emptyMap();
+
+ GridDhtLocalPartition part = grpCtx.topology().localPartition(partKey.partitionId());
+
+ if (part == null || !part.reserve())
+ return Collections.emptyMap();
+
+ HashMap<Integer, GridCacheContext> cacheIdToCtx = new HashMap<>();
+
+ for (GridCacheContext ctx : grpCtx.caches())
+ cacheIdToCtx.put(ctx.cacheId(), ctx);
+
+ try {
+ if (part.state() != GridDhtPartitionState.OWNING)
+ return Collections.emptyMap();
+
+ if (part.updateCounter() != partHashRecord.updateCounter()) {
+ throw new IgniteException("Cluster is not idle: update counter of partition " + partKey.toString() +
+ " changed during hash calculation [before=" + partHashRecord.updateCounter() +
+ ", after=" + part.updateCounter() + "]");
+ }
+
+ for (PartitionEntryHashRecord entryHashRecord : entryHashRecords) {
+ GridCacheContext ctx = cacheIdToCtx.get(entryHashRecord.cacheId());
+
+ if (ctx == null)
+ continue;
+
+ KeyCacheObject key = grpCtx.shared().kernalContext().cacheObjects().toKeyCacheObject(
+ grpCtx.cacheObjectContext(), entryHashRecord.key().cacheObjectType(), entryHashRecord.keyBytes());
+
+ CacheDataRow row = part.dataStore().find(ctx, key);
+
+ if (row == null)
+ continue;
+
+ CacheObject val = row.value();
+
+ Object o = CacheObjectUtils.unwrapBinaryIfNeeded(grpCtx.cacheObjectContext(), val, true, true);
+
+ if (o != null)
+ entryHashRecord.valueString(o.toString());
+
+ entryHashRecord.valueBytes(row.value().valueBytes(grpCtx.cacheObjectContext()));
+ }
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Can't retrieve value for partition " + partKey.toString(), e);
+
+ return Collections.emptyMap();
+ }
+ finally {
+ part.release();
+ }
+
+ return new T2<>(partHashRecord, entryHashRecords);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
new file mode 100644
index 0000000..23aa0e1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
@@ -0,0 +1,280 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
+import org.apache.ignite.compute.ComputeTaskAdapter;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.lang.GridIterator;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task for comparing update counters and checksums between primary and backup partitions of specified caches.
+ * <br>
+ * Argument: Set of cache names, 'null' will trigger verification for all caches.
+ * <br>
+ * Result: If there are any update counter conflicts (which signals about concurrent updates in
+ * cluster), map with all counter conflicts is returned. Otherwise, map with all hash conflicts is returned.
+ * Each conflict is represented by list of {@link PartitionHashRecord} with data from different nodes.
+ * Successful verification always returns empty map.
+ * <br>
+ * Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
+ * concurrently updated.
+ */
+@GridInternal
+public class VerifyBackupPartitionsTask extends ComputeTaskAdapter<Set<String>,
+ Map<PartitionKey, List<PartitionHashRecord>>> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Injected logger. */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
+ List<ClusterNode> subgrid, Set<String> cacheNames) throws IgniteException {
+ Map<ComputeJob, ClusterNode> jobs = new HashMap<>();
+
+ for (ClusterNode node : subgrid)
+ jobs.put(new VerifyBackupPartitionsJob(cacheNames), node);
+
+ return jobs;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Map<PartitionKey, List<PartitionHashRecord>> reduce(List<ComputeJobResult> results)
+ throws IgniteException {
+ Map<PartitionKey, List<PartitionHashRecord>> clusterHashes = new HashMap<>();
+
+ for (ComputeJobResult res : results) {
+ Map<PartitionKey, PartitionHashRecord> nodeHashes = res.getData();
+
+ for (Map.Entry<PartitionKey, PartitionHashRecord> e : nodeHashes.entrySet()) {
+ if (!clusterHashes.containsKey(e.getKey()))
+ clusterHashes.put(e.getKey(), new ArrayList<PartitionHashRecord>());
+
+ clusterHashes.get(e.getKey()).add(e.getValue());
+ }
+ }
+
+ Map<PartitionKey, List<PartitionHashRecord>> hashConflicts = new HashMap<>();
+
+ Map<PartitionKey, List<PartitionHashRecord>> updateCntrConflicts = new HashMap<>();
+
+ for (Map.Entry<PartitionKey, List<PartitionHashRecord>> e : clusterHashes.entrySet()) {
+ Integer partHash = null;
+ Long updateCntr = null;
+
+ for (PartitionHashRecord record : e.getValue()) {
+ if (partHash == null) {
+ partHash = record.partitionHash();
+
+ updateCntr = record.updateCounter();
+ }
+ else {
+ if (record.updateCounter() != updateCntr) {
+ updateCntrConflicts.put(e.getKey(), e.getValue());
+
+ break;
+ }
+
+ if (record.partitionHash() != partHash) {
+ hashConflicts.put(e.getKey(), e.getValue());
+
+ break;
+ }
+ }
+ }
+ }
+
+ return updateCntrConflicts.isEmpty() ? hashConflicts : updateCntrConflicts;
+ }
+
+ /** {@inheritDoc} */
+ @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) {
+ ComputeJobResultPolicy superRes = super.result(res, rcvd);
+
+ // Deny failover.
+ if (superRes == ComputeJobResultPolicy.FAILOVER) {
+ superRes = ComputeJobResultPolicy.WAIT;
+
+ log.warning("VerifyBackupPartitionsJob failed on node " +
+ "[consistentId=" + res.getNode().consistentId() + "]", res.getException());
+ }
+
+ return superRes;
+ }
+
+ /**
+ *
+ */
+ public static class VerifyBackupPartitionsJob extends ComputeJobAdapter {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Ignite instance. */
+ @IgniteInstanceResource
+ private IgniteEx ignite;
+
+ /** Injected logger. */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** Cache names. */
+ private Set<String> cacheNames;
+
+ /**
+ * @param names Names.
+ */
+ private VerifyBackupPartitionsJob(Set<String> names) {
+ cacheNames = names;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<PartitionKey, PartitionHashRecord> execute() throws IgniteException {
+ Set<Integer> grpIds = new HashSet<>();
+
+ Set<String> missingCaches = new HashSet<>();
+
+ if (cacheNames != null) {
+ for (String cacheName : cacheNames) {
+ DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName);
+
+ if (desc == null) {
+ missingCaches.add(cacheName);
+
+ continue;
+ }
+
+ grpIds.add(desc.groupId());
+ }
+
+ if (!missingCaches.isEmpty()) {
+ StringBuilder strBuilder = new StringBuilder("The following caches do not exist: ");
+
+ for (String name : missingCaches)
+ strBuilder.append(name).append(", ");
+
+ strBuilder.delete(strBuilder.length() - 2, strBuilder.length());
+
+ throw new IgniteException(strBuilder.toString());
+ }
+ }
+ else {
+ Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
+
+ for (CacheGroupContext grp : groups) {
+ if (!grp.systemCache() && !grp.isLocal())
+ grpIds.add(grp.groupId());
+ }
+ }
+
+ Map<PartitionKey, PartitionHashRecord> res = new HashMap<>();
+
+ for (Integer grpId : grpIds) {
+ CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
+
+ if (grpCtx == null)
+ continue;
+
+ List<GridDhtLocalPartition> parts = grpCtx.topology().localPartitions();
+
+ for (GridDhtLocalPartition part : parts) {
+ if (!part.reserve())
+ continue;
+
+ int partHash = 0;
+ long partSize;
+ long updateCntrBefore;
+
+ try {
+ if (part.state() != GridDhtPartitionState.OWNING)
+ continue;
+
+ updateCntrBefore = part.updateCounter();
+
+ partSize = part.dataStore().fullSize();
+
+ GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
+
+ while (it.hasNextX()) {
+ CacheDataRow row = it.nextX();
+
+ partHash += row.key().hashCode();
+
+ partHash += Arrays.hashCode(row.value().valueBytes(grpCtx.cacheObjectContext()));
+ }
+
+ long updateCntrAfter = part.updateCounter();
+
+ if (updateCntrBefore != updateCntrAfter) {
+ throw new IgniteException("Cluster is not idle: update counter of partition [grpId=" +
+ grpId + ", partId=" + part.id() + "] changed during hash calculation [before=" +
+ updateCntrBefore + ", after=" + updateCntrAfter + "]");
+ }
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Can't calculate partition hash [grpId=" + grpId +
+ ", partId=" + part.id() + "]", e);
+
+ continue;
+ }
+ finally {
+ part.release();
+ }
+
+ Object consId = ignite.context().discovery().localNode().consistentId();
+
+ boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
+
+ PartitionKey partKey = new PartitionKey(grpId, part.id(), grpCtx.cacheOrGroupName());
+
+ res.put(partKey, new PartitionHashRecord(
+ partKey, isPrimary, consId, partHash, updateCntrBefore, partSize));
+ }
+ }
+
+ return res;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java
new file mode 100644
index 0000000..fcb7a39
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.cluster.DetachedClusterNode;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * BaselineTopology represents a set of "database nodes" - nodes responsible for holding persistent-enabled caches
+ * and persisting their information to durable storage.
+ *
+ * Two major features BaselineTopology allows are:
+ * <ol>
+ * <li>Protection from conflicting updates.</li>
+ * <li>Cluster auto activation.</li>
+ * </ol>
+ *
+ * <h2>Protection from conflicting updates</h2>
+ * <p>
+ * Consider en example: there is a cluster of three nodes, A, B and C, all nodes store persistent data.
+ *
+ * Cluster history looks like this:
+ * [A,B,C]
+ * | \
+ * (1)cluster segmentation, two parts get activated independently
+ * | |
+ * [A,B] [C]
+ * | |
+ * (2)updates to both parts
+ *
+ * After independent updates applied to both parts of cluster at point(2) node C should not be allowed to join
+ * [A,B] part.
+ *
+ * The following algorithm makes sure node C will never join [A,B] part back:
+ * <ol>
+ * <li>
+ * When the cluster is initially activated BaselineTopology is created; its property <b>branchingPntHash</b>
+ * is calculated based on consistent IDs of all nodes.
+ * </li>
+ * <li>
+ * At point(1) two parts are activated separately; BaselineTopology in each is updated:
+ * old <b>branchingPntHash</b> is moved to <b>branchingHistory</b> list;
+ * new one is calculated based on new set of nodes.
+ * </li>
+ * <li>
+ * If node C tries to join other part of cluster (e.g. after network connectivity repair)
+ * [A,B] cluster checks its <b>branchingPntHash</b> and <b>branchingHistory</b>
+ * to see if branching history of C's BaselineTopology has diverged from [A,B].
+ * If divergence is detected [A,B] cluster refuses to let C into topology.
+ * </li>
+ * </ol>
+ *
+ * All new nodes joining a cluster should pass validation process before join; detailed algorithm of the process
+ * is described below.
+ * </p>
+ *
+ * <h3>Joining node validation algorithm</h3>
+ * Node joining a cluster firstly reads its local BaselineTopology from metastore and sends it to the cluster.
+ *
+ * <ol>
+ * <li>
+ * When BaselineTopology is set (e.g. on first activation) or recreated (e.g. with set-baseline command)
+ * its ID on all nodes is incremented by one.
+ *
+ * So when cluster receives a join request with BaselineTopology it firstly compares joining node BlT ID with
+ * local BlT ID.
+ *
+ * If joining node has a BaselineTopology with ID greater than one in cluster it means that BlT was changed
+ * more times there; therefore new node is not allowed to join the cluster.
+ * </li>
+ * <li>
+ * If user manually activates cluster when some of Baseline nodes are offline no new BlT is created.
+ * Instead current set of online nodes from BaselineTopology is used to update {@link BaselineTopology#branchingPntHash}
+ * property of current BaselineTopology.
+ * Old value of the property is moved to {@link BaselineTopology#branchingHist} list.
+ *
+ * If joining node and local BlT IDs are the same then cluster takes <b>branchingPntHash</b> of joining node
+ * and verifies that its local <b>branchingHist</b> contains that hash.
+ *
+ * If joining node hash is not presented in cluster branching history list
+ * it means that joining node was activated independently of currently running cluster;
+ * therefore new node is not allowed to join the cluster.
+ *
+ * If joining node hash is presented in the history, that it is safe to let the node join the cluster.
+ * </li>
+ * <li>
+ * When BaselineTopology is recreated (e.g. with set-baseline command) previous BaselineTopology is moved
+ * to BaselineHistory (consult source code of {@link GridClusterStateProcessor} for more details).
+ *
+ * If cluster sees that joining node BlT ID is less than cluster BlT ID it looks up for BaselineHistory item
+ * for new node ID.
+ * Having this BaselineHistory item cluster verifies that branching history of the item contains
+ * branching point hash of joining node
+ * (similar check as in the case above with only difference that joining node BlT is compared against
+ * BaselineHistory item instead of BaselineTopology).
+ *
+ * If new node branching point hash is found in the history than node is allowed to join;
+ * otherwise it is rejected.
+ * </li>
+ * </ol>
+ *
+ * <h2>Cluster auto activation</h2>
+ */
+public class BaselineTopology implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Consistent ID comparator. */
+ private static final Comparator<Object> CONSISTENT_ID_COMPARATOR = new Comparator<Object>() {
+ @Override public int compare(Object o1, Object o2) {
+ if (o1 instanceof Comparable && o2 instanceof Comparable && o1.getClass().equals(o2.getClass()))
+ return ((Comparable)o1).compareTo(o2);
+
+ return o1.toString().compareTo(o2.toString());
+ }
+ };
+
+ /** */
+ private final int id;
+
+ /** Key - node consistent ID, value - node attribute map. */
+ private final Map<Object, Map<String, Object>> nodeMap;
+
+ /** Compact ID to consistent ID mapping. */
+ private final Map<Short, Object> compactIdMapping;
+
+ /** Consistent ID to compact ID mapping. */
+ private final Map<Object, Short> consistentIdMapping;
+
+ /** */
+ private BranchingPointType lastBranchingPointType;
+
+ /** Branching point hash. */
+ private long branchingPntHash;
+
+ /** History of branching events.
+ * Each time when branching point hash changes previous value is added to this list. */
+ private final List<Long> branchingHist;
+
+ /**
+ * @param nodeMap Map of node consistent ID to it's attributes.
+ */
+ private BaselineTopology(Map<Object, Map<String, Object>> nodeMap, int id) {
+ this.id = id;
+
+ compactIdMapping = U.newHashMap(nodeMap.size());
+ consistentIdMapping = U.newHashMap(nodeMap.size());
+
+ this.nodeMap = nodeMap;
+
+ Set<Object> consistentIds = new TreeSet<>(CONSISTENT_ID_COMPARATOR);
+
+ for (Object o : nodeMap.keySet()){
+ branchingPntHash += (long)o.hashCode();
+
+ consistentIds.add(o);
+ }
+
+ short compactId = 0;
+
+ for (Object consistentId : consistentIds) {
+ compactIdMapping.put(compactId, consistentId);
+
+ consistentIdMapping.put(consistentId, compactId++);
+ }
+
+ lastBranchingPointType = BranchingPointType.NEW_BASELINE_TOPOLOGY;
+
+ branchingHist = new ArrayList<>();
+
+ branchingHist.add(branchingPntHash);
+ }
+
+ /**
+ * @return id of this BaselineTopology.
+ */
+ public int id() {
+ return id;
+ }
+
+ /**
+ * @return Set of consistent IDs.
+ */
+ public Set<Object> consistentIds() {
+ return nodeMap.keySet();
+ }
+
+ /**
+ * @return Activation history.
+ */
+ public List<Long> branchingHistory() {
+ return branchingHist;
+ }
+
+ /**
+ * @return Compact IDs mapping.
+ */
+ public Map<Short, Object> compactIdMapping() {
+ return compactIdMapping;
+ }
+
+ /**
+ * @return Consistent IDs mapping.
+ */
+ public Map<Object, Short> consistentIdMapping() {
+ return consistentIdMapping;
+ }
+
+ /**
+ * @return Short consistent Id.
+ */
+ public Short resolveShortConsistentId(Object constId){
+ return consistentIdMapping.get(constId);
+ }
+
+ /**
+ * @return Object consistent Id.
+ */
+ public Object resolveConsistentId(Short constId){
+ return compactIdMapping.get(constId);
+ }
+
+
+ /**
+ * @return Activation hash.
+ */
+ public long branchingPointHash() {
+ return branchingPntHash;
+ }
+
+ /**
+ * @param consId Node consistent ID.
+ * @return Node attributes map.
+ */
+ public Map<String, Object> attributes(Object consId) {
+ return nodeMap.get(consId);
+ }
+
+ /**
+ *
+ */
+ public List<BaselineNode> currentBaseline() {
+ List<BaselineNode> res = new ArrayList<>();
+
+ for (Map.Entry<Object, Map<String, Object>> consIdAttrsEntry : nodeMap.entrySet())
+ res.add(new DetachedClusterNode(consIdAttrsEntry.getKey(), consIdAttrsEntry.getValue()));
+
+ return res;
+ }
+
+ /**
+ * @param consId Consistent ID.
+ * @return Baseline node, if present in the baseline, or {@code null} if absent.
+ */
+ public ClusterNode baselineNode(Object consId) {
+ Map<String, Object> attrs = nodeMap.get(consId);
+
+ return attrs != null ? new DetachedClusterNode(consId, attrs) : null;
+ }
+
+ /**
+ * @param aliveNodes Sorted list of currently alive nodes.
+ * @param nodeFilter Node filter.
+ * @return Sorted list of baseline topology nodes.
+ */
+ public List<ClusterNode> createBaselineView(
+ List<ClusterNode> aliveNodes,
+ @Nullable IgnitePredicate<ClusterNode> nodeFilter)
+ {
+ List<ClusterNode> res = new ArrayList<>(nodeMap.size());
+
+ for (ClusterNode node : aliveNodes) {
+ if (nodeMap.containsKey(node.consistentId()) && (nodeFilter == null || CU.affinityNode(node, nodeFilter)))
+ res.add(node);
+ }
+
+ assert res.size() <= nodeMap.size();
+
+ if (res.size() == nodeMap.size())
+ return res;
+
+ Map<Object, ClusterNode> consIdMap = new HashMap<>();
+
+ for (ClusterNode node : aliveNodes) {
+ if (nodeMap.containsKey(node.consistentId()) && (nodeFilter == null || CU.affinityNode(node, nodeFilter)))
+ consIdMap.put(node.consistentId(), node);
+ }
+
+ for (Map.Entry<Object, Map<String, Object>> e : nodeMap.entrySet()) {
+ Object consId = e.getKey();
+
+ if (!consIdMap.containsKey(consId)) {
+ DetachedClusterNode node = new DetachedClusterNode(consId, e.getValue());
+
+ if (nodeFilter == null || CU.affinityNode(node, nodeFilter))
+ consIdMap.put(consId, node);
+ }
+ }
+
+ res = new ArrayList<>();
+
+ res.addAll(consIdMap.values());
+
+ Collections.sort(res, NodeOrderComparator.getInstance());
+
+ return res;
+ }
+
+ /**
+ * @param presentedNodes Nodes present in cluster.
+ * @return {@code True} if current topology satisfies baseline.
+ */
+ public boolean isSatisfied(@NotNull Collection<ClusterNode> presentedNodes) {
+ if (presentedNodes.size() < nodeMap.size())
+ return false;
+
+ Set<Object> presentedNodeIds = new HashSet<>();
+
+ for (ClusterNode node : presentedNodes)
+ presentedNodeIds.add(node.consistentId());
+
+ return presentedNodeIds.containsAll(nodeMap.keySet());
+ }
+
+ /**
+ * @return Size of the baseline topology.
+ */
+ public int size() {
+ return nodeMap.size();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ BaselineTopology top = (BaselineTopology)o;
+
+ return nodeMap != null ? nodeMap.keySet().equals(top.nodeMap.keySet()) : top.nodeMap == null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return nodeMap != null ? nodeMap.hashCode() : 0;
+ }
+
+ /**
+ * @param blt1 Baseline topology instance.
+ * @param blt2 Baseline topology instance.
+ * @return {@code True} if equals.
+ */
+ public static boolean equals(BaselineTopology blt1, BaselineTopology blt2) {
+ if (blt1 == null && blt2 == null)
+ return true;
+
+ if (blt1 == null ^ blt2 == null)
+ return false;
+
+ return blt1.equals(blt2);
+ }
+
+ /**
+ * @param nodes Nodes.
+ * @param id ID of BaselineTopology to build.
+ * @return Baseline topology consisting of given nodes.
+ */
+ @Nullable public static BaselineTopology build(Collection<? extends BaselineNode> nodes, int id) {
+ if (nodes == null)
+ return null;
+
+ Map<Object, Map<String, Object>> nodeMap = new HashMap<>();
+
+ for (BaselineNode node : nodes)
+ nodeMap.put(node.consistentId(), node.attributes());
+
+ return new BaselineTopology(nodeMap, id);
+ }
+
+ /**
+ * @param blt BaselineTopology to check.
+ * @return {@code True} if current BaselineTopology is compatible (the same or a newer one) with passed in Blt.
+ */
+ boolean isCompatibleWith(BaselineTopology blt) {
+ return blt == null || (branchingPntHash == blt.branchingPntHash) || branchingHist.contains(blt.branchingPntHash);
+ }
+
+ /**
+ * @param nodes Nodes.
+ */
+ boolean updateHistory(Collection<? extends BaselineNode> nodes) {
+ long newTopHash = calculateTopologyHash(nodes);
+
+ lastBranchingPointType = BranchingPointType.CLUSTER_ACTIVATION;
+
+ if (branchingPntHash != newTopHash) {
+ branchingPntHash = newTopHash;
+
+ branchingHist.add(newTopHash);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * Resets branching history of the current BaselineTopology.
+ * Current branching point hash and list of previous branching hashes are erased
+ * and replaced with the hash passed via method parameter.
+ *
+ * All nodes that were offline when reset happened (thus didn't reset history of their local BaselineTopologies)
+ * won't be allowed to join the cluster when started back.
+ *
+ * @param newBranchingPointHash New branching point hash.
+ */
+ void resetBranchingHistory(long newBranchingPointHash) {
+ lastBranchingPointType = BranchingPointType.BRANCHING_HISTORY_RESET;
+
+ branchingHist.clear();
+
+ branchingPntHash = newBranchingPointHash;
+ branchingHist.add(newBranchingPointHash);
+ }
+
+ /**
+ * @param nodes Nodes.
+ */
+ private long calculateTopologyHash(Collection<? extends BaselineNode> nodes) {
+ long res = 0;
+
+ Set<Object> bltConsIds = nodeMap.keySet();
+
+ for (BaselineNode node : nodes) {
+ if (bltConsIds.contains(node.consistentId()))
+ res += (long) node.consistentId().hashCode();
+ }
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "BaselineTopology [id=" + id
+ + ", branchingHash=" + branchingPntHash
+ + ", branchingType='" + lastBranchingPointType + '\''
+ + ", baselineNodes=" + nodeMap.keySet() + ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistory.java
new file mode 100644
index 0000000..fdc6cbf
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistory.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage;
+
+/**
+ *
+ */
+public class BaselineTopologyHistory implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private static final String METASTORE_BLT_HIST_PREFIX = "bltHist-";
+
+ /** */
+ private final Queue<BaselineTopologyHistoryItem> bufferedForStore =
+ new ConcurrentLinkedQueue<>();
+
+ /** */
+ private final List<BaselineTopologyHistoryItem> hist = new ArrayList<>();
+
+ /** */
+ void restoreHistory(ReadOnlyMetastorage metastorage, int lastId) throws IgniteCheckedException {
+ for (int i = 0; i < lastId; i++) {
+ BaselineTopologyHistoryItem histItem = (BaselineTopologyHistoryItem) metastorage.read(METASTORE_BLT_HIST_PREFIX + i);
+
+ if (histItem != null)
+ hist.add(histItem);
+ else
+ throw new IgniteCheckedException("Restoring of BaselineTopology history has failed, " +
+ "expected history item not found for id=" + i
+ );
+ }
+ }
+
+ /** */
+ BaselineTopologyHistory tailFrom(int id) {
+ BaselineTopologyHistory tail = new BaselineTopologyHistory();
+
+ for (BaselineTopologyHistoryItem item : hist) {
+ if (item.id() >= id)
+ tail.hist.add(item);
+ }
+
+ return tail;
+ }
+
+ /** */
+ void writeHistoryItem(ReadWriteMetastorage metastorage, BaselineTopologyHistoryItem histItem)
+ throws IgniteCheckedException
+ {
+ if (histItem == null)
+ return;
+
+ hist.add(histItem);
+
+ metastorage.write(METASTORE_BLT_HIST_PREFIX + histItem.id(), histItem);
+ }
+
+ /** */
+ void removeHistory(ReadWriteMetastorage metastorage) throws IgniteCheckedException {
+ if (hist.isEmpty())
+ return;
+
+ for (BaselineTopologyHistoryItem histItem : hist)
+ metastorage.remove(METASTORE_BLT_HIST_PREFIX + histItem.id());
+
+ hist.clear();
+ }
+
+ /** */
+ boolean isCompatibleWith(BaselineTopology blt) {
+ BaselineTopologyHistoryItem histBlt = hist.get(blt.id());
+
+ return histBlt.branchingHistory().contains(blt.branchingPointHash());
+ }
+
+ /** */
+ boolean isEmpty() {
+ return hist.isEmpty();
+ }
+
+ /** */
+ void bufferHistoryItemForStore(BaselineTopologyHistoryItem histItem) {
+ hist.add(histItem);
+
+ bufferedForStore.add(histItem);
+ }
+
+ /** */
+ public List<BaselineTopologyHistoryItem> history() {
+ return Collections.unmodifiableList(hist);
+ }
+
+ /** */
+ void flushHistoryItems(ReadWriteMetastorage metastorage) throws IgniteCheckedException {
+ while(!bufferedForStore.isEmpty()) {
+ BaselineTopologyHistoryItem item = bufferedForStore.remove();
+
+ metastorage.write(METASTORE_BLT_HIST_PREFIX + item.id(), item);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistoryItem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistoryItem.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistoryItem.java
new file mode 100644
index 0000000..4b2f0b5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopologyHistoryItem.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class BaselineTopologyHistoryItem implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final int id;
+
+ /** */
+ private final Collection<Object> consIds;
+
+ /** */
+ private final List<Long> branchingHistory;
+
+ /**
+ * @param id Id.
+ * @param consIds Consistent IDs.
+ * @param branchingHistory Activation history.
+ */
+ private BaselineTopologyHistoryItem(int id, Collection<Object> consIds, List<Long> branchingHistory) {
+ this.id = id;
+ this.consIds = consIds;
+ this.branchingHistory = branchingHistory;
+ }
+
+ /**
+ * @param blt Baseline Topology.
+ */
+ public static BaselineTopologyHistoryItem fromBaseline(BaselineTopology blt) {
+ if (blt == null)
+ return null;
+
+ List<Long> fullActivationHistory = new ArrayList<>(blt.branchingHistory().size());
+
+ fullActivationHistory.addAll(blt.branchingHistory());
+
+ return new BaselineTopologyHistoryItem(blt.id(), U.arrayList(blt.consistentIds()), fullActivationHistory);
+ }
+
+ /**
+ * @return ID.
+ */
+ public int id() {
+ return id;
+ }
+
+ /**
+ *
+ */
+ public List<Long> branchingHistory() {
+ return branchingHistory;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BranchingPointType.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BranchingPointType.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BranchingPointType.java
new file mode 100644
index 0000000..97388df
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BranchingPointType.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cluster;
+
+/**
+ * Used only for logging purposes.
+ *
+ * Enables to show in logs what was the cause of the last change
+ * of {@link BaselineTopology#branchingPntHash branching point hash}.
+ */
+enum BranchingPointType {
+ /** */
+ CLUSTER_ACTIVATION("Cluster activation"),
+
+ /** */
+ NEW_BASELINE_TOPOLOGY("New BaselineTopology"),
+
+ /** */
+ BRANCHING_HISTORY_RESET("Branching history reset");
+
+ /** */
+ private String type;
+
+ /** */
+ BranchingPointType(String type) {
+ this.type = type;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return type;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
index a1fbacf..d7dfa16 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
@@ -34,23 +34,27 @@ public class ChangeGlobalStateFinishMessage implements DiscoveryCustomMessage {
private static final long serialVersionUID = 0L;
/** Custom message ID. */
- private IgniteUuid id = IgniteUuid.randomUuid();
+ private final IgniteUuid id = IgniteUuid.randomUuid();
/** State change request ID. */
- private UUID reqId;
+ private final UUID reqId;
/** New cluster state. */
- private boolean clusterActive;
+ private final boolean clusterActive;
+
+ /** State change error. */
+ private Boolean transitionRes;
/**
* @param reqId State change request ID.
* @param clusterActive New cluster state.
*/
- public ChangeGlobalStateFinishMessage(UUID reqId, boolean clusterActive) {
+ public ChangeGlobalStateFinishMessage(UUID reqId, boolean clusterActive, Boolean transitionRes) {
assert reqId != null;
this.reqId = reqId;
this.clusterActive = clusterActive;
+ this.transitionRes = transitionRes;
}
/**
@@ -67,6 +71,13 @@ public class ChangeGlobalStateFinishMessage implements DiscoveryCustomMessage {
return clusterActive;
}
+ /**
+ * @return Transition success status.
+ */
+ public boolean success() {
+ return transitionRes == null ? clusterActive : transitionRes;
+ }
+
/** {@inheritDoc} */
@Override public IgniteUuid id() {
return id;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
index 6a642bc..50fc022 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
@@ -53,6 +53,15 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage {
private List<StoredCacheData> storedCfgs;
/** */
+ @Nullable private BaselineTopology baselineTopology;
+
+ /** */
+ private boolean forceChangeBaselineTopology;
+
+ /** */
+ private long timestamp;
+
+ /** */
@GridToStringExclude
private transient ExchangeActions exchangeActions;
@@ -66,7 +75,10 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage {
UUID reqId,
UUID initiatingNodeId,
@Nullable List<StoredCacheData> storedCfgs,
- boolean activate
+ boolean activate,
+ BaselineTopology baselineTopology,
+ boolean forceChangeBaselineTopology,
+ long timestamp
) {
assert reqId != null;
assert initiatingNodeId != null;
@@ -75,6 +87,9 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage {
this.initiatingNodeId = initiatingNodeId;
this.storedCfgs = storedCfgs;
this.activate = activate;
+ this.baselineTopology = baselineTopology;
+ this.forceChangeBaselineTopology = forceChangeBaselineTopology;
+ this.timestamp = timestamp;
}
/**
@@ -136,6 +151,27 @@ public class ChangeGlobalStateMessage implements DiscoveryCustomMessage {
}
/**
+ * @return Force change BaselineTopology flag.
+ */
+ public boolean forceChangeBaselineTopology() {
+ return forceChangeBaselineTopology;
+ }
+
+ /**
+ * @return Baseline topology.
+ */
+ @Nullable public BaselineTopology baselineTopology() {
+ return baselineTopology;
+ }
+
+ /**
+ * @return Timestamp.
+ */
+ public long timestamp() {
+ return timestamp;
+ }
+
+ /**
* @return State change request ID.
*/
public UUID requestId() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
index 71bf90b..1c8e830 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
@@ -38,6 +38,9 @@ public class DiscoveryDataClusterState implements Serializable {
private final boolean active;
/** */
+ @Nullable private final BaselineTopology baselineTopology;
+
+ /** */
private final UUID transitionReqId;
/** Topology version for state change exchange. */
@@ -51,12 +54,18 @@ public class DiscoveryDataClusterState implements Serializable {
/** Local flag for state transition result (global state is updated asynchronously by custom message). */
private transient volatile Boolean transitionRes;
+ /** */
+ private transient DiscoveryDataClusterState prevState;
+
+ /** */
+ private transient volatile Exception transitionError;
+
/**
* @param active Current status.
* @return State instance.
*/
- static DiscoveryDataClusterState createState(boolean active) {
- return new DiscoveryDataClusterState(active, null, null, null);
+ static DiscoveryDataClusterState createState(boolean active, @Nullable BaselineTopology baselineTopology) {
+ return new DiscoveryDataClusterState(null, active, baselineTopology, null, null, null);
}
/**
@@ -66,28 +75,45 @@ public class DiscoveryDataClusterState implements Serializable {
* @param transitionNodes Nodes participating in state change exchange.
* @return State instance.
*/
- static DiscoveryDataClusterState createTransitionState(boolean active,
+ static DiscoveryDataClusterState createTransitionState(
+ DiscoveryDataClusterState prevState,
+ boolean active,
+ @Nullable BaselineTopology baselineTopology,
UUID transitionReqId,
AffinityTopologyVersion transitionTopVer,
- Set<UUID> transitionNodes) {
+ Set<UUID> transitionNodes
+ ) {
assert transitionReqId != null;
assert transitionTopVer != null;
assert !F.isEmpty(transitionNodes) : transitionNodes;
- return new DiscoveryDataClusterState(active, transitionReqId, transitionTopVer, transitionNodes);
+ return new DiscoveryDataClusterState(
+ prevState,
+ active,
+ baselineTopology,
+ transitionReqId,
+ transitionTopVer,
+ transitionNodes);
}
/**
+ * @param prevState Previous state. May be non-null only for transitional states.
* @param active New state.
* @param transitionReqId State change request ID.
* @param transitionTopVer State change topology version.
* @param transitionNodes Nodes participating in state change exchange.
*/
- private DiscoveryDataClusterState(boolean active,
+ private DiscoveryDataClusterState(
+ DiscoveryDataClusterState prevState,
+ boolean active,
+ @Nullable BaselineTopology baselineTopology,
@Nullable UUID transitionReqId,
@Nullable AffinityTopologyVersion transitionTopVer,
- @Nullable Set<UUID> transitionNodes) {
+ @Nullable Set<UUID> transitionNodes
+ ) {
+ this.prevState = prevState;
this.active = active;
+ this.baselineTopology = baselineTopology;
this.transitionReqId = transitionReqId;
this.transitionTopVer = transitionTopVer;
this.transitionNodes = transitionNodes;
@@ -108,11 +134,8 @@ public class DiscoveryDataClusterState implements Serializable {
* @param active New cluster state.
*/
public void setTransitionResult(UUID reqId, boolean active) {
- if (reqId.equals(transitionReqId)) {
- assert transitionRes == null : this;
-
+ if (reqId.equals(transitionReqId))
transitionRes = active;
- }
}
/**
@@ -144,12 +167,50 @@ public class DiscoveryDataClusterState implements Serializable {
}
/**
+ * @return Baseline topology.
+ */
+ @Nullable public BaselineTopology baselineTopology() {
+ return baselineTopology;
+ }
+
+ /**
* @return Nodes participating in state change exchange.
*/
public Set<UUID> transitionNodes() {
return transitionNodes;
}
+ /**
+ * @return Transition error.
+ */
+ @Nullable public Exception transitionError() {
+ return transitionError;
+ }
+
+ /**
+ * @param ex Exception
+ */
+ public void transitionError(Exception ex) {
+ transitionError = ex;
+ }
+
+ /**
+ * @param success Transition success status.
+ * @return Cluster state that finished transition.
+ */
+ public DiscoveryDataClusterState finish(boolean success) {
+ return success ?
+ new DiscoveryDataClusterState(
+ null,
+ active,
+ baselineTopology,
+ null,
+ null,
+ null
+ ) :
+ prevState;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(DiscoveryDataClusterState.class, this);
[03/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCacheRemoveFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCacheRemoveFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCacheRemoveFailoverTest.java
new file mode 100644
index 0000000..b3da12e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteStableBaselineCacheRemoveFailoverTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractRemoveFailureTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ *
+ */
+public class IgniteStableBaselineCacheRemoveFailoverTest extends GridCacheAbstractRemoveFailureTest {
+ /** */
+ private static final int GRIDS_COUNT = 3;
+
+ /** */
+ private static final int OUT_OF_BASELINE_GRID_ID = GRIDS_COUNT;
+
+ /** {@inheritDoc} */
+ @Override protected CacheMode cacheMode() {
+ return PARTITIONED;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicityMode atomicityMode() {
+ return TRANSACTIONAL;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected NearCacheConfiguration nearCache() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setInitialSize(512 * 1024 * 1024)
+ .setMaxSize(512 * 1024 * 1024)
+ .setCheckpointPageBufferSize(512 * 1024 * 1024)
+ )
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ GridTestUtils.deleteDbFiles();
+
+ startGrids(GRIDS_COUNT);
+
+ grid(0).active(true);
+
+ startGrid(OUT_OF_BASELINE_GRID_ID);
+
+ awaitPartitionMapExchange();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean stop, final AtomicReference<CyclicBarrier> cmp) {
+ return GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ Thread.currentThread().setName("restart-thread");
+
+ while (!stop.get()) {
+ U.sleep(random(1000, 3000));
+
+ if (ThreadLocalRandom.current().nextInt(5) < 3)
+ killAndRestart(stop, random(1, GRIDS_COUNT + 1));
+ else
+ killAndRestart(stop, OUT_OF_BASELINE_GRID_ID);
+
+ CyclicBarrier barrier = cmp.get();
+
+ if (barrier != null) {
+ log.info("Wait data check.");
+
+ barrier.await(60_000, TimeUnit.MILLISECONDS);
+
+ log.info("Finished wait data check.");
+ }
+ }
+
+ return null;
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
new file mode 100644
index 0000000..34662a7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.checkpoint;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.LockSupport;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test generates low load to grid in having some shared groups. Test checks if pages marked dirty after some time will
+ * became reasonable low for 1 put.
+ */
+public class IgniteCheckpointDirtyPagesForLowLoadTest extends GridCommonAbstractTest {
+ /** Caches in group. */
+ private static final int CACHES_IN_GRP = 1;
+ /** Groups. */
+ private static final int GROUPS = 1;
+ /** Parts. */
+ private static final int PARTS = 1024;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ List<CacheConfiguration> ccfgs = new ArrayList<>();
+
+ for (int g = 0; g < GROUPS; g++) {
+ for (int i = 0; i < CACHES_IN_GRP; i++) {
+ CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>().setName("dummyCache" + i + "." + g)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setGroupName("dummyGroup" + g)
+ .setAffinity(new RendezvousAffinityFunction(false, PARTS));
+
+ ccfgs.add(ccfg);
+ }
+ }
+ cfg.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()]));
+
+ DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+ dsCfg.setPageSize(1024); //smaller page to reduce overhead to short values
+ dsCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true));
+ dsCfg.setCheckpointFrequency(500);
+ dsCfg.setWalMode(WALMode.LOG_ONLY);
+ dsCfg.setWalHistorySize(1);
+
+ cfg.setDataStorageConfiguration(dsCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "temp", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testManyCachesAndNotManyPuts() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(0);
+ ignite.active(true);
+
+ log.info("Saving initial data to caches");
+
+ for (int g = 0; g < GROUPS; g++) {
+ for (int c = 0; c < CACHES_IN_GRP; c++) {
+ ignite.cache("dummyCache" + c + "." + g)
+ .putAll(new TreeMap<Long, Long>() {{
+ for (int j = 0; j < PARTS; j++) {
+ // to fill each partition cache with at least 1 element
+ put((long)j, (long)j);
+ }
+ }});
+ }
+ }
+
+ GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)(ignite.context().cache().context().database());
+
+ Collection<Integer> pageCntObserved = new ArrayList<>();
+
+ boolean checkpointWithLowNumOfPagesFound = false;
+
+ for (int i = 0; i < 10; i++) {
+ Random random = new Random();
+ //touch some entry
+ int d = random.nextInt(PARTS) + PARTS;
+ int cIdx = random.nextInt(CACHES_IN_GRP);
+ int gIdx = random.nextInt(GROUPS);
+
+ String fullname = "dummyCache" + cIdx + "." + gIdx;
+
+ ignite.cache(fullname).put(d, d);
+
+ log.info("Put to " + fullname + " value " + d);
+
+ db.wakeupForCheckpoint("").get();
+
+ int currCpPages = waitForCurrentCheckpointPagesCounterUpdated(db);
+
+ pageCntObserved.add(currCpPages);
+
+ log.info("Current CP pages: " + currCpPages);
+
+ if (currCpPages < PARTS * GROUPS) {
+ checkpointWithLowNumOfPagesFound = true; //reasonable number of pages in CP
+ break;
+ }
+ }
+
+ stopGrid(0);
+
+ assertTrue("All checkpoints mark too much pages: " + pageCntObserved,
+ checkpointWithLowNumOfPagesFound);
+
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param db DB shared manager.
+ * @return counter when it becomes non-zero.
+ */
+ private int waitForCurrentCheckpointPagesCounterUpdated(GridCacheDatabaseSharedManager db) {
+ int currCpPages = 0;
+
+ while (currCpPages == 0) {
+ LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1));
+ currCpPages = db.currentCheckpointPagesCount();
+ }
+
+ return currCpPages;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
index 0dd9153..6f222f2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.cache.persistence.db.file;
import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -33,7 +34,6 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.nio.ByteOrder;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.configuration.CacheConfiguration;
@@ -47,35 +47,37 @@ import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.persistence.DummyPageIO;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.CacheObjectContext;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.persistence.DummyPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
+import org.apache.ignite.internal.util.lang.GridCloseableIterator;
+import org.apache.ignite.internal.util.lang.GridFilteredClosableIterator;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -269,11 +271,11 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
wal = shared.wal();
- try (WALIterator it = wal.replay(start)) {
- it.nextX();
+ try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
+ it.next();
for (FullPageId initialWrite : initWrites) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
assertTrue(String.valueOf(tup.get2()), tup.get2() instanceof PageSnapshot);
@@ -283,7 +285,7 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
//there are extra tracking pages, skip them
if (TrackingPageIO.VERSIONS.latest().trackingPageFor(actual.pageId(), mem.pageSize()) == actual.pageId()) {
- tup = it.nextX();
+ tup = it.next();
assertTrue(tup.get2() instanceof PageSnapshot);
@@ -355,8 +357,8 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
db.enableCheckpoints(false).get();
- try (WALIterator it = wal.replay(start)) {
- IgniteBiTuple<WALPointer, WALRecord> cpRecordTup = it.nextX();
+ try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
+ IgniteBiTuple<WALPointer, WALRecord> cpRecordTup = it.next();
assert cpRecordTup.get2() instanceof CheckpointRecord;
@@ -372,7 +374,7 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
CacheObjectContext coctx = cctx.cacheObjectContext();
while (idx < entries.size()) {
- IgniteBiTuple<WALPointer, WALRecord> dataRecTup = it.nextX();
+ IgniteBiTuple<WALPointer, WALRecord> dataRecTup = it.next();
assert dataRecTup.get2() instanceof DataRecord;
@@ -464,8 +466,8 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
db.enableCheckpoints(false);
- try (WALIterator it = wal.replay(start)) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+ try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
assert tup.get2() instanceof CheckpointRecord : tup.get2();
@@ -480,7 +482,7 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
int idx = 0;
while (idx < pageIds.size()) {
- tup = it.nextX();
+ tup = it.next();
assert tup.get2() instanceof PageSnapshot : tup.get2().getClass();
@@ -488,7 +490,7 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
//there are extra tracking pages, skip them
if (TrackingPageIO.VERSIONS.latest().trackingPageFor(snap.fullPageId().pageId(), pageMem.pageSize()) == snap.fullPageId().pageId()) {
- tup = it.nextX();
+ tup = it.next();
assertTrue(tup.get2() instanceof PageSnapshot);
@@ -656,15 +658,15 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
) throws Exception {
Map<FullPageId, byte[]> replay = new HashMap<>();
- try (WALIterator it = wal.replay(start)) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+ try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(start))) {
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
assertTrue("Invalid record: " + tup, tup.get2() instanceof CheckpointRecord);
CheckpointRecord cpRec = (CheckpointRecord)tup.get2();
- while (it.hasNextX()) {
- tup = it.nextX();
+ while (it.hasNext()) {
+ tup = it.next();
WALRecord rec = tup.get2();
@@ -1025,4 +1027,18 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
mem.releasePage(fullId.groupId(), fullId.pageId(), page);
}
}
+
+ /**
+ *
+ */
+ private static class PartitionMetaStateRecordExcludeIterator extends GridFilteredClosableIterator<IgniteBiTuple<WALPointer, WALRecord>> {
+ private PartitionMetaStateRecordExcludeIterator(GridCloseableIterator<? extends IgniteBiTuple<WALPointer, WALRecord>> it) {
+ super(it);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean accept(IgniteBiTuple<WALPointer, WALRecord> tup) {
+ return !(tup.get2() instanceof PartitionMetaStateRecord);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
index 2bcc22f..2a3fc1f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.persistence.db.wal;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.nio.MappedByteBuffer;
import java.nio.file.OpenOption;
import org.apache.ignite.IgniteCache;
@@ -60,6 +61,9 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
/** */
private boolean flushByTimeout;
+ /** */
+ private AtomicBoolean canFail = new AtomicBoolean();
+
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
deleteWorkFiles();
@@ -89,7 +93,7 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
DataStorageConfiguration memCfg = new DataStorageConfiguration()
.setDefaultDataRegionConfiguration(
new DataRegionConfiguration().setMaxSize(2048L * 1024 * 1024).setPersistenceEnabled(true))
- .setFileIOFactory(new FailingFileIOFactory())
+ .setFileIOFactory(new FailingFileIOFactory(canFail))
.setWalMode(WALMode.BACKGROUND)
.setWalBufferSize(128 * 1024)// Setting WAL Segment size to high values forces flushing by timeout.
.setWalSegmentSize(flushByTimeout ? 500_000 : 50_000);
@@ -139,6 +143,8 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
final int iterations = 100;
+ canFail.set(true);
+
for (int i = 0; i < iterations; i++) {
Transaction tx = grid.transactions().txStart(
TransactionConcurrency.PESSIMISTIC, TransactionIsolation.READ_COMMITTED);
@@ -177,8 +183,16 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
private static final long serialVersionUID = 0L;
/** Delegate factory. */
+ private AtomicBoolean fail;
+
+ /** */
private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
+ /** */
+ FailingFileIOFactory(AtomicBoolean fail) {
+ this.fail = fail;
+ }
+
/** {@inheritDoc} */
@Override public FileIO create(File file) throws IOException {
return create(file, CREATE, READ, WRITE);
@@ -192,8 +206,9 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
int writeAttempts = 2;
@Override public int write(ByteBuffer srcBuf) throws IOException {
- if (--writeAttempts == 0)
- throw new RuntimeException("Test exception. Unable to write to file.");
+
+ if (--writeAttempts <= 0 && fail!= null && fail.get())
+ throw new IOException("No space left on device");
return super.write(srcBuf);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index d6c0a9d..76c7851 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -87,7 +87,7 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
/** {@inheritDoc} */
@Override protected long getTestTimeout() {
- return 30_000;
+ return 60_000;
}
/** {@inheritDoc} */
@@ -174,6 +174,8 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
startGrid(gridCount());
+ grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
+
waitForRebalancing();
} catch (Exception expected) {
// There can be any exception. Do nothing.
@@ -194,6 +196,8 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
stopAllGrids();
+ canFail.set(false);
+
Ignite grid0 = startGrids(gridCount() + 1);
grid0.active(true);
@@ -243,7 +247,7 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
int writeAttempts = 2;
@Override public int write(ByteBuffer srcBuf) throws IOException {
- if (--writeAttempts == 0 && fail!= null && fail.get())
+ if (--writeAttempts <= 0 && fail!= null && fail.get())
throw new IOException("No space left on device");
return super.write(srcBuf);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
index 80d49ed..7bb550d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
@@ -69,6 +69,7 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -630,8 +631,6 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
try {
IgniteEx ctrlGrid = startGrid(0);
- ctrlGrid.active(true);
-
fork = true;
IgniteEx cacheGrid = startGrid(1);
@@ -680,8 +679,6 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
try {
IgniteEx ctrlGrid = startGrid(0);
- ctrlGrid.active(true);
-
fork = true;
IgniteEx cacheGrid = startGrid(1);
@@ -737,8 +734,6 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
try {
IgniteEx ctrlGrid = startGrid(0);
- ctrlGrid.active(true);
-
fork = true;
IgniteEx cacheGrid = startGrid(1);
@@ -777,8 +772,6 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
try {
IgniteEx ctrlGrid = startGrid(0);
- ctrlGrid.active(true);
-
fork = true;
IgniteEx cacheGrid = startGrid(1);
@@ -890,6 +883,279 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
}
/**
+ * @throws Exception If fail.
+ */
+ public void testMetastorage() throws Exception {
+ try {
+ int cnt = 5000;
+
+ IgniteEx ignite0 = (IgniteEx)startGrid("node1");
+ IgniteEx ignite1 = (IgniteEx)startGrid("node2");
+
+ ignite1.active(true);
+
+ GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+ GridCacheSharedContext<Object, Object> sharedCtx1 = ignite1.context().cache().context();
+
+ MetaStorage storage0 = sharedCtx0.database().metaStorage();
+ MetaStorage storage1 = sharedCtx1.database().metaStorage();
+
+ assert storage0 != null;
+
+ for (int i = 0; i < cnt; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage0.putData(String.valueOf(i), new byte[]{(byte)(i % 256), 2, 3});
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+
+ byte[] b1 = new byte[i + 3];
+ b1[0] = 1;
+ b1[1] = 2;
+ b1[2] = 3;
+
+ sharedCtx1.database().checkpointReadLock();
+
+ try {
+ storage1.putData(String.valueOf(i), b1);
+ }
+ finally {
+ sharedCtx1.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] d1 = storage0.getData(String.valueOf(i));
+ assertEquals(3, d1.length);
+ assertEquals((byte)(i % 256), d1[0]);
+ assertEquals(2, d1[1]);
+ assertEquals(3, d1[2]);
+
+ byte[] d2 = storage1.getData(String.valueOf(i));
+ assertEquals(i + 3, d2.length);
+ assertEquals(1, d2[0]);
+ assertEquals(2, d2[1]);
+ assertEquals(3, d2[2]);
+ }
+
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testMetastorageLargeArray() throws Exception {
+ try {
+ int cnt = 5000;
+ int arraySize = 32_768;
+
+ IgniteEx ignite = (IgniteEx)startGrid("node1");
+
+ ignite.active(true);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = ignite.context().cache().context();
+
+ MetaStorage storage = sharedCtx.database().metaStorage();
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] b1 = new byte[arraySize];
+ for (int k = 0; k < arraySize; k++) {
+ b1[k] = (byte) (k % 100);
+ }
+
+ sharedCtx.database().checkpointReadLock();
+
+ try {
+ storage.putData(String.valueOf(i), b1);
+ }
+ finally {
+ sharedCtx.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] d2 = storage.getData(String.valueOf(i));
+ assertEquals(arraySize, d2.length);
+
+ for (int k = 0; k < arraySize; k++) {
+ assertEquals((byte) (k % 100), d2[k]);
+ }
+ }
+
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testMetastorageRemove() throws Exception {
+ try {
+ int cnt = 400;
+
+ IgniteEx ignite0 = (IgniteEx)startGrid("node1");
+
+ ignite0.active(true);
+
+ GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+
+ MetaStorage storage = sharedCtx0.database().metaStorage();
+
+ assert storage != null;
+
+ for (int i = 0; i < cnt; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < 10; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage.removeData(String.valueOf(i));
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 10; i < cnt; i++) {
+ byte[] d1 = storage.getData(String.valueOf(i));
+ assertEquals(3, d1.length);
+ assertEquals(1, d1[0]);
+ assertEquals(2, d1[1]);
+ assertEquals(3, d1[2]);
+ }
+
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testMetastorageUpdate() throws Exception {
+ try {
+ int cnt = 2000;
+
+ IgniteEx ignite0 = (IgniteEx)startGrid("node1");
+
+ ignite0.active(true);
+
+ GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+
+ MetaStorage storage = sharedCtx0.database().metaStorage();
+
+ assert storage != null;
+
+ for (int i = 0; i < cnt; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < cnt; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage.putData(String.valueOf(i), new byte[]{2, 2, 3, 4});
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] d1 = storage.getData(String.valueOf(i));
+ assertEquals(4, d1.length);
+ assertEquals(2, d1[0]);
+ assertEquals(2, d1[1]);
+ assertEquals(3, d1[2]);
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testMetastorageWalRestore() throws Exception {
+ try {
+ int cnt = 2000;
+
+ IgniteEx ignite0 = (IgniteEx)startGrid(0);
+
+ ignite0.active(true);
+
+ GridCacheSharedContext<Object, Object> sharedCtx0 = ignite0.context().cache().context();
+
+ MetaStorage storage = sharedCtx0.database().metaStorage();
+
+ assert storage != null;
+
+ for (int i = 0; i < cnt; i++) {
+ sharedCtx0.database().checkpointReadLock();
+
+ try {
+ storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+ }
+ finally {
+ sharedCtx0.database().checkpointReadUnlock();
+ }
+ }
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] value = storage.getData(String.valueOf(i));
+ assert value != null;
+ assert value.length == 3;
+ }
+
+ stopGrid(0);
+
+ ignite0 = startGrid(0);
+
+ ignite0.active(true);
+
+ sharedCtx0 = ignite0.context().cache().context();
+
+ storage = sharedCtx0.database().metaStorage();
+
+ assert storage != null;
+
+ for (int i = 0; i < cnt; i++) {
+ byte[] value = storage.getData(String.valueOf(i));
+ assert value != null;
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
* @throws Exception if failed.
*/
public void testApplyDeltaRecords() throws Exception {
@@ -1576,6 +1842,8 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
return false;
}
+
+ assertTrue(Arrays.equals(data, (long[])val));
}
return true;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalSerializerVersionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalSerializerVersionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalSerializerVersionTest.java
index 60fb1d0..fc61632 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalSerializerVersionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalSerializerVersionTest.java
@@ -23,20 +23,22 @@ import java.util.Iterator;
import java.util.List;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.pagemem.wal.record.TimeStampRecord;
import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer;
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV2Serializer;
+import org.apache.ignite.internal.util.lang.GridCloseableIterator;
+import org.apache.ignite.internal.util.lang.GridFilteredClosableIterator;
import org.apache.ignite.internal.util.typedef.internal.GPC;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
@@ -46,10 +48,10 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.transactions.TransactionState;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+import static org.apache.ignite.transactions.TransactionState.PREPARED;
/**
*
@@ -131,13 +133,7 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
@Override public List<WALRecord> call() throws Exception {
WALRecord rec0 = new DataRecord(Collections.<DataEntry>emptyList());
- WALRecord rec1 = new TxRecord(
- TransactionState.PREPARED,
- null,
- null,
- null,
- null
- );
+ WALRecord rec1 = new TxRecord(PREPARED,null,null,null);
return Arrays.asList(rec0, rec1);
}
@@ -236,7 +232,7 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
}
}
else
- fail();
+ fail(String.valueOf(act));
}
}
@@ -273,7 +269,7 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
Iterator<Long> itToCheck = checker.getTimeStamps().iterator();
- try (WALIterator it = wal.replay(p)) {
+ try (PartitionMetaStateRecordExcludeIterator it = new PartitionMetaStateRecordExcludeIterator(wal.replay(p))) {
while (it.hasNext()) {
IgniteBiTuple<WALPointer, WALRecord> tup0 = it.next();
@@ -293,6 +289,8 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
stopAllGrids();
deleteWorkFiles();
+
+ System.clearProperty(IGNITE_WAL_SERIALIZER_VERSION);
}
/** {@inheritDoc} */
@@ -302,6 +300,8 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
stopAllGrids();
deleteWorkFiles();
+
+ System.clearProperty(IGNITE_WAL_SERIALIZER_VERSION);
}
/** {@inheritDoc} */
@@ -315,4 +315,18 @@ public class IgniteWalSerializerVersionTest extends GridCommonAbstractTest {
private void deleteWorkFiles() throws IgniteCheckedException {
deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
}
+
+ /**
+ *
+ */
+ private static class PartitionMetaStateRecordExcludeIterator extends GridFilteredClosableIterator<IgniteBiTuple<WALPointer, WALRecord>> {
+ private PartitionMetaStateRecordExcludeIterator(GridCloseableIterator<? extends IgniteBiTuple<WALPointer, WALRecord>> it) {
+ super(it);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean accept(IgniteBiTuple<WALPointer, WALRecord> tup) {
+ return !(tup.get2() instanceof PartitionMetaStateRecord);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
index 6b9f06a..d204347 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
@@ -254,7 +254,13 @@ public class WalCompactionTest extends GridCommonAbstractTest {
File[] cpMarkers = cpMarkersDir.listFiles(new FilenameFilter() {
@Override public boolean accept(File dir, String name) {
- return !(name.equals(cpMarkersToSave[0].getName()) || name.equals(cpMarkersToSave[1].getName()));
+ return !(
+ name.equals(cpMarkersToSave[0].getName()) ||
+ name.equals(cpMarkersToSave[1].getName()) ||
+ name.equals(cpMarkersToSave[2].getName()) ||
+ name.equals(cpMarkersToSave[3].getName()) ||
+ name.equals(cpMarkersToSave[4].getName())
+ );
}
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
index 10b6110..0b3d016 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
@@ -48,13 +48,14 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
-import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
import org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseListImpl;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -489,7 +490,7 @@ public class WalRecoveryTxLogicalRecordsTest extends GridCommonAbstractTest {
File[] cpFiles = cpDir.listFiles();
- assertTrue(cpFiles.length <= WAL_HIST_SIZE * 2);
+ assertTrue(cpFiles.length <= WAL_HIST_SIZE * 2 + 1); // starts & ends + node_start
}
finally {
stopAllGrids();
@@ -897,14 +898,14 @@ public class WalRecoveryTxLogicalRecordsTest extends GridCommonAbstractTest {
boolean foundTails = false;
for (GridDhtLocalPartition part : parts) {
- FreeListImpl freeList = GridTestUtils.getFieldValue(part.dataStore(), "freeList");
+ CacheFreeListImpl freeList = GridTestUtils.getFieldValue(part.dataStore(), "freeList");
if (freeList == null)
// Lazy store.
continue;
AtomicReferenceArray<PagesList.Stripe[]> buckets = GridTestUtils.getFieldValue(freeList,
- FreeListImpl.class, "buckets");
+ AbstractFreeList.class, "buckets");
//AtomicIntegerArray cnts = GridTestUtils.getFieldValue(freeList, PagesList.class, "cnts");
assertNotNull(buckets);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
index aa8d57f..63c219b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
@@ -1096,11 +1096,13 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
unwrappedKeyObj = key instanceof BinaryObject ? key : key.value(null, false);
}
- log.info("//Entry operation " + entry.op() + "; cache Id" + entry.cacheId() + "; " +
- "under transaction: " + globalTxId +
- //; entry " + entry +
- "; Key: " + unwrappedKeyObj +
- "; Value: " + unwrappedValObj);
+
+ if (dumpRecords)
+ log.info("//Entry operation " + entry.op() + "; cache Id" + entry.cacheId() + "; " +
+ "under transaction: " + globalTxId +
+ //; entry " + entry +
+ "; Key: " + unwrappedKeyObj +
+ "; Value: " + unwrappedValObj);
if (cacheObjHnd != null && (unwrappedKeyObj != null || unwrappedValObj != null))
cacheObjHnd.apply(unwrappedKeyObj, unwrappedValObj);
@@ -1113,8 +1115,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final TxRecord txRecord = (TxRecord)walRecord;
final GridCacheVersion globalTxId = txRecord.nearXidVersion();
- log.info("//Tx Record, state: " + txRecord.state() +
- "; nearTxVersion" + globalTxId);
+ if (dumpRecords)
+ log.info("//Tx Record, state: " + txRecord.state() +
+ "; nearTxVersion" + globalTxId);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloaderTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloaderTest.java
new file mode 100644
index 0000000..6d6e0c8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/FileDownloaderTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import static org.junit.Assert.*;
+
+/**
+ * FileDownloader test
+ */
+public class FileDownloaderTest extends GridCommonAbstractTest {
+
+ private static final Path DOWNLOADER_PATH = new File("download").toPath();
+ private static final Path UPLOADER_PATH = new File("upload").toPath();
+
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ if (DOWNLOADER_PATH.toFile().exists())
+ DOWNLOADER_PATH.toFile().delete();
+
+ if (UPLOADER_PATH.toFile().exists())
+ UPLOADER_PATH.toFile().delete();
+ }
+
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ if (DOWNLOADER_PATH.toFile().exists())
+ DOWNLOADER_PATH.toFile().delete();
+
+ if (UPLOADER_PATH.toFile().exists())
+ UPLOADER_PATH.toFile().delete();
+ }
+
+ public void test() throws Exception {
+ //todo uncomment and fix
+ /* assertTrue(UPLOADER_PATH.toFile().createNewFile());
+ assertTrue(!DOWNLOADER_PATH.toFile().exists());
+
+ PrintWriter writer = new PrintWriter(UPLOADER_PATH.toFile());
+
+ for (int i = 0; i < 1_000_000; i++)
+ writer.write("HELLO WORLD");
+
+ writer.close();
+
+ FileDownloader downloader = new FileDownloader(DOWNLOADER_PATH, Executors.newSingleThreadExecutor());
+
+ InetSocketAddress address = downloader.start();
+
+ FileUploader uploader = new FileUploader(UPLOADER_PATH, address, Executors.newSingleThreadExecutor());
+
+ long size = uploader.upload().get();
+
+ downloader.download(size).get();
+
+ assertTrue(DOWNLOADER_PATH.toFile().exists());
+
+ assertEquals(UPLOADER_PATH.toFile().length(), DOWNLOADER_PATH.toFile().length());
+
+ assertArrayEquals(Files.readAllBytes(UPLOADER_PATH), Files.readAllBytes(DOWNLOADER_PATH));*/
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
new file mode 100644
index 0000000..4b5dbc9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.database.IndexStorageSelfTest;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+
+/**
+ *
+ */
+public class IndexStoragePageMemoryImplTest extends IndexStorageSelfTest {
+ /** Make sure page is small enough to trigger multiple pages in a linked list. */
+ public static final int PAGE_SIZE = 1024;
+
+ /** */
+ private static File allocationPath;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
+ }
+
+ /**
+ * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
+ * new empty page memory.
+ * @return Page memory instance.
+ */
+ @Override protected PageMemory memory(boolean clean) throws Exception {
+ long[] sizes = new long[10];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 1024 * 1024;
+
+ DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(log),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ return new PageMemoryImpl(
+ provider, sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
+ }
+ }, new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new DataRegionMetricsImpl(new DataRegionConfiguration()),
+ false
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
deleted file mode 100644
index d5492ab..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.pagemem;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
-import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
-import org.apache.ignite.internal.processors.database.MetadataStorageSelfTest;
-import org.apache.ignite.internal.util.lang.GridInClosure3X;
-import org.apache.ignite.internal.util.typedef.CIX3;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.junits.GridTestKernalContext;
-
-/**
- *
- */
-public class MetadataStoragePageMemoryImplTest extends MetadataStorageSelfTest{
- /** Make sure page is small enough to trigger multiple pages in a linked list. */
- public static final int PAGE_SIZE = 1024;
-
- /** */
- private static File allocationPath;
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
- }
-
- /**
- * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
- * new empty page memory.
- * @return Page memory instance.
- */
- @Override protected PageMemory memory(boolean clean) throws Exception {
- long[] sizes = new long[10];
-
- for (int i = 0; i < sizes.length; i++)
- sizes[i] = 1024 * 1024;
-
- DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
-
- GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
- new GridTestKernalContext(log),
- null,
- null,
- null,
- new NoOpPageStoreManager(),
- new NoOpWALManager(),
- new IgniteCacheDatabaseSharedManager(),
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null
- );
-
- return new PageMemoryImpl(
- provider, sizes,
- sharedCtx,
- PAGE_SIZE,
- new CIX3<FullPageId, ByteBuffer, Integer>() {
- @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
- assert false : "No evictions should happen during the test";
- }
- },
- new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
- @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
- }
- }, new CheckpointLockStateChecker() {
- @Override public boolean checkpointLockIsHeldByThread() {
- return true;
- }
- },
- new DataRegionMetricsImpl(new DataRegionConfiguration()),
- false
- );
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index 40887e8c..8f72ed6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -59,6 +59,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
}
/** {@inheritDoc} */
+ @Override public void initializeForMetastorage() throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public void shutdownForCacheGroup(CacheGroupContext grp, boolean destroy) throws IgniteCheckedException {
// No-op.
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index b95208c..00f3f8f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -72,7 +72,7 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
}
/** {@inheritDoc} */
- @Override public int truncate(WALPointer ptr) {
+ @Override public int truncate(WALPointer low, WALPointer high) {
return 0;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
index 71107d4..71d23f2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
@@ -198,6 +198,7 @@ public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbst
IgniteConfiguration cfg = getConfiguration(name);
cfg.setConsistentId(node);
+ cfg.setAutoActivationEnabled(false);
((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(backUpIpFinder);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
index 02a21f4..1ef269e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
@@ -127,7 +127,7 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
}
});
- sleep(60_000);
+ sleep(30_000);
stop.set(true);
@@ -255,7 +255,7 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
}
});
- sleep(60_000);
+ sleep(30_000);
stop.set(true);
@@ -362,7 +362,7 @@ public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalState
}
});
- sleep(60_000);
+ sleep(30_000);
stop.set(true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
index 5552d70..0d9d9f2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
@@ -102,6 +102,8 @@ public abstract class IgniteAbstractStandByClientReconnectTest extends GridCommo
@Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(name);
+ cfg.setAutoActivationEnabled(false);
+
if (!nodeClient.equals(name))
cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(vmIpFinder));
else {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
index 2da32e3..871098f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
@@ -109,7 +109,7 @@ public class IgniteStandByClientReconnectToNewClusterTest extends IgniteAbstract
/**
* @throws Exception If failed.
*/
- public void testActiveClientReconnectToInActiveCluster() throws Exception {
+ public void testActiveClientReconnectToInactiveCluster() throws Exception {
startNodes(null);
info(">>>> star grid");
@@ -189,7 +189,7 @@ public class IgniteStandByClientReconnectToNewClusterTest extends IgniteAbstract
/**
* @throws Exception If failed.
*/
- public void testInActiveClientReconnectToActiveCluster() throws Exception {
+ public void testInactiveClientReconnectToActiveCluster() throws Exception {
CountDownLatch activateLatch = new CountDownLatch(1);
startNodes(activateLatch);
@@ -247,7 +247,7 @@ public class IgniteStandByClientReconnectToNewClusterTest extends IgniteAbstract
/**
* @throws Exception If failed.
*/
- public void testInActiveClientReconnectToInActiveCluster() throws Exception {
+ public void testInActiveClientReconnectToInactiveCluster() throws Exception {
startNodes(null);
IgniteEx ig1 = grid(node1);
[16/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 8eaa003..cf331f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -17,73 +17,643 @@
package org.apache.ignite.internal.commandline;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.apache.ignite.internal.client.GridClient;
+import org.apache.ignite.internal.client.GridClientAuthenticationException;
+import org.apache.ignite.internal.client.GridClientClosedException;
import org.apache.ignite.internal.client.GridClientClusterState;
+import org.apache.ignite.internal.client.GridClientCompute;
import org.apache.ignite.internal.client.GridClientConfiguration;
+import org.apache.ignite.internal.client.GridClientDisconnectedException;
import org.apache.ignite.internal.client.GridClientException;
import org.apache.ignite.internal.client.GridClientFactory;
+import org.apache.ignite.internal.client.GridClientHandshakeException;
+import org.apache.ignite.internal.client.GridClientNode;
+import org.apache.ignite.internal.client.GridServerUnreachableException;
+import org.apache.ignite.internal.client.impl.connection.GridClientConnectionResetException;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineNode;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineOperation;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineTask;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskArg;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskResult;
+
+import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR;
+import static org.apache.ignite.internal.IgniteVersionUtils.COPYRIGHT;
+import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.ADD;
+import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.COLLECT;
+import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.REMOVE;
+import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.SET;
+import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.VERSION;
/**
- *
+ * Class that execute several commands passed via command line.
*/
public class CommandHandler {
+ /** */
+ private static final String DFLT_HOST = "127.0.0.1";
+
+ /** */
+ private static final String DFLT_PORT = "11211";
+
+ /** */
+ private static final String CMD_HELP = "--help";
+
+ /** */
+ private static final String CMD_HOST = "--host";
+
+ /** */
+ private static final String CMD_PORT = "--port";
+
+ /** */
+ private static final String CMD_ACTIVATE = "--activate";
+
+ /** */
+ private static final String CMD_DEACTIVATE = "--deactivate";
+
+ /** */
+ private static final String CMD_STATE = "--state";
+
+ /** */
+ private static final String CMD_BASE_LINE = "--baseline";
+
+ /** */
+ private static final String BASELINE_ADD = "add";
+
+ /** */
+ private static final String BASELINE_REMOVE = "remove";
+
+ /** */
+ private static final String BASELINE_SET = "set";
+
+ /** */
+ private static final String BASELINE_SET_VERSION = "version";
+
+ /** */
+ private static final String DELIM = "--------------------------------------------------------------------------------";
+
+ /** */
+ public static final int EXIT_CODE_OK = 0;
+
+ /** */
+ public static final int EXIT_CODE_INVALID_ARGUMENTS = 1;
+
+ /** */
+ public static final int EXIT_CODE_CONNECTION_FAILED = 2;
+
+ /** */
+ public static final int ERR_AUTHENTICATION_FAILED = 3;
+
+ /** */
+ public static final int EXIT_CODE_UNEXPECTED_ERROR = 4;
+
+ /**
+ * Output specified string to console.
+ *
+ * @param s String to output.
+ */
+ private void log(String s) {
+ System.out.println(s);
+ }
+
/**
- * @param args Args.
+ * Output empty line.
*/
- public static void main(String[] args) throws GridClientException {
- String host = "127.0.0.1";
- String port = "11211";
- Boolean activate = null;
+ private void nl() {
+ System.out.println("");
+ }
+
+ /**
+ * Print error to console.
+ *
+ * @param errCode Error code to return.
+ * @param s Optional message.
+ * @param e Error to print.
+ */
+ private int error(int errCode, String s, Throwable e) {
+ if (!F.isEmpty(s))
+ log(s);
- if (args.length == 1 && "--help".equals(args[0])){
- System.out.println("Example: --host {ip} --port {port} --{activate/deactivate} " +
- "or without command --host {ip} --port {port} then will print status.");
+ String msg = e.getMessage();
- return;
+ if (F.isEmpty(msg))
+ msg = e.getClass().getName();
+
+ if (msg.startsWith("Failed to handle request")) {
+ int p = msg.indexOf("err=");
+
+ msg = msg.substring(p + 4, msg.length() - 1);
}
- if (args.length > 5)
- throw new IllegalArgumentException("incorrect number of arguments");
+ log("Error: " + msg);
+
+ return errCode;
+ }
+
+ /**
+ * Print command usage.
+ *
+ * @param desc Command description.
+ * @param cmd Command.
+ */
+ private void usage(String desc, String cmd) {
+ log(desc);
+ log(" control.sh [--host HOST_OR_IP] [--port PORT] " + cmd);
+ nl();
+ }
+
+ /**
+ * Extract next argument.
+ *
+ * @param it Arguments iterator.
+ * @param err Error message.
+ * @return Next argument value.
+ */
+ private String nextArg(Iterator<String> it, String err) {
+ if (it.hasNext()) {
+ String arg = it.next();
+
+ if (arg.startsWith("--"))
+ throw new IllegalArgumentException("Unexpected argument: " + arg);
+
+ return arg;
+ }
+
+ throw new IllegalArgumentException(err);
+ }
+
+ /**
+ * Activate cluster.
+ *
+ * @param client Client.
+ * @throws GridClientException If failed to activate.
+ */
+ private void activate(GridClient client) throws Throwable {
+ try {
+ GridClientClusterState state = client.state();
+
+ state.active(true);
- for (int i = 0; i < args.length; i++) {
- String str = args[i];
+ log("Cluster activated");
+ }
+ catch (Throwable e) {
+ log("Failed to activate cluster.");
- if ("--host".equals(str))
- host = args[i + 1];
- else if ("--port".equals(str))
- port = args[i + 1];
- else if ("--activate".equals(str))
- activate = true;
- else if ("--deactivate".equals(str))
- activate = false;
+ throw e;
}
+ }
- if (host == null)
- throw new IllegalArgumentException("host can not be empty");
+ /**
+ * Deactivate cluster.
+ *
+ * @param client Client.
+ * @throws Throwable If failed to deactivate.
+ */
+ private void deactivate(GridClient client) throws Throwable {
+ try {
+ GridClientClusterState state = client.state();
- if (port == null)
- throw new IllegalArgumentException("port can not be empty");
+ state.active(false);
- GridClientConfiguration cfg = new GridClientConfiguration();
- cfg.setServers(Collections.singletonList(host + ":" + port));
+ log("Cluster deactivated");
+ }
+ catch (Throwable e) {
+ log("Failed to deactivate cluster.");
- try (GridClient client = GridClientFactory.start(cfg)) {
+ throw e;
+ }
+ }
+
+ /**
+ * Print cluster state.
+ *
+ * @param client Client.
+ * @throws Throwable If failed to print state.
+ */
+ private void state(GridClient client) throws Throwable {
+ try {
GridClientClusterState state = client.state();
- if (activate != null)
+ log("Cluster is " + (state.active() ? "active" : "inactive"));
+ }
+ catch (Throwable e) {
+ log("Failed to get cluster state.");
+
+ throw e;
+ }
+ }
+
+ /**
+ *
+ * @param client Client
+ * @return Task result.
+ * @throws GridClientException If failed to execute task.
+ */
+ private <R> R executeTask(GridClient client, Class<?> taskCls, Object taskArgs) throws GridClientException {
+ GridClientCompute compute = client.compute();
+
+ List<GridClientNode> nodes = new ArrayList<>();
+
+ for (GridClientNode node : compute.nodes())
+ if (node.connectable())
+ nodes.add(node);
+
+ if (F.isEmpty(nodes))
+ throw new GridClientDisconnectedException("Connectable node not found", null);
+
+ GridClientNode node = compute.balancer().balancedNode(nodes);
+
+ return compute.projection(node).execute(taskCls.getName(),
+ new VisorTaskArgument<>(node.nodeId(), taskArgs, false));
+ }
+
+ /**
+ * Change baseline.
+ *
+ * @param client Client.
+ * @param baselineAct Baseline action to execute. @throws GridClientException If failed to execute baseline action.
+ * @param baselineArgs Baseline action arguments.
+ * @throws Throwable If failed to execute baseline action.
+ */
+ private void baseline(GridClient client, String baselineAct, String baselineArgs) throws Throwable {
+ switch (baselineAct) {
+ case BASELINE_ADD:
+ baselineAdd(client, baselineArgs);
+ break;
+
+ case BASELINE_REMOVE:
+ baselineRemove(client, baselineArgs);
+ break;
+
+ case BASELINE_SET:
+ baselineSet(client, baselineArgs);
+ break;
+
+ case BASELINE_SET_VERSION:
+ baselineVersion(client, baselineArgs);
+ break;
+
+ default:
+ baselinePrint(client);
+ }
+ }
+
+ /**
+ * Prepare task argument.
+ *
+ * @param op Operation.
+ * @param s Argument from command line.
+ * @return Task argument.
+ */
+ private VisorBaselineTaskArg arg(VisorBaselineOperation op, String s) {
+ switch (op) {
+ case ADD:
+ case REMOVE:
+ case SET:
+ if(F.isEmpty(s))
+ throw new IllegalArgumentException("Empty list of consistent IDs");
+
+ List<String> consistentIds = new ArrayList<>();
+
+ for (String consistentId : s.split(","))
+ consistentIds.add(consistentId.trim());
+
+ return new VisorBaselineTaskArg(op, -1, consistentIds);
+
+ case VERSION:
try {
- state.active(activate);
+ long topVer = Long.parseLong(s);
- System.out.println(host + ":" + port + " - was " + (activate ? "activate" : "deactivate"));
+ return new VisorBaselineTaskArg(op, topVer, null);
}
- catch (Exception e) {
- System.out.println("Something fail during " + (activate ? "activation" : "deactivation")
- + ", exception message: " + e.getMessage());
+ catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid topology version: " + s, e);
}
- else
- System.out.println(host + ":" + port + " - " + (state.active() ? "active" : "inactive"));
+ default:
+ return new VisorBaselineTaskArg(op, -1, null);
}
}
+
+ /**
+ * Print baseline topology.
+ *
+ * @param res Task result with baseline topology.
+ */
+ private void baselinePrint0(VisorBaselineTaskResult res) {
+ log("Cluster state: " + (res.isActive() ? "active" : "inactive"));
+ log("Current topology version: " + res.getTopologyVersion());
+ nl();
+
+ Map<String, VisorBaselineNode> baseline = res.getBaseline();
+ Map<String, VisorBaselineNode> servers = res.getServers();
+
+ if (F.isEmpty(baseline))
+ log("Baseline nodes not found.");
+ else {
+ log("Baseline nodes:");
+
+ for(VisorBaselineNode node : baseline.values()) {
+ log(" ConsistentID=" + node.getConsistentId() + ", STATE=" +
+ (servers.containsKey(node.getConsistentId()) ? "ONLINE" : "OFFLINE"));
+ }
+
+ log(DELIM);
+ log("Number of baseline nodes: " + baseline.size());
+
+ nl();
+
+ List<VisorBaselineNode> others = new ArrayList<>();
+
+ for (VisorBaselineNode node : servers.values()) {
+ if (!baseline.containsKey(node.getConsistentId()))
+ others.add(node);
+ }
+
+ if (F.isEmpty(others))
+ log("Other nodes not found.");
+ else {
+ log("Other nodes:");
+
+ for(VisorBaselineNode node : others)
+ log(" ConsistentID=" + node.getConsistentId());
+
+ log("Number of other nodes: " + others.size());
+ }
+ }
+ }
+
+ /**
+ * Print current baseline.
+ *
+ * @param client Client.
+ */
+ private void baselinePrint(GridClient client) throws GridClientException {
+ VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(COLLECT, ""));
+
+ baselinePrint0(res);
+ }
+
+ /**
+ * Add nodes to baseline.
+ *
+ * @param client Client.
+ * @param baselineArgs Baseline action arguments.
+ * @throws Throwable If failed to add nodes to baseline.
+ */
+ private void baselineAdd(GridClient client, String baselineArgs) throws Throwable {
+ try {
+ VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(ADD, baselineArgs));
+
+ baselinePrint0(res);
+ }
+ catch (Throwable e) {
+ log("Failed to add nodes to baseline.");
+
+ throw e;
+ }
+ }
+
+ /**
+ * Remove nodes from baseline.
+ *
+ * @param client Client.
+ * @param consistentIds Consistent IDs.
+ * @throws Throwable If failed to remove nodes from baseline.
+ */
+ private void baselineRemove(GridClient client, String consistentIds) throws Throwable {
+ try {
+ VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(REMOVE, consistentIds));
+
+ baselinePrint0(res);
+ }
+ catch (Throwable e) {
+ log("Failed to remove nodes from baseline.");
+
+ throw e;
+ }
+ }
+
+ /**
+ * Set baseline.
+ *
+ * @param client Client.
+ * @param consistentIds Consistent IDs.
+ * @throws Throwable If failed to set baseline.
+ */
+ private void baselineSet(GridClient client, String consistentIds) throws Throwable {
+ try {
+ VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(SET, consistentIds));
+
+ baselinePrint0(res);
+ }
+ catch (Throwable e) {
+ log("Failed to set baseline.");
+
+ throw e;
+ }
+ }
+
+ /**
+ * Set baseline by topology version.
+ *
+ * @param client Client.
+ * @param arg Argument from command line.
+ */
+ private void baselineVersion(GridClient client, String arg) throws GridClientException {
+ try {
+ VisorBaselineTaskResult res = executeTask(client, VisorBaselineTask.class, arg(VERSION, arg));
+
+ baselinePrint0(res);
+ }
+ catch (Throwable e) {
+ log("Failed to set baseline with specified topology version.");
+
+ throw e;
+ }
+ }
+
+ /**
+ * @param e Exception to check.
+ * @return {@code true} if specified exception is {@link GridClientAuthenticationException}.
+ */
+ private boolean isAuthError(Throwable e) {
+ return X.hasCause(e, GridClientAuthenticationException.class);
+ }
+
+ /**
+ * @param e Exception to check.
+ * @return {@code true} if specified exception is a connection error.
+ */
+ private boolean isConnectionError(Throwable e) {
+ return e instanceof GridClientClosedException ||
+ e instanceof GridClientConnectionResetException ||
+ e instanceof GridClientDisconnectedException ||
+ e instanceof GridClientHandshakeException ||
+ e instanceof GridServerUnreachableException;
+ }
+
+ /**
+ * Parse and execute command.
+ *
+ * @param args Arguments to parse and execute.
+ * @return Exit code.
+ */
+ public int execute(String... args) {
+ log("Control utility [ver. " + ACK_VER_STR + "]");
+ log(COPYRIGHT);
+ log("User: " + System.getProperty("user.name"));
+ log(DELIM);
+
+ try {
+ if (F.isEmpty(args) || (args.length == 1 && CMD_HELP.equalsIgnoreCase(args[0]))){
+ log("This utility can do the following commands:");
+
+ usage(" Activate cluster:", CMD_ACTIVATE);
+ usage(" Deactivate cluster:", CMD_DEACTIVATE);
+ usage(" Print current cluster state:", CMD_STATE);
+ usage(" Print cluster baseline topology:", CMD_BASE_LINE);
+ usage(" Add nodes into baseline topology:", CMD_BASE_LINE + " add consistentId1[,consistentId2,....,consistentIdN]");
+ usage(" Remove nodes from baseline topology:", CMD_BASE_LINE + " remove consistentId1[,consistentId2,....,consistentIdN]");
+ usage(" Set baseline topology:", CMD_BASE_LINE + " set consistentId1[,consistentId2,....,consistentIdN]");
+ usage(" Set baseline topology based on version:", CMD_BASE_LINE + " version topologyVersion");
+
+ log("Default values:");
+ log(" HOST_OR_IP=" + DFLT_HOST);
+ log(" PORT=" + DFLT_PORT);
+ nl();
+
+ log("Exit codes:");
+ log(" " + EXIT_CODE_OK + " - successful execution.");
+ log(" " + EXIT_CODE_INVALID_ARGUMENTS + " - invalid arguments.");
+ log(" " + EXIT_CODE_CONNECTION_FAILED + " - connection failed.");
+ log(" " + ERR_AUTHENTICATION_FAILED + " - authentication failed.");
+ log(" " + EXIT_CODE_UNEXPECTED_ERROR + " - unexpected error.");
+
+ return EXIT_CODE_OK;
+ }
+
+ String host = DFLT_HOST;
+
+ String port = DFLT_PORT;
+
+ String baselineAct = "";
+
+ String baselineArgs = "";
+
+ List<String> commands = new ArrayList<>();
+
+ Iterator<String> it = Arrays.asList(args).iterator();
+
+ while (it.hasNext()) {
+ String str = it.next().toLowerCase();
+
+ switch (str) {
+ case CMD_HOST:
+ host = nextArg(it, "Expected host name");
+ break;
+
+ case CMD_PORT:
+ port = nextArg(it, "Expected port number");
+
+ try {
+ int p = Integer.parseInt(port);
+
+ if (p <= 0 || p > 65535)
+ throw new IllegalArgumentException("Invalid value for port: " + port);
+ }
+ catch (NumberFormatException ignored) {
+ throw new IllegalArgumentException("Invalid value for port: " + port);
+ }
+ break;
+
+ case CMD_ACTIVATE:
+ case CMD_DEACTIVATE:
+ case CMD_STATE:
+ commands.add(str);
+ break;
+
+ case CMD_BASE_LINE:
+ commands.add(CMD_BASE_LINE);
+
+ if (it.hasNext()) {
+ baselineAct = it.next().toLowerCase();
+
+ if (BASELINE_ADD.equals(baselineAct) || BASELINE_REMOVE.equals(baselineAct) ||
+ BASELINE_SET.equals(baselineAct) || BASELINE_SET_VERSION.equals(baselineAct))
+ baselineArgs = nextArg(it, "Expected baseline arguments");
+ else
+ throw new IllegalArgumentException("Unexpected argument for " + CMD_BASE_LINE + ": "
+ + baselineAct);
+ }
+
+ }
+ }
+
+ int sz = commands.size();
+
+ if (sz < 1)
+ throw new IllegalArgumentException("No action was specified");
+
+ if (sz > 1)
+ throw new IllegalArgumentException("Only one action can be specified, but found: " + sz);
+
+ GridClientConfiguration cfg = new GridClientConfiguration();
+
+ cfg.setServers(Collections.singletonList(host + ":" + port));
+
+ try (GridClient client = GridClientFactory.start(cfg)) {
+ String cmd = commands.get(0);
+
+ switch (cmd) {
+ case CMD_ACTIVATE:
+ activate(client);
+ break;
+
+ case CMD_DEACTIVATE:
+ deactivate(client);
+ break;
+
+ case CMD_STATE:
+ state(client);
+ break;
+
+ case CMD_BASE_LINE:
+ baseline(client, baselineAct, baselineArgs);
+ break;
+ }
+ }
+
+ return 0;
+ }
+ catch (IllegalArgumentException e) {
+ return error(EXIT_CODE_INVALID_ARGUMENTS, "Check arguments.", e);
+ }
+ catch (Throwable e) {
+ if (isAuthError(e))
+ return error(ERR_AUTHENTICATION_FAILED, "Authentication error.", e);
+
+ if (isConnectionError(e))
+ return error(EXIT_CODE_CONNECTION_FAILED, "Connection to cluster failed.", e);
+
+
+ return error(EXIT_CODE_UNEXPECTED_ERROR, "", e);
+ }
+ }
+
+ /**
+ * @param args Arguments to parse and apply.
+ */
+ public static void main(String[] args) {
+ CommandHandler hnd = new CommandHandler();
+
+ System.exit(hnd.execute(args));
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
index a151eb5..74f5a10 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
@@ -558,6 +558,20 @@ public abstract class GridManagerAdapter<T extends IgniteSpi> implements GridMan
return null;
}
+ @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag discoData) {
+ for (GridComponent comp : ctx) {
+ if (comp.discoveryDataType() == null)
+ continue;
+
+ IgniteNodeValidationResult err = comp.validateNode(node, discoData.newJoinerDiscoveryData(comp.discoveryDataType().ordinal()));
+
+ if (err != null)
+ return err;
+ }
+
+ return null;
+ }
+
@Override public Collection<SecuritySubject> authenticatedSubjects() {
try {
return ctx.security().authenticatedSubjects();
@@ -705,6 +719,11 @@ public abstract class GridManagerAdapter<T extends IgniteSpi> implements GridMan
}
/** {@inheritDoc} */
+ @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag.JoiningNodeDiscoveryData discoData) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public final String toString() {
return S.toString(GridManagerAdapter.class, this, "name", getClass().getName());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdMapper.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdMapper.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdMapper.java
index c524331..59f773d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdMapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdMapper.java
@@ -19,10 +19,11 @@ package org.apache.ignite.internal.managers.discovery;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.Map;
import java.util.UUID;
-import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
@@ -31,71 +32,104 @@ import org.jetbrains.annotations.Nullable;
*/
public class ConsistentIdMapper {
/** Discovery manager. */
- private final GridDiscoveryManager discoveryManager;
+ private final GridDiscoveryManager discoveryMgr;
/**
* Create an instance of mapper.
*
- * @param discoveryManager Discovery manager.
+ * @param discoveryMgr Discovery manager.
*/
- public ConsistentIdMapper(GridDiscoveryManager discoveryManager) {
- this.discoveryManager = discoveryManager;
+ public ConsistentIdMapper(GridDiscoveryManager discoveryMgr) {
+ this.discoveryMgr = discoveryMgr;
}
/**
- * Map UUID to consistent id.
+ * Maps UUID to compact ID for given baseline topology.
*
* @param topVer Topology version.
* @param nodeId UUID of node.
- * @return Consistent id of node.
+ * @return Compact ID of node for given baseline topology.
*/
- public Object mapToConsistentId(AffinityTopologyVersion topVer, UUID nodeId) {
- ClusterNode node = discoveryManager.node(topVer, nodeId);
+ public short mapToCompactId(AffinityTopologyVersion topVer, UUID nodeId) {
+ Map<UUID, Short> m = discoveryMgr.consistentId(topVer);
- if (node == null)
- throw new IllegalStateException("Unable to find node by UUID [nodeId=" + nodeId + ", topVer=" + topVer + ']');
+ if (m == null)
+ throw new IllegalStateException("Unable to find consistent id map [topVer" + topVer + ']');
- return node.consistentId();
+ Short constId = m.get(nodeId);
+
+ if (constId == null)
+ throw new IllegalStateException("Unable to find consistentId by UUID [nodeId=" + nodeId + ", topVer=" + topVer + ']');
+
+ return constId;
}
/**
- * Map consistent id to UUID.
+ * Maps UUID to compact ID for given baseline topology.
*
- * @param consistentId Consistent id of node.
- * @return UUID of node.
+ * @param topVer Topology version.
+ * @param nodeConstId UUID of node.
+ * @return Compact ID of node for given baseline topology.
*/
- @Nullable public UUID mapToUUID(Object consistentId) {
- for (ClusterNode node : discoveryManager.allNodes())
- if (node.consistentId().equals(consistentId))
- return node.id();
+ public UUID mapToUuid(AffinityTopologyVersion topVer, short nodeConstId) {
+ Map<Short, UUID> map = discoveryMgr.nodeIdMap(topVer);
+
+ if (map == null)
+ return null;
+
+ UUID constId = map.get(nodeConstId);
- return null;
+ if (constId == null)
+ throw new IllegalStateException("Unable to find UUID by constId [nodeId=" + nodeConstId + ", topVer=" + topVer + ']');
+
+ return constId;
}
/**
- * Map primary -> backup node UUIDs to consistent ids.
+ * Map primary -> backup node compact ID accordingly to baseline topology..
*
* @param txNodes Primary -> backup UUID nodes.
- * @return Primary -> backup consistent id nodes.
+ * @return Primary -> backup compact ID nodes.
*/
- public Map<Object, Collection<Object>> mapToConsistentIds(AffinityTopologyVersion topVer, @Nullable Map<UUID, Collection<UUID>> txNodes) {
+ public Map<Short, Collection<Short>> mapToCompactIds(
+ AffinityTopologyVersion topVer,
+ @Nullable Map<UUID, Collection<UUID>> txNodes,
+ BaselineTopology baselineTop
+ ) {
if (txNodes == null)
return null;
- Map<Object, Collection<Object>> consistentMap = U.newHashMap(txNodes.keySet().size());
+ Map<Object, Short> constIdMap = baselineTop.consistentIdMapping();
+
+ Map<UUID, Short> m = discoveryMgr.consistentId(topVer);
- for (UUID node : txNodes.keySet()) {
- Collection<UUID> backupNodes = txNodes.get(node);
+ int bltNodes = m.size();
- Collection<Object> consistentIdsBackups = new ArrayList<>(backupNodes.size());
+ Map<Short, Collection<Short>> consistentMap = U.newHashMap(txNodes.size());
- for (UUID backup : backupNodes)
- consistentIdsBackups.add(mapToConsistentId(topVer, backup));
+ int nodeCnt = 0;
- consistentMap.put(mapToConsistentId(topVer, node), consistentIdsBackups);
+ for (Map.Entry<UUID, Collection<UUID>> e : txNodes.entrySet()) {
+ UUID node = e.getKey();
+
+ Collection<UUID> backupNodes = e.getValue();
+
+ Collection<Short> backups = new ArrayList<>(backupNodes.size());
+
+ for (UUID backup : backupNodes) {
+ if (m.containsKey(backup))
+ nodeCnt++;
+
+ backups.add(mapToCompactId(topVer, backup));
+ }
+
+ // Optimization for short store full nodes set.
+ if (backups.size() == nodeCnt && nodeCnt == (bltNodes - 1))
+ backups = Collections.singletonList(Short.MAX_VALUE);
+
+ consistentMap.put(mapToCompactId(topVer, node), backups);
}
return consistentMap;
}
-
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index 9ed70aa..aa47168 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -23,11 +23,13 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.ignite.cluster.BaselineNode;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.C1;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.P1;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -40,6 +42,13 @@ import org.jetbrains.annotations.Nullable;
*/
public class DiscoCache {
/** */
+ private static final C1<BaselineNode, ClusterNode> BASELINE_TO_CLUSTER = new C1<BaselineNode, ClusterNode>() {
+ @Override public ClusterNode apply(BaselineNode baselineNode) {
+ return (ClusterNode)baselineNode;
+ }
+ };
+
+ /** */
private final DiscoveryDataClusterState state;
/** Local node. */
@@ -57,6 +66,9 @@ public class DiscoCache {
/** Daemon nodes. */
private final List<ClusterNode> daemonNodes;
+ /** Baseline nodes. */
+ private final List<? extends BaselineNode> baselineNodes;
+
/** All remote nodes with at least one cache configured. */
@GridToStringInclude
private final List<ClusterNode> rmtNodesWithCaches;
@@ -81,6 +93,18 @@ public class DiscoCache {
/** */
private final AffinityTopologyVersion topVer;
+ /** */
+ final Map<UUID, Short> nodeIdToConsIdx;
+
+ /** */
+ final Map<Short, UUID> consIdxToNodeId;
+
+ /** */
+ private final P1<BaselineNode> aliveBaselineNodePred;
+
+ /** */
+ private final P1<ClusterNode> aliveNodePred;
+
/**
* @param topVer Topology version.
* @param state Current cluster state.
@@ -90,10 +114,13 @@ public class DiscoCache {
* @param srvNodes Server nodes.
* @param daemonNodes Daemon nodes.
* @param rmtNodesWithCaches Remote nodes with at least one cache configured.
+ * @param baselineNodes Baseline nodes or {@code null} if baseline was not set.
* @param allCacheNodes Cache nodes by cache name.
* @param cacheGrpAffNodes Affinity nodes by cache group ID.
* @param nodeMap Node map.
- * @param alives Alive nodes.
+ * @param alives0 Alive nodes.
+ * @param nodeIdToConsIdx Node ID to consistent ID mapping.
+ * @param consIdxToNodeId Consistent ID to node ID mapping.
* @param minNodeVer Minimum node version.
*/
DiscoCache(
@@ -105,11 +132,15 @@ public class DiscoCache {
List<ClusterNode> srvNodes,
List<ClusterNode> daemonNodes,
List<ClusterNode> rmtNodesWithCaches,
+ @Nullable List<? extends BaselineNode> baselineNodes,
Map<Integer, List<ClusterNode>> allCacheNodes,
Map<Integer, List<ClusterNode>> cacheGrpAffNodes,
Map<UUID, ClusterNode> nodeMap,
- Set<UUID> alives,
- IgniteProductVersion minNodeVer) {
+ Set<UUID> alives0,
+ @Nullable Map<UUID, Short> nodeIdToConsIdx,
+ @Nullable Map<Short, UUID> consIdxToNodeId,
+ IgniteProductVersion minNodeVer
+ ) {
this.topVer = topVer;
this.state = state;
this.loc = loc;
@@ -118,11 +149,28 @@ public class DiscoCache {
this.srvNodes = srvNodes;
this.daemonNodes = daemonNodes;
this.rmtNodesWithCaches = rmtNodesWithCaches;
+ this.baselineNodes = baselineNodes;
this.allCacheNodes = allCacheNodes;
this.cacheGrpAffNodes = cacheGrpAffNodes;
this.nodeMap = nodeMap;
- this.alives.addAll(alives);
+ alives.addAll(alives0);
this.minNodeVer = minNodeVer;
+ this.nodeIdToConsIdx = nodeIdToConsIdx;
+ this.consIdxToNodeId = consIdxToNodeId;
+
+ aliveBaselineNodePred = new P1<BaselineNode>() {
+ @Override
+ public boolean apply(BaselineNode node) {
+ return node instanceof ClusterNode && alives.contains(((ClusterNode)node).id());
+
+ }
+ };
+
+ aliveNodePred = new P1<ClusterNode>() {
+ @Override public boolean apply(ClusterNode node) {
+ return alives.contains(node.id());
+ }
+ };
}
/**
@@ -156,6 +204,15 @@ public class DiscoCache {
return rmtNodes;
}
+ /**
+ * Returns a collection of baseline nodes.
+ *
+ * @return A collection of baseline nodes or {@code null} if baseline topology was not set.
+ */
+ @Nullable public List<? extends BaselineNode> baselineNodes() {
+ return baselineNodes;
+ }
+
/** @return All nodes. */
public List<ClusterNode> allNodes() {
return allNodes;
@@ -171,17 +228,23 @@ public class DiscoCache {
return daemonNodes;
}
+ /** @return Consistent id map UUID -> Short (compacted consistent id). */
+ public Map<UUID, Short> consistentIdMap() {
+ return nodeIdToConsIdx;
+ }
+
+ /** @return Consistent id map Short (compacted consistent id) -> UUID. */
+ public Map<Short, UUID> nodeIdMap() {
+ return consIdxToNodeId;
+ }
+
/**
* Gets all alive remote nodes that have at least one cache configured.
*
* @return Collection of nodes.
*/
public Collection<ClusterNode> remoteAliveNodesWithCaches() {
- return F.view(rmtNodesWithCaches, new P1<ClusterNode>() {
- @Override public boolean apply(ClusterNode node) {
- return alives.contains(node.id());
- }
- });
+ return F.view(rmtNodesWithCaches, aliveNodePred);
}
/**
@@ -190,17 +253,26 @@ public class DiscoCache {
* @return Collection of nodes.
*/
public Collection<ClusterNode> aliveServerNodes() {
- return F.view(serverNodes(), new P1<ClusterNode>() {
- @Override public boolean apply(ClusterNode node) {
- return alives.contains(node.id());
- }
- });
+ return F.view(serverNodes(), aliveNodePred);
+ }
+
+ /**
+ * Returns a collection of live baseline nodes.
+ *
+ * @return A view of baseline nodes that are currently present in the cluster or {@code null} if baseline
+ * topology was not set.
+ */
+ @Nullable public Collection<ClusterNode> aliveBaselineNodes() {
+ return baselineNodes == null ? null : F.viewReadOnly(baselineNodes, BASELINE_TO_CLUSTER, aliveBaselineNodePred);
+
}
/**
* @return Oldest alive server node.
*/
- public @Nullable ClusterNode oldestAliveServerNode(){
+ @SuppressWarnings("ForLoopReplaceableByForEach")
+ @Nullable public ClusterNode oldestAliveServerNode(){
+ // Avoid iterator allocation.
for (int i = 0; i < srvNodes.size(); i++) {
ClusterNode srv = srvNodes.get(i);
@@ -330,10 +402,13 @@ public class DiscoCache {
srvNodes,
daemonNodes,
rmtNodesWithCaches,
+ baselineNodes,
allCacheNodes,
cacheGrpAffNodes,
nodeMap,
alives,
+ nodeIdToConsIdx,
+ consIdxToNodeId,
minNodeVer);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 088c9ce..65cc666 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -50,6 +50,7 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteInterruptedException;
import org.apache.ignite.cache.CacheMetrics;
import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.BaselineNode;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
@@ -59,10 +60,10 @@ import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.GridComponent;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.GridNodeOrderComparator;
import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
import org.apache.ignite.internal.events.DiscoveryCustomEvent;
import org.apache.ignite.internal.managers.GridManagerAdapter;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
@@ -74,12 +75,12 @@ import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest;
import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
-import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
+import org.apache.ignite.internal.processors.cluster.IGridClusterStateProcessor;
import org.apache.ignite.internal.processors.security.SecurityContext;
import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
import org.apache.ignite.internal.util.GridAtomicLong;
@@ -705,11 +706,19 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
Snapshot snapshot = topSnap.get();
if (customMsg == null) {
- discoCache = createDiscoCache(nextTopVer,
+ discoCache = createDiscoCache(
+ nextTopVer,
ctx.state().clusterState(),
locNode,
topSnapshot);
}
+ else if (customMsg instanceof ChangeGlobalStateMessage) {
+ discoCache = createDiscoCache(
+ nextTopVer,
+ ctx.state().pendingState((ChangeGlobalStateMessage)customMsg),
+ locNode,
+ topSnapshot);
+ }
else
discoCache = customMsg.createDiscoCache(GridDiscoveryManager.this, nextTopVer, snapshot.discoCache);
@@ -784,7 +793,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
topSnap.set(new Snapshot(AffinityTopologyVersion.ZERO,
createDiscoCache(AffinityTopologyVersion.ZERO, ctx.state().clusterState(), locNode,
- Collections.<ClusterNode>singleton(locNode))
+ Collections.singleton(locNode))
));
}
else if (type == EVT_CLIENT_NODE_RECONNECTED) {
@@ -846,7 +855,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
if (ctx.localNodeId().equals(dataBag.joiningNodeId())) {
// NodeAdded msg reached joining node after round-trip over the ring.
- GridClusterStateProcessor stateProc = ctx.state();
+ IGridClusterStateProcessor stateProc = ctx.state();
stateProc.onGridDataReceived(dataBag.gridDiscoveryData(
stateProc.discoveryDataType().ordinal()));
@@ -858,7 +867,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
}
else {
// Discovery data from newly joined node has to be applied to the current old node.
- GridClusterStateProcessor stateProc = ctx.state();
+ IGridClusterStateProcessor stateProc = ctx.state();
JoiningNodeDiscoveryData data0 = dataBag.newJoinerDiscoveryData(
stateProc.discoveryDataType().ordinal());
@@ -1810,6 +1819,15 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
}
/**
+ * @param topVer Topology version.
+ * @return All baseline nodes for given topology version or {@code null} if baseline was not set for the
+ * given topology version.
+ */
+ @Nullable public List<? extends BaselineNode> baselineNodes(AffinityTopologyVersion topVer) {
+ return resolveDiscoCache(CU.cacheId(null), topVer).baselineNodes();
+ }
+
+ /**
* Gets node from history for given topology version.
*
* @param topVer Topology version.
@@ -1821,6 +1839,26 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
}
/**
+ * Gets consistentId from history for given topology version.
+ *
+ * @param topVer Topology version.
+ * @return Compacted consistent id.
+ */
+ public Map<UUID, Short> consistentId(AffinityTopologyVersion topVer) {
+ return resolveDiscoCache(CU.cacheId(null), topVer).consistentIdMap();
+ }
+
+ /**
+ * Gets consistentId from history for given topology version.
+ *
+ * @param topVer Topology version.
+ * @return Compacted consistent id map.
+ */
+ public Map<Short, UUID> nodeIdMap(AffinityTopologyVersion topVer) {
+ return resolveDiscoCache(CU.cacheId(null), topVer).nodeIdMap();
+ }
+
+ /**
* Gets cache nodes for cache with given name.
*
* @param cacheName Cache name.
@@ -2004,7 +2042,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
/**
* @return Consistent ID.
- * @deprecated Use PdsConsistentIdProcessor to get actual consistent ID
+ * @deprecated Use {@link ClusterNode#consistentId()} of local node to get actual consistent ID.
*/
@Deprecated
public Serializable consistentId() {
@@ -2206,6 +2244,10 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
ArrayList<ClusterNode> rmtNodes = new ArrayList<>(topSnapshot.size());
ArrayList<ClusterNode> allNodes = new ArrayList<>(topSnapshot.size());
+ Map<UUID, Short> nodeIdToConsIdx;
+ Map<Short, UUID> consIdxToNodeId;
+ List<? extends BaselineNode> baselineNodes;
+
IgniteProductVersion minVer = null;
for (ClusterNode node : topSnapshot) {
@@ -2237,10 +2279,52 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
Map<Integer, List<ClusterNode>> allCacheNodes = U.newHashMap(allNodes.size());
Map<Integer, List<ClusterNode>> cacheGrpAffNodes = U.newHashMap(allNodes.size());
- Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(GridNodeOrderComparator.INSTANCE);
+ Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(NodeOrderComparator.getInstance());
fillAffinityNodeCaches(allNodes, allCacheNodes, cacheGrpAffNodes, rmtNodesWithCaches);
+ BaselineTopology blt = state.baselineTopology();
+
+ if (blt != null) {
+ nodeIdToConsIdx = U.newHashMap(srvNodes.size());
+ consIdxToNodeId = U.newHashMap(srvNodes.size());
+
+ Map<Object, Short> m = blt.consistentIdMapping();
+
+ Map<Object, ClusterNode> aliveNodesByConsId = U.newHashMap(srvNodes.size());
+
+ for (ClusterNode node : srvNodes) {
+ Short compactedId = m.get(node.consistentId());
+
+ if (compactedId != null) {
+ nodeIdToConsIdx.put(node.id(), compactedId);
+
+ consIdxToNodeId.put(compactedId, node.id());
+ }
+
+ aliveNodesByConsId.put(node.consistentId(), node);
+ }
+
+ List<BaselineNode >baselineNodes0 = new ArrayList<>(blt.size());
+
+ for (Object consId : blt.consistentIds()) {
+ ClusterNode srvNode = aliveNodesByConsId.get(consId);
+
+ if (srvNode != null)
+ baselineNodes0.add(srvNode);
+ else
+ baselineNodes0.add(blt.baselineNode(consId));
+ }
+
+ baselineNodes = baselineNodes0;
+ }
+ else {
+ nodeIdToConsIdx = null;
+ consIdxToNodeId = null;
+
+ baselineNodes = null;
+ }
+
return new DiscoCache(
topVer,
state,
@@ -2250,10 +2334,13 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
Collections.unmodifiableList(srvNodes),
Collections.unmodifiableList(daemonNodes),
U.sealList(rmtNodesWithCaches),
+ baselineNodes == null ? null : Collections.unmodifiableList(baselineNodes),
Collections.unmodifiableMap(allCacheNodes),
Collections.unmodifiableMap(cacheGrpAffNodes),
Collections.unmodifiableMap(nodeMap),
alives,
+ nodeIdToConsIdx == null ? null : Collections.unmodifiableMap(nodeIdToConsIdx),
+ consIdxToNodeId == null ? null : Collections.unmodifiableMap(consIdxToNodeId),
minVer);
}
@@ -2364,8 +2451,12 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
discoWrk.addEvent(EVT_NODE_SEGMENTED,
AffinityTopologyVersion.NONE,
node,
- createDiscoCache(AffinityTopologyVersion.NONE, null, node, locNodeOnlyTop),
- locNodeOnlyTop,
+ createDiscoCache(
+ AffinityTopologyVersion.NONE,
+ ctx.state().clusterState(),
+ node,
+ locNodeOnlyTop
+ ), locNodeOnlyTop,
null);
lastSegChkRes.set(false);
@@ -3056,12 +3147,14 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
* @param discoCache Current disco cache.
* @return New discovery cache.
*/
- public DiscoCache createDiscoCacheOnCacheChange(AffinityTopologyVersion topVer,
- DiscoCache discoCache) {
+ public DiscoCache createDiscoCacheOnCacheChange(
+ AffinityTopologyVersion topVer,
+ DiscoCache discoCache
+ ) {
List<ClusterNode> allNodes = discoCache.allNodes();
Map<Integer, List<ClusterNode>> allCacheNodes = U.newHashMap(allNodes.size());
Map<Integer, List<ClusterNode>> cacheGrpAffNodes = U.newHashMap(allNodes.size());
- Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(GridNodeOrderComparator.INSTANCE);
+ Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(NodeOrderComparator.getInstance());
fillAffinityNodeCaches(allNodes, allCacheNodes, cacheGrpAffNodes, rmtNodesWithCaches);
@@ -3074,10 +3167,13 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
discoCache.serverNodes(),
discoCache.daemonNodes(),
U.sealList(rmtNodesWithCaches),
+ discoCache.baselineNodes(),
allCacheNodes,
cacheGrpAffNodes,
discoCache.nodeMap,
discoCache.alives,
+ discoCache.nodeIdToConsIdx,
+ discoCache.consIdxToNodeId,
discoCache.minimumNodeVersion());
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
index bf6e807..276e10e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
@@ -57,13 +57,15 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider {
/** {@inheritDoc} */
@Override public void shutdown() {
- for (Iterator<DirectMemoryRegion> it = regions.iterator(); it.hasNext(); ) {
- DirectMemoryRegion chunk = it.next();
+ if (regions != null) {
+ for (Iterator<DirectMemoryRegion> it = regions.iterator(); it.hasNext(); ) {
+ DirectMemoryRegion chunk = it.next();
- GridUnsafe.freeMemory(chunk.address());
+ GridUnsafe.freeMemory(chunk.address());
- // Safety.
- it.remove();
+ // Safety.
+ it.remove();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index 64c5927..6802a3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -52,6 +52,11 @@ public interface IgnitePageStoreManager extends GridCacheSharedManager, IgniteCh
throws IgniteCheckedException;
/**
+ * Initializes disk cache store structures.
+ */
+ public void initializeForMetastorage() throws IgniteCheckedException;
+
+ /**
* Callback called when a cache is stopping. After this callback is invoked, no data associated with
* the given cache will be stored on disk.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 42d9611..19b47e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -96,10 +96,11 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
* the given pointer will be kept because there is a configurable WAL history size. Those entries may be used
* for partial partition rebalancing.
*
- * @param ptr Pointer for which it is safe to clear the log.
+ * @param low Pointer since which WAL will be truncated. If null, WAL will be truncated from the oldest segment.
+ * @param high Pointer for which it is safe to clear the log.
* @return Number of deleted WAL segments.
*/
- public int truncate(WALPointer ptr);
+ public int truncate(WALPointer low, WALPointer high);
/**
* Gives a hint to WAL manager to compact WAL until given pointer (exclusively).
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java
new file mode 100644
index 0000000..48b60b3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record;
+
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Record for storing baseline topology compact node ID to consistent node ID mapping.
+ */
+public class BaselineTopologyRecord extends WALRecord {
+ /** Id. */
+ private int id;
+
+ /** Compact ID to consistent ID mapping. */
+ private Map<Short, Object> mapping;
+
+ /**
+ * Default constructor.
+ */
+ private BaselineTopologyRecord() {
+ // No-op, used from factory methods.
+ }
+
+ /**
+ * @param id Baseline topology ID.
+ * @param mapping Compact ID to consistent ID mapping.
+ */
+ public BaselineTopologyRecord(int id, Map<Short, Object> mapping) {
+ this.id = id;
+ this.mapping = mapping;
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordType type() {
+ return RecordType.BASELINE_TOP_RECORD;
+ }
+
+ /**
+ * Returns baseline topology ID.
+ *
+ * @return Baseline topology ID.
+ */
+ public int id() {
+ return id;
+ }
+
+ /**
+ * Returns mapping.
+ *
+ * @return Compact ID to consistent ID mapping.
+ */
+ public Map<Short, Object> mapping() {
+ return mapping;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(BaselineTopologyRecord.class, this, "super", super.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/CacheState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/CacheState.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/CacheState.java
index 41d38d0..49025c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/CacheState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/CacheState.java
@@ -30,6 +30,9 @@ public class CacheState {
private long[] vals;
/** */
+ private byte[] states;
+
+ /** */
private int idx;
/**
@@ -38,6 +41,7 @@ public class CacheState {
public CacheState(int partsCnt) {
parts = new int[partsCnt];
vals = new long[partsCnt * 2];
+ states = new byte[partsCnt];
}
/**
@@ -46,6 +50,16 @@ public class CacheState {
* @param cntr Partition counter.
*/
public void addPartitionState(int partId, long size, long cntr) {
+ addPartitionState(partId, size, cntr, (byte)-1);
+ }
+
+ /**
+ * @param partId Partition ID to add.
+ * @param size Partition size.
+ * @param cntr Partition counter.
+ * @param state Partition state.
+ */
+ public void addPartitionState(int partId, long size, long cntr, byte state) {
if (idx == parts.length)
throw new IllegalStateException("Failed to add new partition to the partitions state " +
"(no enough space reserved) [partId=" + partId + ", reserved=" + parts.length + ']');
@@ -57,6 +71,8 @@ public class CacheState {
}
parts[idx] = partId;
+ states[idx] = state;
+
vals[2 * idx] = size;
vals[2 * idx + 1] = cntr;
@@ -97,6 +113,14 @@ public class CacheState {
/**
* @param idx Index to get.
+ * @return State partition.
+ */
+ public byte stateByIndex(int idx) {
+ return states[idx];
+ }
+
+ /**
+ * @param idx Index to get.
* @return Partition size by index.
*/
public long partitionSizeByIndex(int idx) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
index cb6b482..3511aff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
@@ -97,8 +97,8 @@ public class DataEntry {
this.partId = partId;
this.partCnt = partCnt;
- // Only CREATE, UPDATE and DELETE operations should be stored in WAL.
- assert op == GridCacheOperation.CREATE || op == GridCacheOperation.UPDATE || op == GridCacheOperation.DELETE : op;
+ // Only READ, CREATE, UPDATE and DELETE operations should be stored in WAL.
+ assert op == GridCacheOperation.READ || op == GridCacheOperation.CREATE || op == GridCacheOperation.UPDATE || op == GridCacheOperation.DELETE : op;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
index ac569bd..7a4d6b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
/**
* Logical data record with cache operation description.
@@ -48,19 +49,18 @@ public class DataRecord extends TimeStampRecord {
* @param writeEntry Write entry.
*/
public DataRecord(DataEntry writeEntry) {
- this(Collections.singletonList(writeEntry));
+ this(writeEntry, U.currentTimeMillis());
}
/**
* @param writeEntries Write entries.
*/
public DataRecord(List<DataEntry> writeEntries) {
- this.writeEntries = writeEntries;
+ this(writeEntries, U.currentTimeMillis());
}
/**
* @param writeEntry Write entry.
- * @param timestamp TimeStamp.
*/
public DataRecord(DataEntry writeEntry, long timestamp) {
this(Collections.singletonList(writeEntry), timestamp);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ExchangeRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ExchangeRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ExchangeRecord.java
new file mode 100644
index 0000000..87bc4e6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/ExchangeRecord.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Partition map exchange WAL record.
+ */
+public class ExchangeRecord extends TimeStampRecord {
+ /** Event. */
+ private Short constId;
+
+ /** Type. */
+ private Type type;
+
+ /**
+ * @param constId Const id.
+ * @param type Type.
+ * @param timeStamp TimeStamp.
+ */
+ public ExchangeRecord(Short constId, Type type, long timeStamp) {
+ super(timeStamp);
+
+ this.constId = constId;
+ this.type = type;
+ }
+
+ /**
+ * @param constId Const id.
+ * @param type Type.
+ */
+ public ExchangeRecord(Short constId, Type type) {
+ this.constId = constId;
+ this.type = type;
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordType type() {
+ return RecordType.EXCHANGE;
+ }
+
+ /**
+ *
+ */
+ public Short getConstId() {
+ return constId;
+ }
+
+ /**
+ *
+ */
+ public Type getType() {
+ return type;
+ }
+
+ /**
+ *
+ */
+ public enum Type {
+ /** Join. */
+ JOIN,
+ /** Left. */
+ LEFT
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ExchangeRecord.class, this, "super", super.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
index 519e825..e077e5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/FilteredRecord.java
@@ -24,6 +24,9 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRe
* {@link AbstractWalRecordsIterator}.
*/
public class FilteredRecord extends WALRecord {
+ /** Instance. */
+ public static final FilteredRecord INSTANCE = new FilteredRecord();
+
/** {@inheritDoc} */
@Override public RecordType type() {
return null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
new file mode 100644
index 0000000..e269de2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.pagemem.wal.record;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class MetastoreDataRecord extends WALRecord {
+ /** */
+ private final String key;
+
+ /** */
+ @Nullable private final byte[] value;
+
+ /**
+ * @param key Key.
+ * @param value Value.
+ */
+ public MetastoreDataRecord(String key, @Nullable byte[] value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ /** */
+ public String key() {
+ return key;
+ }
+
+ /** */
+ @Nullable public byte[] value() {
+ return value;
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordType type() {
+ return RecordType.METASTORE_DATA_RECORD;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(MetastoreDataRecord.class, this, "super", super.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
index 6ee96a4..6740221 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/PageSnapshot.java
@@ -25,6 +25,7 @@ import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import sun.nio.ch.DirectBuffer;
/**
*
@@ -92,9 +93,12 @@ public class PageSnapshot extends WALRecord {
+ "],\nsuper = ["
+ super.toString() + "]]";
}
- catch (IgniteCheckedException e) {
+ catch (IgniteCheckedException ignored) {
return "Error during call'toString' of PageSnapshot [fullPageId=" + fullPageId() +
", pageData = " + Arrays.toString(pageData) + ", super=" + super.toString() + "]";
}
+ finally {
+ ((DirectBuffer)buf).cleaner().clean();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java
index 3c3a77b..c6b6329 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/SnapshotRecord.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.pagemem.wal.record;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
/**
* Wal snapshot record.
*/
@@ -49,10 +51,20 @@ public class SnapshotRecord extends WALRecord {
return full;
}
+ /** {@inheritDoc} */
+ @Override public boolean rollOver() {
+ return true;
+ }
+
/**
*
*/
@Override public RecordType type() {
return RecordType.SNAPSHOT;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(SnapshotRecord.class, this, "super", super.toString());
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TimeStampRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TimeStampRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TimeStampRecord.java
index 3f29dfd..c1b8584 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TimeStampRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TimeStampRecord.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.pagemem.wal.record;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
/**
@@ -25,6 +27,7 @@ import org.apache.ignite.internal.util.typedef.internal.U;
*/
public abstract class TimeStampRecord extends WALRecord {
/** Timestamp. */
+ @GridToStringInclude
protected long timestamp;
/**
@@ -54,4 +57,9 @@ public abstract class TimeStampRecord extends WALRecord {
public long timestamp() {
return timestamp;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(TimeStampRecord.class, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TxRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TxRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TxRecord.java
index f933fa9..90df004 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TxRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/TxRecord.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.pagemem.wal.record;
import java.util.Collection;
import java.util.Map;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.transactions.TransactionState;
import org.jetbrains.annotations.Nullable;
@@ -30,9 +31,11 @@ import org.jetbrains.annotations.Nullable;
*/
public class TxRecord extends TimeStampRecord {
/** Transaction state. */
+ @GridToStringInclude
private TransactionState state;
/** Global transaction identifier within cluster, assigned by transaction coordinator. */
+ @GridToStringInclude
private GridCacheVersion nearXidVer;
/** Transaction entries write topology version. */
@@ -42,33 +45,27 @@ public class TxRecord extends TimeStampRecord {
* Transaction participating nodes.
*
* Structure:
- * Primary node -> [Backup nodes...]
+ * Primary node -> [Backup nodes...], where nodes are identified by compact ID for some baseline topology.
**/
- @Nullable private Map<Object, Collection<Object>> participatingNodes;
-
- /** If transaction is remote, primary node for this backup node. */
- @Nullable private Object primaryNode;
+ @Nullable private Map<Short, Collection<Short>> participatingNodes;
/**
*
* @param state Transaction state.
* @param nearXidVer Transaction id.
* @param writeVer Transaction entries write topology version.
- * @param participatingNodes Primary -> Backup nodes participating in transaction.
- * @param primaryNode Primary node.
+ * @param participatingNodes Primary -> Backup nodes compact IDs participating in transaction.
*/
public TxRecord(
TransactionState state,
GridCacheVersion nearXidVer,
GridCacheVersion writeVer,
- @Nullable Map<Object, Collection<Object>> participatingNodes,
- @Nullable Object primaryNode
+ @Nullable Map<Short, Collection<Short>> participatingNodes
) {
this.state = state;
this.nearXidVer = nearXidVer;
this.writeVer = writeVer;
this.participatingNodes = participatingNodes;
- this.primaryNode = primaryNode;
}
/**
@@ -76,24 +73,21 @@ public class TxRecord extends TimeStampRecord {
* @param nearXidVer Transaction id.
* @param writeVer Transaction entries write topology version.
* @param participatingNodes Primary -> Backup nodes participating in transaction.
- * @param primaryNode Primary node.
- * @param timestamp TimeStamp.
+ * @param ts TimeStamp.
*/
public TxRecord(
TransactionState state,
GridCacheVersion nearXidVer,
GridCacheVersion writeVer,
- @Nullable Map<Object, Collection<Object>> participatingNodes,
- @Nullable Object primaryNode,
- long timestamp
+ @Nullable Map<Short, Collection<Short>> participatingNodes,
+ long ts
) {
- super(timestamp);
+ super(ts);
this.state = state;
this.nearXidVer = nearXidVer;
this.writeVer = writeVer;
this.participatingNodes = participatingNodes;
- this.primaryNode = primaryNode;
}
/** {@inheritDoc} */
@@ -144,33 +138,19 @@ public class TxRecord extends TimeStampRecord {
}
/**
- * @return Primary -> backup participating nodes.
+ * @return Primary -> backup participating nodes compact IDs.
*/
- public Map<Object, Collection<Object>> participatingNodes() {
+ public Map<Short, Collection<Short>> participatingNodes() {
return participatingNodes;
}
/**
- * @param participatingNodeIds Primary -> backup participating nodes.
+ * @param participatingNodeIds Primary -> backup participating nodes compact IDs.
*/
- public void participatingNodes(Map<Object, Collection<Object>> participatingNodeIds) {
+ public void participatingNodes(Map<Short, Collection<Short>> participatingNodeIds) {
this.participatingNodes = participatingNodeIds;
}
- /**
- * @return Is transaction remote for backup.
- */
- public boolean remote() {
- return primaryNode != null;
- }
-
- /**
- * @return Primary node for backup if transaction is remote.
- */
- @Nullable public Object primaryNode() {
- return primaryNode;
- }
-
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(TxRecord.class, this, "super", super.toString());
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
index 7446916..8362a69 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
@@ -166,7 +166,16 @@ public abstract class WALRecord {
PARTITION_DESTROY,
/** Snapshot record. */
- SNAPSHOT;
+ SNAPSHOT,
+
+ /** Metastore data record. */
+ METASTORE_DATA_RECORD,
+
+ /** Exchange record. */
+ EXCHANGE,
+
+ /** Baseline topology record. */
+ BASELINE_TOP_RECORD;
/** */
private static final RecordType[] VALS = RecordType.values();
@@ -241,6 +250,13 @@ public abstract class WALRecord {
}
/**
+ * @return Need wal rollOver.
+ */
+ public boolean rollOver(){
+ return false;
+ }
+
+ /**
* @return Entry type.
*/
public abstract RecordType type();
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
index e07c388..5324d56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
@@ -19,7 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -54,7 +55,7 @@ public class DataPageInsertFragmentRecord extends PageDeltaRecord {
/** {@inheritDoc} */
@Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
- DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+ AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
io.addRowFragment(pageAddr, payload, lastLink, pageMem.pageSize());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
index f315058..2c9a8e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
@@ -19,7 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -55,7 +56,7 @@ public class DataPageInsertRecord extends PageDeltaRecord {
@Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
assert payload != null;
- DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+ AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
io.addRow(pageAddr, payload, pageMem.pageSize());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
index 484ec87..f7776be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
@@ -19,7 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
@@ -50,7 +51,7 @@ public class DataPageRemoveRecord extends PageDeltaRecord {
/** {@inheritDoc} */
@Override public void applyDelta(PageMemory pageMem, long pageAddr)
throws IgniteCheckedException {
- DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+ AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
io.removeRow(pageAddr, itemId, pageMem.pageSize());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
index 0ade484..e679611 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
@@ -19,7 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.typedef.internal.S;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_PAGE_SET_FREE_LIST_PAGE;
@@ -51,7 +52,7 @@ public class DataPageSetFreeListPageRecord extends PageDeltaRecord {
/** {@inheritDoc} */
@Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
- DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+ AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
io.setFreeListPageId(pageAddr, freeListPage);
}
[05/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java
new file mode 100644
index 0000000..a3559bd
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTaskResult.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Result for {@link VisorBaselineTask}.
+ */
+public class VisorBaselineTaskResult extends VisorDataTransferObject {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Cluster state. */
+ private boolean active;
+
+ /** Current topology version. */
+ private long topVer;
+
+ /** Current baseline nodes. */
+ private Map<String, VisorBaselineNode> baseline;
+
+ /** Current server nodes. */
+ private Map<String, VisorBaselineNode> servers;
+
+ /**
+ * Default constructor.
+ */
+ public VisorBaselineTaskResult() {
+ // No-op.
+ }
+
+ /**
+ * @param nodes Nodes to process.
+ * @return Map of DTO objects.
+ */
+ private static Map<String, VisorBaselineNode> toMap(Collection<? extends BaselineNode> nodes) {
+ if (F.isEmpty(nodes))
+ return null;
+
+ Map<String, VisorBaselineNode> map = new TreeMap<>();
+
+ for (BaselineNode node : nodes) {
+ VisorBaselineNode dto = new VisorBaselineNode(node);
+
+ map.put(dto.getConsistentId(), dto);
+ }
+
+ return map;
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param active Cluster state.
+ * @param topVer Current topology version.
+ * @param baseline Current baseline nodes.
+ * @param servers Current server nodes.
+ */
+ public VisorBaselineTaskResult(
+ boolean active,
+ long topVer,
+ Collection<? extends BaselineNode> baseline,
+ Collection<? extends BaselineNode> servers) {
+ this.active = active;
+ this.topVer = topVer;
+ this.baseline = toMap(baseline);
+ this.servers = toMap(servers);
+ }
+
+ /**
+ * @return Cluster state.
+ */
+ public boolean isActive() {
+ return active;
+ }
+
+ /**
+ * @return Current topology version.
+ */
+ public long getTopologyVersion() {
+ return topVer;
+ }
+
+ /**
+ * @return Baseline nodes.
+ */
+ public Map<String, VisorBaselineNode> getBaseline() {
+ return baseline;
+ }
+
+ /**
+ * @return Server nodes.
+ */
+ public Map<String, VisorBaselineNode> getServers() {
+ return servers;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+ out.writeBoolean(active);
+ out.writeLong(topVer);
+ U.writeMap(out, baseline);
+ U.writeMap(out, servers);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+ active = in.readBoolean();
+ topVer = in.readLong();
+ baseline = U.readTreeMap(in);
+ servers = U.readTreeMap(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorBaselineTaskResult.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index 48c3abb..4e41537 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -301,6 +301,9 @@ public class VisorGatewayTask implements ComputeTask<Object[], Object> {
return res;
}
+ if (cls.isEnum())
+ return Enum.valueOf(cls, val);
+
return val;
}
@@ -360,10 +363,11 @@ public class VisorGatewayTask implements ComputeTask<Object[], Object> {
// Length of arguments that required to constructor by influence of nested complex objects.
int needArgs = args;
- for (Class type: types)
+ for (Class type: types) {
// When constructor required specified types increase length of required arguments.
if (TYPE_ARG_LENGTH.containsKey(type))
needArgs += TYPE_ARG_LENGTH.get(type);
+ }
if (needArgs == beanArgsCnt) {
Object[] initArgs = new Object[args];
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index 07ba214..1754cc8 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -54,6 +54,7 @@ import org.apache.ignite.plugin.extensions.communication.MessageWriter;
import org.apache.ignite.plugin.security.SecuritySubject;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
@@ -899,6 +900,11 @@ public abstract class IgniteSpiAdapter implements IgniteSpi {
}
/** {@inheritDoc} */
+ @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag discoData) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
@Override public Collection<SecuritySubject> authenticatedSubjects() {
return Collections.emptyList();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
index 96b3e61..108c4d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
@@ -31,6 +31,7 @@ import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.plugin.extensions.communication.MessageFactory;
import org.apache.ignite.plugin.extensions.communication.MessageFormatter;
import org.apache.ignite.plugin.security.SecuritySubject;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
import org.jetbrains.annotations.Nullable;
/**
@@ -296,6 +297,12 @@ public interface IgniteSpiContext {
@Nullable public IgniteNodeValidationResult validateNode(ClusterNode node);
/**
+ * @param node Node.
+ * @param discoData Disco data.
+ */
+ @Nullable public IgniteNodeValidationResult validateNode(ClusterNode node, DiscoveryDataBag discoData);
+
+ /**
* Gets collection of authenticated subjects together with their permissions.
*
* @return Collection of authenticated subjects.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index b9f381b..6f79720 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -3574,36 +3574,43 @@ class ServerImpl extends TcpDiscoveryImpl {
}
}
- final IgniteNodeValidationResult err = spi.getSpiContext().validateNode(node);
+ IgniteNodeValidationResult err;
+
+ err = spi.getSpiContext().validateNode(node);
+
+ if (err == null)
+ err = spi.getSpiContext().validateNode(node, msg.gridDiscoveryData().unmarshalJoiningNodeData(spi.marshaller(), U.resolveClassLoader(spi.ignite().configuration()), false, log));
if (err != null) {
+ final IgniteNodeValidationResult err0 = err;
+
if (log.isDebugEnabled())
log.debug("Node validation failed [res=" + err + ", node=" + node + ']');
utilityPool.execute(
new Runnable() {
@Override public void run() {
- boolean ping = node.id().equals(err.nodeId()) ? pingNode(node) : pingNode(err.nodeId());
+ boolean ping = node.id().equals(err0.nodeId()) ? pingNode(node) : pingNode(err0.nodeId());
if (!ping) {
if (log.isDebugEnabled())
log.debug("Conflicting node has already left, need to wait for event. " +
"Will ignore join request for now since it will be recent [req=" + msg +
- ", err=" + err.message() + ']');
+ ", err=" + err0.message() + ']');
// Ignore join request.
return;
}
- LT.warn(log, err.message());
+ LT.warn(log, err0.message());
// Always output in debug.
if (log.isDebugEnabled())
- log.debug(err.message());
+ log.debug(err0.message());
try {
trySendMessageDirectly(node,
- new TcpDiscoveryCheckFailedMessage(err.nodeId(), err.sendMessage()));
+ new TcpDiscoveryCheckFailedMessage(err0.nodeId(), err0.sendMessage()));
}
catch (IgniteSpiException e) {
if (log.isDebugEnabled())
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 3b83b2e..713bbab 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -972,17 +972,23 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
initAddresses();
- final Serializable cfgId = ignite.configuration().getConsistentId();
+ IgniteConfiguration cfg = ignite.configuration();
+
+ final Serializable cfgId = cfg.getConsistentId();
if (cfgId == null) {
- final List<String> sortedAddrs = new ArrayList<>(addrs.get1());
+ if (cfg.isClientMode() == Boolean.TRUE)
+ consistentId = cfg.getNodeId();
+ else {
+ List<String> sortedAddrs = new ArrayList<>(addrs.get1());
- Collections.sort(sortedAddrs);
+ Collections.sort(sortedAddrs);
- if (getBoolean(IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT))
- consistentId = U.consistentId(sortedAddrs);
- else
- consistentId = U.consistentId(sortedAddrs, impl.boundPort());
+ if (getBoolean(IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT))
+ consistentId = U.consistentId(sortedAddrs);
+ else
+ consistentId = U.consistentId(sortedAddrs, impl.boundPort());
+ }
}
else
consistentId = cfgId;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java
index 1134de6..5f70f3c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/DiscoveryDataPacket.java
@@ -48,6 +48,9 @@ public class DiscoveryDataPacket implements Serializable {
private Map<Integer, byte[]> joiningNodeData = new HashMap<>();
/** */
+ private transient Map<Integer, Serializable> unmarshalledJoiningNodeData;
+
+ /** */
private Map<Integer, byte[]> commonData = new HashMap<>();
/** */
@@ -154,14 +157,14 @@ public class DiscoveryDataPacket implements Serializable {
DiscoveryDataBag dataBag = new DiscoveryDataBag(joiningNodeId);
if (joiningNodeData != null && !joiningNodeData.isEmpty()) {
- Map<Integer, Serializable> unmarshJoiningNodeData = unmarshalData(
+ unmarshalledJoiningNodeData = unmarshalData(
joiningNodeData,
marsh,
clsLdr,
clientNode,
log);
- dataBag.joiningNodeData(unmarshJoiningNodeData);
+ dataBag.joiningNodeData(unmarshalledJoiningNodeData);
}
return dataBag;
@@ -340,6 +343,11 @@ public class DiscoveryDataPacket implements Serializable {
* (e.g. on nodes prior in cluster to the one where this method is called).
*/
public DiscoveryDataBag bagForDataCollection() {
- return new DiscoveryDataBag(joiningNodeId, commonData.keySet());
+ DiscoveryDataBag dataBag = new DiscoveryDataBag(joiningNodeId, commonData.keySet());
+
+ if (unmarshalledJoiningNodeData != null)
+ dataBag.joiningNodeData(unmarshalledJoiningNodeData);
+
+ return dataBag;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
index 38c2a1b..01534f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
@@ -623,8 +623,6 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Cluste
Object consistentIdAttr = attrs.get(ATTR_NODE_CONSISTENT_ID);
- consistentId = consistentIdAttr != null ? consistentIdAttr : U.consistentId(addrs, discPort);
-
// Cluster metrics
byte[] mtr = U.readByteArray(in);
@@ -648,6 +646,11 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Cluste
intOrder = in.readLong();
ver = (IgniteProductVersion)in.readObject();
clientRouterNodeId = U.readUuid(in);
+
+ if (isClient())
+ consistentId = consistentIdAttr != null ? consistentIdAttr : id;
+ else
+ consistentId = consistentIdAttr != null ? consistentIdAttr : U.consistentId(addrs, discPort);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/resources/META-INF/classnames.properties
----------------------------------------------------------------------
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index c27681e..cb70857 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -209,7 +209,6 @@ org.apache.ignite.internal.GridKernalGatewayImpl
org.apache.ignite.internal.GridKernalState
org.apache.ignite.internal.GridLoggerProxy
org.apache.ignite.internal.GridMessageListenHandler
-org.apache.ignite.internal.GridNodeOrderComparator
org.apache.ignite.internal.GridTaskCancelRequest
org.apache.ignite.internal.GridTaskNameHashKey
org.apache.ignite.internal.GridTaskSessionRequest
@@ -302,6 +301,8 @@ org.apache.ignite.internal.cluster.IgniteClusterImpl
org.apache.ignite.internal.cluster.IgniteClusterImpl$1
org.apache.ignite.internal.cluster.IgniteKillTask
org.apache.ignite.internal.cluster.IgniteKillTask$IgniteKillJob
+org.apache.ignite.internal.cluster.NodeOrderComparator
+org.apache.ignite.internal.cluster.NodeOrderLegacyComparator
org.apache.ignite.internal.compute.ComputeTaskCancelledCheckedException
org.apache.ignite.internal.compute.ComputeTaskTimeoutCheckedException
org.apache.ignite.internal.direct.DirectMessageReader$1
@@ -311,6 +312,14 @@ org.apache.ignite.internal.executor.GridExecutorService
org.apache.ignite.internal.executor.GridExecutorService$1
org.apache.ignite.internal.executor.GridExecutorService$TaskTerminateListener
org.apache.ignite.internal.igfs.common.IgfsIpcCommand
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$1
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$BooleanProperty
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$ConnectionProperty
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$IntegerProperty
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$NumberProperty
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$PropertyValidator
+org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$StringProperty
org.apache.ignite.internal.jdbc2.JdbcBatchUpdateTask
org.apache.ignite.internal.jdbc2.JdbcConnection$JdbcConnectionValidationTask
org.apache.ignite.internal.jdbc2.JdbcDatabaseMetadata$UpdateMetadataTask
@@ -335,11 +344,14 @@ org.apache.ignite.internal.managers.deployment.GridDeploymentResponse
org.apache.ignite.internal.managers.discovery.CustomMessageWrapper
org.apache.ignite.internal.managers.discovery.DiscoCache$1
org.apache.ignite.internal.managers.discovery.DiscoCache$2
+org.apache.ignite.internal.managers.discovery.DiscoCache$3
org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$1
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$2
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4$1
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$6
+org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$8
+org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$9
org.apache.ignite.internal.managers.discovery.GridLocalMetrics
org.apache.ignite.internal.managers.eventstorage.GridEventStorageMessage
org.apache.ignite.internal.managers.indexing.GridIndexingManager$1
@@ -351,6 +363,7 @@ org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl$Segment
org.apache.ignite.internal.pagemem.wal.StorageException
org.apache.ignite.internal.pagemem.wal.WALIterator
org.apache.ignite.internal.pagemem.wal.WALPointer
+org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord$Type
org.apache.ignite.internal.pagemem.wal.record.WALRecord$RecordType
org.apache.ignite.internal.pagemem.wal.record.delta.DeltaApplicationException
org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion
@@ -414,6 +427,7 @@ org.apache.ignite.internal.processors.cache.CacheObjectImpl
org.apache.ignite.internal.processors.cache.CacheOperationContext
org.apache.ignite.internal.processors.cache.CacheOperationFilter
org.apache.ignite.internal.processors.cache.CachePartialUpdateCheckedException
+org.apache.ignite.internal.processors.cache.CacheStatisticsModeChangeMessage
org.apache.ignite.internal.processors.cache.CacheStoppedException
org.apache.ignite.internal.processors.cache.CacheStorePartialUpdateException
org.apache.ignite.internal.processors.cache.CacheType
@@ -436,9 +450,9 @@ org.apache.ignite.internal.processors.cache.GridCacheAdapter$12
org.apache.ignite.internal.processors.cache.GridCacheAdapter$13
org.apache.ignite.internal.processors.cache.GridCacheAdapter$14
org.apache.ignite.internal.processors.cache.GridCacheAdapter$15
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$16$1
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$16
org.apache.ignite.internal.processors.cache.GridCacheAdapter$17
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$18
+org.apache.ignite.internal.processors.cache.GridCacheAdapter$18$1
org.apache.ignite.internal.processors.cache.GridCacheAdapter$2
org.apache.ignite.internal.processors.cache.GridCacheAdapter$26$1
org.apache.ignite.internal.processors.cache.GridCacheAdapter$28
@@ -529,10 +543,12 @@ org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$Ex
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$MessageHandler
org.apache.ignite.internal.processors.cache.GridCacheProcessor$1
org.apache.ignite.internal.processors.cache.GridCacheProcessor$10
+org.apache.ignite.internal.processors.cache.GridCacheProcessor$11
+org.apache.ignite.internal.processors.cache.GridCacheProcessor$2
org.apache.ignite.internal.processors.cache.GridCacheProcessor$3
org.apache.ignite.internal.processors.cache.GridCacheProcessor$4
org.apache.ignite.internal.processors.cache.GridCacheProcessor$5
-org.apache.ignite.internal.processors.cache.GridCacheProcessor$7
+org.apache.ignite.internal.processors.cache.GridCacheProcessor$6
org.apache.ignite.internal.processors.cache.GridCacheProcessor$8
org.apache.ignite.internal.processors.cache.GridCacheProcessor$9
org.apache.ignite.internal.processors.cache.GridCacheProcessor$LocalAffinityFunction
@@ -555,6 +571,7 @@ org.apache.ignite.internal.processors.cache.GridCacheUtils$18
org.apache.ignite.internal.processors.cache.GridCacheUtils$19
org.apache.ignite.internal.processors.cache.GridCacheUtils$2
org.apache.ignite.internal.processors.cache.GridCacheUtils$20
+org.apache.ignite.internal.processors.cache.GridCacheUtils$22
org.apache.ignite.internal.processors.cache.GridCacheUtils$3
org.apache.ignite.internal.processors.cache.GridCacheUtils$4
org.apache.ignite.internal.processors.cache.GridCacheUtils$5
@@ -562,6 +579,7 @@ org.apache.ignite.internal.processors.cache.GridCacheUtils$6
org.apache.ignite.internal.processors.cache.GridCacheUtils$7
org.apache.ignite.internal.processors.cache.GridCacheUtils$8
org.apache.ignite.internal.processors.cache.GridCacheUtils$9
+org.apache.ignite.internal.processors.cache.GridCacheUtils$BackupPostProcessingClosure
org.apache.ignite.internal.processors.cache.GridCacheValueCollection
org.apache.ignite.internal.processors.cache.GridCacheValueCollection$1
org.apache.ignite.internal.processors.cache.GridChangeGlobalStateMessageResponse
@@ -898,12 +916,13 @@ org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$5
org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$8
org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$9
org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter$RowData
-org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$6
-org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$9
+org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$11
+org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$7
org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$CheckpointEntryType
org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$RebalanceIteratorAdapter
org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager$1
org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory
+org.apache.ignite.internal.processors.cache.persistence.file.FileDownloader$1
org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory
org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory
org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl$Segment
@@ -914,10 +933,16 @@ org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$DestroyBa
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Result
org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO$EntryPart
org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator
+org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator$StartSeekingFilter
org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer
org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$FileArchiver$1
+org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$FileArchiver$2
+org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressor$1
org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$RecordsIterator
org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException
+org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBuffer$BufferMode
+org.apache.ignite.internal.processors.cache.persistence.wal.SingleSegmentLogicalRecordsIterator
+org.apache.ignite.internal.processors.cache.persistence.wal.SingleSegmentLogicalRecordsIterator$LogicalRecordsFilter
org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException
org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException
org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneWalRecordsIterator
@@ -1049,6 +1074,15 @@ org.apache.ignite.internal.processors.cache.transactions.TxLock
org.apache.ignite.internal.processors.cache.transactions.TxLockList
org.apache.ignite.internal.processors.cache.transactions.TxLocksRequest
org.apache.ignite.internal.processors.cache.transactions.TxLocksResponse
+org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysTask
+org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysTask$CollectPartitionEntryHashesJob
+org.apache.ignite.internal.processors.cache.verify.PartitionEntryHashRecord
+org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord
+org.apache.ignite.internal.processors.cache.verify.PartitionKey
+org.apache.ignite.internal.processors.cache.verify.RetrieveConflictPartitionValuesTask
+org.apache.ignite.internal.processors.cache.verify.RetrieveConflictPartitionValuesTask$RetrieveConflictValuesJob
+org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask
+org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask$VerifyBackupPartitionsJob
org.apache.ignite.internal.processors.cache.version.GridCacheRawVersionedEntry
org.apache.ignite.internal.processors.cache.version.GridCacheVersion
org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext$State
@@ -1075,6 +1109,10 @@ org.apache.ignite.internal.processors.closure.GridClosureProcessor$T8
org.apache.ignite.internal.processors.closure.GridClosureProcessor$T9
org.apache.ignite.internal.processors.closure.GridClosureProcessor$TaskNoReduceAdapter
org.apache.ignite.internal.processors.closure.GridPeerDeployAwareTaskAdapter
+org.apache.ignite.internal.processors.cluster.BaselineTopology
+org.apache.ignite.internal.processors.cluster.BaselineTopologyHistory
+org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem
+org.apache.ignite.internal.processors.cluster.BranchingPointType
org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage
org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage
org.apache.ignite.internal.processors.cluster.ClusterProcessor$3
@@ -1086,6 +1124,7 @@ org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$3
org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$4
org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$5
org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$7
+org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$BaselineStateAndHistoryData
org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$CheckGlobalStateComputeRequest
org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$ClientChangeGlobalStateComputeRequest
org.apache.ignite.internal.processors.continuous.AbstractContinuousMessage
@@ -1388,6 +1427,7 @@ org.apache.ignite.internal.processors.query.schema.message.SchemaProposeDiscover
org.apache.ignite.internal.processors.query.schema.operation.SchemaAbstractAlterTableOperation
org.apache.ignite.internal.processors.query.schema.operation.SchemaAbstractOperation
org.apache.ignite.internal.processors.query.schema.operation.SchemaAlterTableAddColumnOperation
+org.apache.ignite.internal.processors.query.schema.operation.SchemaAlterTableDropColumnOperation
org.apache.ignite.internal.processors.query.schema.operation.SchemaIndexAbstractOperation
org.apache.ignite.internal.processors.query.schema.operation.SchemaIndexCreateOperation
org.apache.ignite.internal.processors.query.schema.operation.SchemaIndexDropOperation
@@ -1621,6 +1661,7 @@ org.apache.ignite.internal.util.lang.GridAbsClosure
org.apache.ignite.internal.util.lang.GridAbsClosureX
org.apache.ignite.internal.util.lang.GridCloseableIterator
org.apache.ignite.internal.util.lang.GridClosureException
+org.apache.ignite.internal.util.lang.GridFunc$1
org.apache.ignite.internal.util.lang.GridIterable
org.apache.ignite.internal.util.lang.GridIterableAdapter
org.apache.ignite.internal.util.lang.GridIterableAdapter$IteratorWrapper
@@ -1727,6 +1768,7 @@ org.apache.ignite.internal.util.snaptree.EpochNode
org.apache.ignite.internal.util.snaptree.EpochNode$Child
org.apache.ignite.internal.util.snaptree.SnapTreeMap
org.apache.ignite.internal.util.snaptree.SnapTreeMap$SubMap
+org.apache.ignite.internal.util.tostring.SBLimitedLength
org.apache.ignite.internal.util.typedef.C1
org.apache.ignite.internal.util.typedef.C2
org.apache.ignite.internal.util.typedef.CA
@@ -1767,6 +1809,12 @@ org.apache.ignite.internal.visor.VisorJob
org.apache.ignite.internal.visor.VisorMultiNodeTask
org.apache.ignite.internal.visor.VisorOneNodeTask
org.apache.ignite.internal.visor.VisorTaskArgument
+org.apache.ignite.internal.visor.baseline.VisorBaselineNode
+org.apache.ignite.internal.visor.baseline.VisorBaselineOperation
+org.apache.ignite.internal.visor.baseline.VisorBaselineTask
+org.apache.ignite.internal.visor.baseline.VisorBaselineTask$VisorBaselineJob
+org.apache.ignite.internal.visor.baseline.VisorBaselineTaskArg
+org.apache.ignite.internal.visor.baseline.VisorBaselineTaskResult
org.apache.ignite.internal.visor.binary.VisorBinaryMetadata
org.apache.ignite.internal.visor.binary.VisorBinaryMetadataCollectorTask
org.apache.ignite.internal.visor.binary.VisorBinaryMetadataCollectorTask$VisorBinaryCollectMetadataJob
@@ -1802,7 +1850,7 @@ org.apache.ignite.internal.visor.cache.VisorCacheMetricsCollectorTask
org.apache.ignite.internal.visor.cache.VisorCacheMetricsCollectorTask$VisorCacheMetricsCollectorJob
org.apache.ignite.internal.visor.cache.VisorCacheMetricsCollectorTaskArg
org.apache.ignite.internal.visor.cache.VisorCacheModifyTask
-org.apache.ignite.internal.visor.cache.VisorCacheModifyTask$VisorCacheClearJob
+org.apache.ignite.internal.visor.cache.VisorCacheModifyTask$VisorCacheModifyJob
org.apache.ignite.internal.visor.cache.VisorCacheModifyTaskArg
org.apache.ignite.internal.visor.cache.VisorCacheModifyTaskResult
org.apache.ignite.internal.visor.cache.VisorCacheNearConfiguration
@@ -2057,23 +2105,23 @@ org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointData
org.apache.ignite.spi.collision.jobstealing.JobStealingRequest
org.apache.ignite.spi.collision.priorityqueue.PriorityQueueCollisionSpi$PriorityGridCollisionJobContextComparator
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$1
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$10
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$11
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$12
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$1
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$2
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure$1
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosureNew
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosureNew$1
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$3
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$4
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$5
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeClosure
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeException
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeMessage
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeMessage2
org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeTimeoutException
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$NodeIdMessage
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$RecoveryLastReceivedMessage
+org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage
+org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2
+org.apache.ignite.spi.communication.tcp.messages.NodeIdMessage
+org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage
org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage
org.apache.ignite.spi.discovery.tcp.ClientImpl$State
org.apache.ignite.spi.discovery.tcp.ServerImpl$IpFinderCleaner$1
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 1c10bf1..518e674 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -1024,7 +1024,11 @@ public class IgniteClientReconnectCacheTest extends IgniteClientReconnectAbstrac
List<IgniteCache> caches = new ArrayList<>();
for (int i = 0; i < CLIENTS; i++) {
- Ignite client = startGrid(SRV_CNT + i);
+ int g = SRV_CNT + i;
+
+ IgniteEx client = startGrid(g);
+
+ info(">>>>> Started client: " + g);
addListener(client, disconnectLatch, reconnectLatch);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractRemoveFailureTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractRemoveFailureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractRemoveFailureTest.java
index faa19dc..11f19b2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractRemoveFailureTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractRemoveFailureTest.java
@@ -156,7 +156,7 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
* @throws Exception If failed.
*/
public void testPutAndRemove() throws Exception {
- putAndRemove(DUR, null, null);
+ putAndRemove(duration(), null, null);
}
/**
@@ -166,7 +166,7 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
if (atomicityMode() != CacheAtomicityMode.TRANSACTIONAL)
return;
- putAndRemove(30_000, PESSIMISTIC, REPEATABLE_READ);
+ putAndRemove(duration(), PESSIMISTIC, REPEATABLE_READ);
}
/**
@@ -176,7 +176,12 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
if (atomicityMode() != CacheAtomicityMode.TRANSACTIONAL)
return;
- putAndRemove(30_000, OPTIMISTIC, SERIALIZABLE);
+ putAndRemove(duration(), OPTIMISTIC, SERIALIZABLE);
+ }
+
+ /** */
+ protected long duration() {
+ return DUR;
}
/**
@@ -314,29 +319,7 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
}
});
- IgniteInternalFuture<?> killFut = GridTestUtils.runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- Thread.currentThread().setName("restart-thread");
-
- while (!stop.get()) {
- U.sleep(random(KILL_DELAY.get1(), KILL_DELAY.get2()));
-
- killAndRestart(stop);
-
- CyclicBarrier barrier = cmp.get();
-
- if (barrier != null) {
- log.info("Wait data check.");
-
- barrier.await(60_000, TimeUnit.MILLISECONDS);
-
- log.info("Finished wait data check.");
- }
- }
-
- return null;
- }
- });
+ IgniteInternalFuture killFut = createAndRunConcurrentAction(stop, cmp);
try {
long stopTime = duration + U.currentTimeMillis() ;
@@ -408,25 +391,50 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
log.info("Test finished. Update errors: " + errCntr.get());
}
+ /** */
+ protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean stop, final AtomicReference<CyclicBarrier> cmp) {
+ return GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ Thread.currentThread().setName("restart-thread");
+
+ while (!stop.get()) {
+ U.sleep(random(KILL_DELAY.get1(), KILL_DELAY.get2()));
+
+ killAndRestart(stop, random(1, GRID_CNT + 1));
+
+ CyclicBarrier barrier = cmp.get();
+
+ if (barrier != null) {
+ log.info("Wait data check.");
+
+ barrier.await(60_000, TimeUnit.MILLISECONDS);
+
+ log.info("Finished wait data check.");
+ }
+ }
+
+ return null;
+ }
+ });
+ }
+
/**
* @param stop Stop flag.
* @throws Exception If failed.
*/
- private void killAndRestart(AtomicBoolean stop) throws Exception {
+ protected void killAndRestart(AtomicBoolean stop, int nodeIdx) throws Exception {
if (stop.get())
return;
- int idx = random(1, GRID_CNT + 1);
-
- log.info("Killing node " + idx);
+ log.info("Killing node " + nodeIdx);
- stopGrid(idx);
+ stopGrid(nodeIdx);
U.sleep(random(START_DELAY.get1(), START_DELAY.get2()));
- log.info("Restarting node " + idx);
+ log.info("Restarting node " + nodeIdx);
- startGrid(idx);
+ startGrid(nodeIdx);
if (stop.get())
return;
@@ -475,7 +483,7 @@ public abstract class GridCacheAbstractRemoveFailureTest extends GridCommonAbstr
* @param max Max possible value (exclusive).
* @return Random value.
*/
- private static int random(int min, int max) {
+ protected static int random(int min, int max) {
if (max == min)
return max;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
index 81aadb9..df93ae4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
@@ -34,7 +34,6 @@ import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.store.CacheStore;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
@@ -113,7 +112,7 @@ public abstract class GridCacheAbstractSelfTest extends GridCommonAbstractTest {
*
* @throws IgniteCheckedException If failed.
*/
- void initStoreStrategy() throws IgniteCheckedException {
+ protected void initStoreStrategy() throws IgniteCheckedException {
if (storeStgy == null)
storeStgy = isMultiJvm() ? new H2CacheStoreStrategy() : new MapCacheStoreStrategy();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 3d1b014..e40700d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -459,7 +459,7 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
@Nullable GridCacheVersion dhtVer,
@Nullable Long updateCntr)
throws IgniteCheckedException, GridCacheEntryRemovedException {
- return new GridCacheUpdateTxResult(true, rawPut(val, ttl));
+ return new GridCacheUpdateTxResult(true, rawPut(val, ttl), null);
}
/** {@inheritDoc} */
@@ -545,7 +545,7 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
val = null;
- return new GridCacheUpdateTxResult(true, old);
+ return new GridCacheUpdateTxResult(true, old, null);
}
/** @inheritDoc */
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 1827c65..267cb18 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
@@ -34,6 +35,7 @@ import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.TestRecordingCommunicationSpi;
@@ -784,7 +786,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
startWithCaches1(SRVS, CLIENTS);
final Ignite srv = ignite(0);
- Ignite client = ignite(SRVS);
+ IgniteEx client = grid(SRVS);
if (persistenceEnabled())
ignite(0).active(true);
@@ -822,12 +824,11 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
}
});
- checkCache(client, CACHE_NAME_PREFIX + 0, false);
-
if (transition) {
assertFalse(stateFut.get().isDone());
- assertFalse(client.active());
+ // Public API method would block forever because we blocked the exchange message.
+ assertFalse(client.context().state().publicApiActiveState(false));
spi1.waitForBlocked();
@@ -889,7 +890,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
startWithCaches1(SRVS, CLIENTS);
final Ignite srv = ignite(0);
- Ignite client = ignite(SRVS);
+ IgniteEx client = grid(SRVS);
checkNoCaches(SRVS + CLIENTS);
@@ -922,12 +923,10 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
}
});
- checkCache(client, CACHE_NAME_PREFIX + 0, !transition);
-
if (transition) {
assertFalse(stateFut.get().isDone());
- assertFalse(client.active());
+ assertTrue(client.context().state().clusterState().transition());
spi1.waitForBlocked();
@@ -1140,70 +1139,6 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
}
/**
- * @throws Exception If failed.
- */
- public void testActivateFailover3() throws Exception {
- stateChangeFailover3(true);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testDeactivateFailover3() throws Exception {
- stateChangeFailover3(false);
- }
-
- /**
- * @param activate If {@code true} tests activation, otherwise deactivation.
- * @throws Exception If failed.
- */
- private void stateChangeFailover3(boolean activate) throws Exception {
- testDiscoSpi = true;
-
- startNodesAndBlockStatusChange(4, 0, 0, !activate);
-
- client = false;
-
- IgniteInternalFuture startFut1 = GridTestUtils.runAsync(new Callable() {
- @Override public Object call() throws Exception {
- startGrid(4);
-
- return null;
- }
- }, "start-node1");
-
- IgniteInternalFuture startFut2 = GridTestUtils.runAsync(new Callable() {
- @Override public Object call() throws Exception {
- startGrid(5);
-
- return null;
- }
- }, "start-node2");
-
- U.sleep(1000);
-
- // Stop all nodes participating in state change and not allow last node to finish exchange.
- for (int i = 0; i < 4; i++)
- ((TestTcpDiscoverySpi)ignite(i).configuration().getDiscoverySpi()).simulateNodeFailure();
-
- for (int i = 0; i < 4; i++)
- stopGrid(getTestIgniteInstanceName(i), true, false);
-
- startFut1.get();
- startFut2.get();
-
- assertFalse(ignite(4).active());
- assertFalse(ignite(5).active());
-
- ignite(4).active(true);
-
- for (int i = 0; i < 4; i++)
- startGrid(i);
-
- checkCaches1(6);
- }
-
- /**
* @param exp If {@code true} there should be recorded messages.
*/
private void checkRecordedMessages(boolean exp) {
@@ -1274,7 +1209,11 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
* @param node Node.
* @param exp {@code True} if expect that cache is started on node.
*/
- void checkCache(Ignite node, String cacheName, boolean exp) {
+ void checkCache(Ignite node, String cacheName, boolean exp) throws IgniteCheckedException {
+ ((IgniteEx)node).context().cache().context().exchange().lastTopologyFuture().get();
+
+ ((IgniteEx)node).context().state().publicApiActiveState(true);
+
GridCacheAdapter cache = ((IgniteKernal)node).context().cache().internalCache(cacheName);
if (exp)
@@ -1288,6 +1227,8 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
*/
final void checkNoCaches(int nodes) {
for (int i = 0; i < nodes; i++) {
+ grid(i).context().state().publicApiActiveState(true);
+
GridCacheProcessor cache = ((IgniteKernal)ignite(i)).context().cache();
assertTrue(cache.caches().isEmpty());
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
index 15e67b2..76078bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
@@ -20,10 +20,14 @@ package org.apache.ignite.internal.processors.cache;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.concurrent.CountDownLatch;
import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.testframework.GridTestUtils;
@@ -51,6 +55,11 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
GridTestUtils.deleteDbFiles();
}
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ return super.getConfiguration(igniteInstanceName).setAutoActivationEnabled(false);
+ }
+
/**
* @throws Exception If failed.
*/
@@ -79,12 +88,8 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
activateCachesRestore(5, true);
}
- /**
- * @param srvs Number of server nodes.
- * @param withNewCaches If {@code true} then after restart has new caches in configuration.
- * @throws Exception If failed.
- */
- private void activateCachesRestore(int srvs, boolean withNewCaches) throws Exception {
+ /** */
+ private Map<Integer, Integer> startGridsAndLoadData(int srvs) throws Exception {
Ignite srv = startGrids(srvs);
srv.active(true);
@@ -101,6 +106,17 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
}
}
+ return cacheData;
+ }
+
+ /**
+ * @param srvs Number of server nodes.
+ * @param withNewCaches If {@code true} then after restart has new caches in configuration.
+ * @throws Exception If failed.
+ */
+ private void activateCachesRestore(int srvs, boolean withNewCaches) throws Exception {
+ Map<Integer, Integer> cacheData = startGridsAndLoadData(srvs);
+
stopAllGrids();
for (int i = 0; i < srvs; i++) {
@@ -110,7 +126,7 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
startGrid(i);
}
- srv = ignite(0);
+ Ignite srv = ignite(0);
checkNoCaches(srvs);
@@ -160,6 +176,47 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
}
/**
+ * @see <a href="https://issues.apache.org/jira/browse/IGNITE-7330">IGNITE-7330</a> for more information about context of the test
+ */
+ public void testClientJoinsWhenActivationIsInProgress() throws Exception {
+ startGridsAndLoadData(5);
+
+ stopAllGrids();
+
+ Ignite srv = startGrids(5);
+
+ final CountDownLatch clientStartLatch = new CountDownLatch(1);
+
+ IgniteInternalFuture clStartFut = GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ clientStartLatch.await();
+
+ Thread.sleep(10);
+
+ client = true;
+
+ Ignite cl = startGrid("client0");
+
+ IgniteCache<Object, Object> atomicCache = cl.cache(CACHE_NAME_PREFIX + '0');
+ IgniteCache<Object, Object> txCache = cl.cache(CACHE_NAME_PREFIX + '1');
+
+ assertEquals(100, atomicCache.size());
+ assertEquals(100, txCache.size());
+ }
+ catch (Exception e) {
+ log.error("Error occurred", e);
+ }
+ }
+ }, "client-starter-thread");
+
+ clientStartLatch.countDown();
+ srv.active(true);
+
+ clStartFut.get();
+ }
+
+ /**
* Checks that persistent caches are present with actual data and volatile caches are missing.
*
* @param cacheData Cache data.
@@ -214,8 +271,4 @@ public class IgniteClusterActivateDeactivateTestWithPersistence extends IgniteCl
checkNoCaches(SRVS);
}
-
- @Override public void testActivateFailover3() throws Exception {
- super.testActivateFailover3();
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
new file mode 100644
index 0000000..5fbd752
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.PartitionLossPolicy;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE;
+
+/**
+ *
+ */
+public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
+ /** */
+ private static final String CACHE_NAME = "cache";
+
+ /** */
+ private boolean client;
+
+ /** */
+ private static final int NODE_COUNT = 4;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+
+ client = false;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDataStorageConfiguration(
+ new DataStorageConfiguration().setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setMaxSize(100 * 1024 * 1024)
+ .setInitialSize(100 * 1024 * 1024)
+ )
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ if (client)
+ cfg.setClientMode(true);
+
+ return cfg;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTopologyChangesWithFixedBaseline() throws Exception {
+ startGrids(NODE_COUNT);
+
+ IgniteEx ignite = grid(0);
+
+ ignite.active(true);
+
+ awaitPartitionMapExchange();
+
+ Map<ClusterNode, Ignite> nodes = new HashMap<>();
+
+ for (int i = 0; i < NODE_COUNT; i++) {
+ Ignite ig = grid(i);
+
+ nodes.put(ig.cluster().localNode(), ig);
+ }
+
+ IgniteCache<Integer, Integer> cache =
+ ignite.createCache(
+ new CacheConfiguration<Integer, Integer>()
+ .setName(CACHE_NAME)
+ .setCacheMode(PARTITIONED)
+ .setBackups(1)
+ .setPartitionLossPolicy(READ_ONLY_SAFE)
+ );
+
+ int key = -1;
+
+ for (int k = 0; k < 100_000; k++) {
+ if (!ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(k).contains(ignite.localNode())) {
+ key = k;
+ break;
+ }
+ }
+
+ assert key >= 0;
+
+ int part = ignite.affinity(CACHE_NAME).partition(key);
+
+ Collection<ClusterNode> initialMapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert initialMapping.size() == 2 : initialMapping;
+
+ ignite.cluster().setBaselineTopology(baselineNodes(nodes.keySet()));
+
+ awaitPartitionMapExchange();
+
+ cache.put(key, 1);
+
+ Collection<ClusterNode> mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert initialMapping.size() == mapping.size() : mapping;
+ assert initialMapping.containsAll(mapping) : mapping;
+
+ IgniteEx newIgnite = startGrid(4);
+
+ awaitPartitionMapExchange();
+
+ mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert initialMapping.size() == mapping.size() : mapping;
+ assert initialMapping.containsAll(mapping) : mapping;
+
+ mapping = newIgnite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert initialMapping.size() == mapping.size() : mapping;
+ assert initialMapping.containsAll(mapping) : mapping;
+
+ Set<String> stoppedNodeNames = new HashSet<>();
+
+ ClusterNode node = mapping.iterator().next();
+
+ stoppedNodeNames.add(nodes.get(node).name());
+
+ nodes.get(node).close();
+
+ nodes.remove(node);
+
+ awaitPartitionMapExchange(true, true, null);
+
+ mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert mapping.size() == 1 : mapping;
+ assert initialMapping.containsAll(mapping);
+
+ node = mapping.iterator().next();
+
+ stoppedNodeNames.add(nodes.get(node).name());
+
+ nodes.get(node).close();
+
+ nodes.remove(node);
+
+ awaitPartitionMapExchange();
+
+ mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert mapping.isEmpty() : mapping;
+
+ GridDhtPartitionTopology topology = ignite.cachex(CACHE_NAME).context().topology();
+
+ assert topology.lostPartitions().contains(part);
+
+ for (String nodeName : stoppedNodeNames) {
+ startGrid(nodeName);
+ }
+
+ assert ignite.cluster().nodes().size() == NODE_COUNT + 1;
+
+ mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert initialMapping.size() == mapping.size() : mapping;
+
+ for (ClusterNode n1 : initialMapping) {
+ boolean found = false;
+
+ for (ClusterNode n2 : mapping) {
+ if (n2.consistentId().equals(n1.consistentId())) {
+ found = true;
+
+ break;
+ }
+ }
+
+ assert found;
+ }
+
+ ignite.resetLostPartitions(Collections.singleton(CACHE_NAME));
+
+ cache.put(key, 2);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testBaselineTopologyChangesFromServer() throws Exception {
+ testBaselineTopologyChanges(false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testBaselineTopologyChangesFromClient() throws Exception {
+ testBaselineTopologyChanges(true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ private void testBaselineTopologyChanges(boolean fromClient) throws Exception {
+ startGrids(NODE_COUNT);
+
+ IgniteEx ignite;
+
+ if (fromClient) {
+ client = true;
+
+ ignite = startGrid(NODE_COUNT + 10);
+
+ client = false;
+ }
+ else
+ ignite = grid(0);
+
+ ignite.active(true);
+
+ awaitPartitionMapExchange();
+
+ Map<ClusterNode, Ignite> nodes = new HashMap<>();
+
+ for (int i = 0; i < NODE_COUNT; i++) {
+ Ignite ig = grid(i);
+
+ nodes.put(ig.cluster().localNode(), ig);
+ }
+
+ IgniteCache<Integer, Integer> cache =
+ ignite.createCache(
+ new CacheConfiguration<Integer, Integer>()
+ .setName(CACHE_NAME)
+ .setCacheMode(PARTITIONED)
+ .setBackups(1)
+ .setPartitionLossPolicy(READ_ONLY_SAFE)
+ );
+
+ int key = -1;
+
+ for (int k = 0; k < 100_000; k++) {
+ if (!ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(k).contains(ignite.localNode())) {
+ key = k;
+ break;
+ }
+ }
+
+ assert key >= 0;
+
+ int part = ignite.affinity(CACHE_NAME).partition(key);
+
+ Collection<ClusterNode> initialMapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert initialMapping.size() == 2 : initialMapping;
+
+ ignite.cluster().setBaselineTopology(baselineNodes(nodes.keySet()));
+
+ Set<String> stoppedNodeNames = new HashSet<>();
+
+ ClusterNode node = initialMapping.iterator().next();
+
+ stoppedNodeNames.add(nodes.get(node).name());
+
+ nodes.get(node).close();
+
+ nodes.remove(node);
+
+ awaitPartitionMapExchange();
+
+ Collection<ClusterNode> mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert mapping.size() == 1 : mapping;
+ assert initialMapping.containsAll(mapping);
+
+ Set<ClusterNode> blt2 = new HashSet<>(ignite.cluster().nodes());
+
+ ignite.cluster().setBaselineTopology(baselineNodes(blt2));
+
+ awaitPartitionMapExchange();
+
+ Collection<ClusterNode> initialMapping2 = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert initialMapping2.size() == 2 : initialMapping2;
+
+ Ignite newIgnite = startGrid(NODE_COUNT);
+
+ awaitPartitionMapExchange();
+
+ mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert mapping.size() == initialMapping2.size() : mapping;
+ assert mapping.containsAll(initialMapping2);
+
+ assert ignite.affinity(CACHE_NAME).primaryPartitions(newIgnite.cluster().localNode()).length == 0;
+
+ Set<ClusterNode> blt3 = new HashSet<>(ignite.cluster().nodes());
+
+ ignite.cluster().setBaselineTopology(baselineNodes(blt3));
+
+ awaitPartitionMapExchange();
+
+ Collection<ClusterNode> initialMapping3 = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert initialMapping3.size() == 2;
+
+ assert ignite.affinity(CACHE_NAME).primaryPartitions(newIgnite.cluster().localNode()).length > 0;
+
+ newIgnite = startGrid(NODE_COUNT + 1);
+
+ awaitPartitionMapExchange();
+
+ mapping = ignite.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert mapping.size() == initialMapping3.size() : mapping;
+ assert mapping.containsAll(initialMapping3);
+
+ assert ignite.affinity(CACHE_NAME).primaryPartitions(newIgnite.cluster().localNode()).length == 0;
+
+ ignite.cluster().setBaselineTopology(null);
+
+ awaitPartitionMapExchange();
+
+ assert ignite.affinity(CACHE_NAME).primaryPartitions(newIgnite.cluster().localNode()).length > 0;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testPrimaryLeft() throws Exception {
+ startGrids(NODE_COUNT);
+
+ IgniteEx ig = grid(0);
+
+ ig.cluster().active(true);
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<Integer, Integer> cache =
+ ig.createCache(
+ new CacheConfiguration<Integer, Integer>()
+ .setName(CACHE_NAME)
+ .setCacheMode(PARTITIONED)
+ .setBackups(1)
+ .setPartitionLossPolicy(READ_ONLY_SAFE)
+ .setReadFromBackup(true)
+ .setWriteSynchronizationMode(FULL_SYNC)
+ .setRebalanceDelay(-1)
+ );
+
+ int key = 1;
+
+ List<ClusterNode> affNodes = (List<ClusterNode>) ig.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert affNodes.size() == 2;
+
+ int primaryIdx = -1;
+
+ IgniteEx primary = null;
+ IgniteEx backup = null;
+
+ for (int i = 0; i < NODE_COUNT; i++) {
+ if (grid(i).localNode().equals(affNodes.get(0))) {
+ primaryIdx = i;
+ primary = grid(i);
+ }
+ else if (grid(i).localNode().equals(affNodes.get(1)))
+ backup = grid(i);
+ }
+
+ assert primary != null;
+ assert backup != null;
+
+ Integer val1 = 1;
+ Integer val2 = 2;
+
+ cache.put(key, val1);
+
+ assertEquals(val1, primary.cache(CACHE_NAME).get(key));
+ assertEquals(val1, backup.cache(CACHE_NAME).get(key));
+
+ if (ig == primary) {
+ ig = backup;
+
+ cache = ig.cache(CACHE_NAME);
+ }
+
+ primary.close();
+
+ assertEquals(backup.localNode(), ig.affinity(CACHE_NAME).mapKeyToNode(key));
+
+ cache.put(key, val2);
+
+ assertEquals(val2, backup.cache(CACHE_NAME).get(key));
+
+ primary = startGrid(primaryIdx);
+
+ assertEquals(backup.localNode(), ig.affinity(CACHE_NAME).mapKeyToNode(key));
+
+ primary.cache(CACHE_NAME).rebalance().get();
+
+ awaitPartitionMapExchange();
+
+ assertEquals(primary.localNode(), ig.affinity(CACHE_NAME).mapKeyToNode(key));
+
+ assertEquals(val2, primary.cache(CACHE_NAME).get(key));
+ assertEquals(val2, backup.cache(CACHE_NAME).get(key));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testPrimaryLeftAndClusterRestart() throws Exception {
+ startGrids(NODE_COUNT);
+
+ IgniteEx ig = grid(0);
+
+ ig.active(true);
+
+ IgniteCache<Integer, Integer> cache =
+ ig.createCache(
+ new CacheConfiguration<Integer, Integer>()
+ .setName(CACHE_NAME)
+ .setWriteSynchronizationMode(FULL_SYNC)
+ .setCacheMode(PARTITIONED)
+ .setBackups(1)
+ .setPartitionLossPolicy(READ_ONLY_SAFE)
+ .setReadFromBackup(true)
+ .setRebalanceDelay(-1)
+ );
+
+ int key = 1;
+
+ List<ClusterNode> affNodes = (List<ClusterNode>) ig.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assert affNodes.size() == 2;
+
+ int primaryIdx = -1;
+ int backupIdx = -1;
+
+ IgniteEx primary = null;
+ IgniteEx backup = null;
+
+ for (int i = 0; i < NODE_COUNT; i++) {
+ if (grid(i).localNode().equals(affNodes.get(0))) {
+ primaryIdx = i;
+ primary = grid(i);
+ }
+ else if (grid(i).localNode().equals(affNodes.get(1))) {
+ backupIdx = i;
+ backup = grid(i);
+ }
+ }
+
+ assert primary != null;
+ assert backup != null;
+
+ Integer val1 = 1;
+ Integer val2 = 2;
+
+ cache.put(key, val1);
+
+ assertEquals(val1, primary.cache(CACHE_NAME).get(key));
+ assertEquals(val1, backup.cache(CACHE_NAME).get(key));
+
+ if (ig == primary) {
+ ig = backup;
+
+ cache = ig.cache(CACHE_NAME);
+ }
+
+ stopGrid(primaryIdx, false);
+
+ assertEquals(backup.localNode(), ig.affinity(CACHE_NAME).mapKeyToNode(key));
+
+ cache.put(key, val2);
+
+ assertEquals(val2, backup.cache(CACHE_NAME).get(key));
+
+ stopAllGrids(false);
+
+ startGrids(NODE_COUNT);
+
+ ig = grid(0);
+ primary = grid(primaryIdx);
+ backup = grid(backupIdx);
+
+ boolean activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ for (int i = 0; i < NODE_COUNT; i++)
+ if (!grid(i).active())
+ return false;
+
+ return true;
+ }
+ }, 10_000);
+
+ assert activated;
+
+// assertEquals(backup.localNode(), ig.affinity(CACHE_NAME).mapKeyToNode(key));
+
+ assertEquals(val2, primary.cache(CACHE_NAME).get(key));
+ assertEquals(val2, backup.cache(CACHE_NAME).get(key));
+
+ primary.cache(CACHE_NAME).rebalance().get();
+
+ affNodes = (List<ClusterNode>) ig.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(key);
+
+ assertEquals(primary.localNode(), affNodes.get(0));
+ assertEquals(backup.localNode(), affNodes.get(1));
+
+ assertEquals(val2, primary.cache(CACHE_NAME).get(key));
+ assertEquals(val2, backup.cache(CACHE_NAME).get(key));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testMetadataUpdate() throws Exception {
+ startGrids(5);
+
+ Ignite ignite3 = grid(3);
+
+ ignite3.active(true);
+
+ CacheConfiguration<Object, Object> repCacheCfg = new CacheConfiguration<>("replicated")
+ .setCacheMode(CacheMode.REPLICATED)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+ IgniteCache<Object, Object> cache = ignite3.getOrCreateCache(repCacheCfg);
+
+ stopGrid(0);
+ stopGrid(1);
+ stopGrid(2);
+ stopGrid(4);
+
+ for (int i = 0; i < 100; i++)
+ cache.put(i, new TestValue(i));
+
+ stopAllGrids();
+
+ startGrids(5);
+
+ GridTestUtils.waitForCondition(new PA() {
+ @Override
+ public boolean apply() {
+ return grid(0).cluster().active();
+ }
+ }, getTestTimeout());
+
+ for (int g = 0; g < 5; g++) {
+ for (int i = 0; i < 100; i++)
+ assertEquals(new TestValue(i), grid(g).cache("replicated").get(i));
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testClusterRestoredOnRestart() throws Exception {
+ startGrids(5);
+
+ Ignite ignite3 = grid(3);
+
+ ignite3.active(true);
+
+ stopGrid(0);
+
+ CacheConfiguration<Object, Object> cacheConfiguration = new CacheConfiguration<>("unknown_cache");
+
+ cacheConfiguration.setBackups(3);
+
+ IgniteCache<Object, Object> cache0 = ignite3.getOrCreateCache(cacheConfiguration);
+
+ for (int i = 0; i < 2048; i++)
+ cache0.put(i, 0);
+
+ awaitPartitionMapExchange();
+
+ stopAllGrids();
+
+ startGrids(5);
+
+ GridTestUtils.waitForCondition(new PA() {
+ @Override
+ public boolean apply() {
+ return grid(0).cluster().active();
+ }
+ }, getTestTimeout());
+
+ for (int g = 0; g < 5; g++) {
+ for (int i = 0; i < 2048; i++)
+ assertEquals("For key: " + i, 0, grid(g).cache("unknown_cache").get(i));
+ }
+ }
+
+ /** */
+ private Collection<BaselineNode> baselineNodes(Collection<ClusterNode> clNodes) {
+ Collection<BaselineNode> res = new ArrayList<>(clNodes.size());
+
+ for (ClusterNode clN : clNodes)
+ res.add(clN);
+
+ return res;
+ }
+
+ /**
+ *
+ */
+ private static class TestValue {
+ /** */
+ int f1;
+
+ /** */
+ int f2;
+
+ /** */
+ int f3;
+
+ /** */
+ int f4;
+
+ /**
+ * @param a Init value.
+ */
+ private TestValue(int a) {
+ f1 = f2 = f3 = f4 = a;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof TestValue))
+ return false;
+
+ TestValue other = (TestValue)o;
+
+ return
+ f1 == other.f1 &&
+ f2 == other.f2 &&
+ f3 == other.f3 &&
+ f4 == other.f4;
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int result = f1;
+
+ result = 31 * result + f2;
+ result = 31 * result + f3;
+ result = 31 * result + f4;
+
+ return result;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
index f93d60c..9660a76 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
@@ -82,6 +82,7 @@ import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheMode.REPLICATED;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
+import static org.apache.ignite.testframework.GridTestUtils.mergeExchangeWaitVersion;
/**
*
@@ -1178,15 +1179,6 @@ public class CacheExchangeMergeTest extends GridCommonAbstractTest {
}
/**
- * @param node Node.
- * @param topVer Ready exchange version to wait for before trying to merge exchanges.
- */
- private void mergeExchangeWaitVersion(Ignite node, long topVer) {
- ((IgniteKernal)node).context().cache().context().exchange().mergeExchangesTestWaitVersion(
- new AffinityTopologyVersion(topVer, 0));
- }
-
- /**
* @throws Exception If failed.
*/
private void checkCaches() throws Exception {
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index ab07611..0704dbd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -53,7 +53,8 @@ import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.GridNodeOrderComparator;
+import org.apache.ignite.internal.cluster.NodeOrderComparator;
+import org.apache.ignite.internal.cluster.NodeOrderLegacyComparator;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.IgniteNodeAttributes;
@@ -2852,7 +2853,7 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
affNodes.add(n);
}
- Collections.sort(affNodes, GridNodeOrderComparator.INSTANCE);
+ Collections.sort(affNodes, NodeOrderComparator.getInstance());
AffinityFunctionContext affCtx = new GridAffinityFunctionContextImpl(
affNodes,
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePutAllFailoverAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePutAllFailoverAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePutAllFailoverAbstractTest.java
index 9a0fb03..57a150f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePutAllFailoverAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePutAllFailoverAbstractTest.java
@@ -125,21 +125,7 @@ public abstract class CachePutAllFailoverAbstractTest extends GridCacheAbstractS
final long endTime = System.currentTimeMillis() + TEST_TIME;
- IgniteInternalFuture<Object> restartFut = GridTestUtils.runAsync(new Callable<Object>() {
- @Override public Object call() throws Exception {
- Thread.currentThread().setName("restart-thread");
-
- while (!finished.get() && System.currentTimeMillis() < endTime) {
- startGrid(NODE_CNT);
-
- U.sleep(500);
-
- stopGrid(NODE_CNT);
- }
-
- return null;
- }
- });
+ IgniteInternalFuture<Object> restartFut = createAndRunConcurrentAction(finished, endTime);
try {
final IgniteCache<TestKey, TestValue> cache = ignite(0).cache(DEFAULT_CACHE_NAME);
@@ -235,6 +221,25 @@ public abstract class CachePutAllFailoverAbstractTest extends GridCacheAbstractS
}
}
+ /** */
+ protected IgniteInternalFuture createAndRunConcurrentAction(final AtomicBoolean finished, final long endTime) {
+ return GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ Thread.currentThread().setName("restart-thread");
+
+ while (!finished.get() && System.currentTimeMillis() < endTime) {
+ startGrid(NODE_CNT);
+
+ U.sleep(500);
+
+ stopGrid(NODE_CNT);
+ }
+
+ return null;
+ }
+ });
+ }
+
/**
*
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
index 86c0fa4..34bd0a0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
@@ -32,6 +32,7 @@ import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.CI2;
+import org.apache.ignite.internal.util.typedef.PA;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -298,6 +299,12 @@ public class IgniteOptimisticTxSuspendResumeTest extends GridCommonAbstractTest
}
}).get(FUT_TIMEOUT);
+ assertTrue(GridTestUtils.waitForCondition(new PA() {
+ @Override public boolean apply() {
+ return tx.state() == ROLLED_BACK;
+ }
+ }, getTestTimeout()));
+
assertEquals(ROLLED_BACK, tx.state());
assertFalse(cache.containsKey(1));
@@ -442,6 +449,12 @@ public class IgniteOptimisticTxSuspendResumeTest extends GridCommonAbstractTest
}
}, TransactionTimeoutException.class);
+ assertTrue(GridTestUtils.waitForCondition(new PA() {
+ @Override public boolean apply() {
+ return tx.state() == ROLLED_BACK;
+ }
+ }, getTestTimeout()));
+
assertEquals(ROLLED_BACK, tx.state());
tx.close();
@@ -476,6 +489,12 @@ public class IgniteOptimisticTxSuspendResumeTest extends GridCommonAbstractTest
}
}, TransactionTimeoutException.class);
+ assertTrue(GridTestUtils.waitForCondition(new PA() {
+ @Override public boolean apply() {
+ return tx.state() == ROLLED_BACK;
+ }
+ }, getTestTimeout()));
+
assertEquals(ROLLED_BACK, tx.state());
tx.close();
[11/17] ignite git commit: IEP-4 Baseline topology for persistent
caches (Phase 1) Contributed by: Dmitriy Govorukhin
Dmitry Pavlov Eduard
Shangareev Ily
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
index d2f0099..bdca21c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
@@ -19,16 +19,16 @@ package org.apache.ignite.internal.processors.cache.persistence.freelist;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.Storable;
/**
*/
-public interface FreeList {
+public interface FreeList<T extends Storable> {
/**
* @param row Row.
* @throws IgniteCheckedException If failed.
*/
- public void insertDataRow(CacheDataRow row) throws IgniteCheckedException;
+ public void insertDataRow(T row) throws IgniteCheckedException;
/**
* @param link Row link.
@@ -36,7 +36,7 @@ public interface FreeList {
* @return {@code True} if was able to update row.
* @throws IgniteCheckedException If failed.
*/
- public boolean updateDataRow(long link, CacheDataRow row) throws IgniteCheckedException;
+ public boolean updateDataRow(long link, T row) throws IgniteCheckedException;
/**
* @param link Row link.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
deleted file mode 100644
index 5bacc53..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
+++ /dev/null
@@ -1,619 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.freelist;
-
-import java.util.concurrent.atomic.AtomicReferenceArray;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord;
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
-import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
-import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
-import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- */
-public class FreeListImpl extends PagesList implements FreeList, ReuseList {
- /** */
- private static final int BUCKETS = 256; // Must be power of 2.
-
- /** */
- private static final int REUSE_BUCKET = BUCKETS - 1;
-
- /** */
- private static final Integer COMPLETE = Integer.MAX_VALUE;
-
- /** */
- private static final Integer FAIL_I = Integer.MIN_VALUE;
-
- /** */
- private static final Long FAIL_L = Long.MAX_VALUE;
-
- /** */
- private static final int MIN_PAGE_FREE_SPACE = 8;
-
- /** */
- private final int shift;
-
- /** */
- private final AtomicReferenceArray<Stripe[]> buckets = new AtomicReferenceArray<>(BUCKETS);
-
- /** */
- private final int MIN_SIZE_FOR_DATA_PAGE;
-
- /** */
- private final int emptyDataPagesBucket;
-
- /** */
- private final PageHandler<CacheDataRow, Boolean> updateRow = new UpdateRowHandler();
-
- /** */
- private final DataRegionMetricsImpl memMetrics;
-
- /** */
- private final PageEvictionTracker evictionTracker;
-
- /**
- *
- */
- private final class UpdateRowHandler extends PageHandler<CacheDataRow, Boolean> {
- @Override public Boolean run(
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageIO iox,
- Boolean walPlc,
- CacheDataRow row,
- int itemId)
- throws IgniteCheckedException {
- DataPageIO io = (DataPageIO)iox;
-
- int rowSize = getRowSize(row, row.cacheId() != 0);
-
- boolean updated = io.updateRow(pageAddr, itemId, pageSize(), null, row, rowSize);
-
- evictionTracker.touchPage(pageId);
-
- if (updated && needWalDeltaRecord(pageId, page, walPlc)) {
- // TODO This record must contain only a reference to a logical WAL record with the actual data.
- byte[] payload = new byte[rowSize];
-
- DataPagePayload data = io.readPayload(pageAddr, itemId, pageSize());
-
- assert data.payloadSize() == rowSize;
-
- PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
-
- wal.log(new DataPageUpdateRecord(
- cacheId,
- pageId,
- itemId,
- payload));
- }
-
- return updated;
- }
- }
-
- /** */
- private final PageHandler<CacheDataRow, Integer> writeRow = new WriteRowHandler();
-
- /**
- *
- */
- private final class WriteRowHandler extends PageHandler<CacheDataRow, Integer> {
- @Override public Integer run(
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageIO iox,
- Boolean walPlc,
- CacheDataRow row,
- int written)
- throws IgniteCheckedException {
- DataPageIO io = (DataPageIO)iox;
-
- int rowSize = getRowSize(row, row.cacheId() != 0);
- int oldFreeSpace = io.getFreeSpace(pageAddr);
-
- assert oldFreeSpace > 0 : oldFreeSpace;
-
- // If the full row does not fit into this page write only a fragment.
- written = (written == 0 && oldFreeSpace >= rowSize) ? addRow(pageId, page, pageAddr, io, row, rowSize):
- addRowFragment(pageId, page, pageAddr, io, row, written, rowSize);
-
- // Reread free space after update.
- int newFreeSpace = io.getFreeSpace(pageAddr);
-
- if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
- int bucket = bucket(newFreeSpace, false);
-
- put(null, pageId, page, pageAddr, bucket);
- }
-
- if (written == rowSize)
- evictionTracker.touchPage(pageId);
-
- // Avoid boxing with garbage generation for usual case.
- return written == rowSize ? COMPLETE : written;
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param io IO.
- * @param row Row.
- * @param rowSize Row size.
- * @return Written size which is always equal to row size here.
- * @throws IgniteCheckedException If failed.
- */
- private int addRow(
- long pageId,
- long page,
- long pageAddr,
- DataPageIO io,
- CacheDataRow row,
- int rowSize
- ) throws IgniteCheckedException {
- io.addRow(pageId, pageAddr, row, rowSize, pageSize());
-
- if (needWalDeltaRecord(pageId, page, null)) {
- // TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data.
- byte[] payload = new byte[rowSize];
-
- DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
-
- assert data.payloadSize() == rowSize;
-
- PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
-
- wal.log(new DataPageInsertRecord(
- grpId,
- pageId,
- payload));
- }
-
- return rowSize;
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param io IO.
- * @param row Row.
- * @param written Written size.
- * @param rowSize Row size.
- * @return Updated written size.
- * @throws IgniteCheckedException If failed.
- */
- private int addRowFragment(
- long pageId,
- long page,
- long pageAddr,
- DataPageIO io,
- CacheDataRow row,
- int written,
- int rowSize
- ) throws IgniteCheckedException {
- // Read last link before the fragment write, because it will be updated there.
- long lastLink = row.link();
-
- int payloadSize = io.addRowFragment(pageMem, pageAddr, row, written, rowSize, pageSize());
-
- assert payloadSize > 0 : payloadSize;
-
- if (needWalDeltaRecord(pageId, page, null)) {
- // TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data.
- byte[] payload = new byte[payloadSize];
-
- DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
-
- PageUtils.getBytes(pageAddr, data.offset(), payload, 0, payloadSize);
-
- wal.log(new DataPageInsertFragmentRecord(grpId, pageId, payload, lastLink));
- }
-
- return written + payloadSize;
- }
- }
-
-
- /** */
- private final PageHandler<Void, Long> rmvRow;
-
- /**
- *
- */
- private final class RemoveRowHandler extends PageHandler<Void, Long> {
- /** Indicates whether partition ID should be masked from page ID. */
- private final boolean maskPartId;
-
- /** */
- RemoveRowHandler(boolean maskPartId) {
- this.maskPartId = maskPartId;
- }
-
- @Override public Long run(
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageIO iox,
- Boolean walPlc,
- Void ignored,
- int itemId)
- throws IgniteCheckedException {
- DataPageIO io = (DataPageIO)iox;
-
- int oldFreeSpace = io.getFreeSpace(pageAddr);
-
- assert oldFreeSpace >= 0: oldFreeSpace;
-
- long nextLink = io.removeRow(pageAddr, itemId, pageSize());
-
- if (needWalDeltaRecord(pageId, page, walPlc))
- wal.log(new DataPageRemoveRecord(cacheId, pageId, itemId));
-
- int newFreeSpace = io.getFreeSpace(pageAddr);
-
- if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
- int newBucket = bucket(newFreeSpace, false);
-
- if (oldFreeSpace > MIN_PAGE_FREE_SPACE) {
- int oldBucket = bucket(oldFreeSpace, false);
-
- if (oldBucket != newBucket) {
- // It is possible that page was concurrently taken for put, in this case put will handle bucket change.
- pageId = maskPartId ? PageIdUtils.maskPartitionId(pageId) : pageId;
- if (removeDataPage(pageId, page, pageAddr, io, oldBucket))
- put(null, pageId, page, pageAddr, newBucket);
- }
- }
- else
- put(null, pageId, page, pageAddr, newBucket);
-
- if (io.isEmpty(pageAddr))
- evictionTracker.forgetPage(pageId);
- }
-
- // For common case boxed 0L will be cached inside of Long, so no garbage will be produced.
- return nextLink;
- }
- }
-
- /**
- * @param cacheId Cache ID.
- * @param name Name (for debug purpose).
- * @param memMetrics Memory metrics.
- * @param memPlc Data region.
- * @param reuseList Reuse list or {@code null} if this free list will be a reuse list for itself.
- * @param wal Write ahead log manager.
- * @param metaPageId Metadata page ID.
- * @param initNew {@code True} if new metadata should be initialized.
- * @throws IgniteCheckedException If failed.
- */
- public FreeListImpl(
- int cacheId,
- String name,
- DataRegionMetricsImpl memMetrics,
- DataRegion memPlc,
- ReuseList reuseList,
- IgniteWriteAheadLogManager wal,
- long metaPageId,
- boolean initNew) throws IgniteCheckedException {
- super(cacheId, name, memPlc.pageMemory(), BUCKETS, wal, metaPageId);
-
- rmvRow = new RemoveRowHandler(cacheId == 0);
-
- this.evictionTracker = memPlc.evictionTracker();
- this.reuseList = reuseList == null ? this : reuseList;
- int pageSize = pageMem.pageSize();
-
- assert U.isPow2(pageSize) : "Page size must be a power of 2: " + pageSize;
- assert U.isPow2(BUCKETS);
- assert BUCKETS <= pageSize : pageSize;
-
- // TODO this constant is used because currently we cannot reuse data pages as index pages
- // TODO and vice-versa. It should be removed when data storage format is finalized.
- MIN_SIZE_FOR_DATA_PAGE = pageSize - DataPageIO.MIN_DATA_PAGE_OVERHEAD;
-
- int shift = 0;
-
- while (pageSize > BUCKETS) {
- shift++;
- pageSize >>>= 1;
- }
-
- this.shift = shift;
-
- this.memMetrics = memMetrics;
-
- emptyDataPagesBucket = bucket(MIN_SIZE_FOR_DATA_PAGE, false);
-
- init(metaPageId, initNew);
- }
-
- /**
- * Calculates average fill factor over FreeListImpl instance.
- *
- * @return Tuple (numenator, denominator).
- */
- public T2<Long, Long> fillFactor() {
- long pageSize = pageSize();
-
- long totalSize = 0;
- long loadSize = 0;
-
- for (int b = BUCKETS - 2; b > 0; b--) {
- long bsize = pageSize - ((REUSE_BUCKET - b) << shift);
-
- long pages = bucketsSize[b].longValue();
-
- loadSize += pages * (pageSize - bsize);
-
- totalSize += pages * pageSize;
- }
-
- return totalSize == 0 ? new T2<>(0L, 0L) : new T2<>(loadSize, totalSize);
- }
-
- /** {@inheritDoc} */
- @Override public void dumpStatistics(IgniteLogger log) {
- long dataPages = 0;
-
- final boolean dumpBucketsInfo = false;
-
- for (int b = 0; b < BUCKETS; b++) {
- long size = bucketsSize[b].longValue();
-
- if (!isReuseBucket(b))
- dataPages += size;
-
- if (dumpBucketsInfo) {
- Stripe[] stripes = getBucket(b);
-
- boolean empty = true;
-
- if (stripes != null) {
- for (Stripe stripe : stripes) {
- if (!stripe.empty) {
- empty = false;
-
- break;
- }
- }
- }
-
- if (log.isInfoEnabled())
- log.info("Bucket [b=" + b +
- ", size=" + size +
- ", stripes=" + (stripes != null ? stripes.length : 0) +
- ", stripesEmpty=" + empty + ']');
- }
- }
-
- if (dataPages > 0) {
- if (log.isInfoEnabled())
- log.info("FreeList [name=" + name +
- ", buckets=" + BUCKETS +
- ", dataPages=" + dataPages +
- ", reusePages=" + bucketsSize[REUSE_BUCKET].longValue() + "]");
- }
- }
-
- /**
- * @param freeSpace Page free space.
- * @param allowReuse {@code True} if it is allowed to get reuse bucket.
- * @return Bucket.
- */
- private int bucket(int freeSpace, boolean allowReuse) {
- assert freeSpace > 0 : freeSpace;
-
- int bucket = freeSpace >>> shift;
-
- assert bucket >= 0 && bucket < BUCKETS : bucket;
-
- if (!allowReuse && isReuseBucket(bucket))
- bucket--;
-
- return bucket;
- }
-
- /**
- * @param part Partition.
- * @return Page ID.
- * @throws IgniteCheckedException If failed.
- */
- private long allocateDataPage(int part) throws IgniteCheckedException {
- assert part <= PageIdAllocator.MAX_PARTITION_ID;
- assert part != PageIdAllocator.INDEX_PARTITION;
-
- return pageMem.allocatePage(grpId, part, PageIdAllocator.FLAG_DATA);
- }
-
- /** {@inheritDoc} */
- @Override public void insertDataRow(CacheDataRow row) throws IgniteCheckedException {
- int rowSize = getRowSize(row, row.cacheId() != 0);
-
- int written = 0;
-
- do {
- if (written != 0)
- memMetrics.incrementLargeEntriesPages();
-
- int freeSpace = Math.min(MIN_SIZE_FOR_DATA_PAGE, rowSize - written);
-
- long pageId = 0L;
-
- if (freeSpace == MIN_SIZE_FOR_DATA_PAGE)
- pageId = takeEmptyPage(emptyDataPagesBucket, DataPageIO.VERSIONS);
-
- boolean reuseBucket = false;
-
- // TODO: properly handle reuse bucket.
- if (pageId == 0L) {
- for (int b = bucket(freeSpace, false) + 1; b < BUCKETS - 1; b++) {
- pageId = takeEmptyPage(b, DataPageIO.VERSIONS);
-
- if (pageId != 0L) {
- reuseBucket = isReuseBucket(b);
-
- break;
- }
- }
- }
-
- boolean allocated = pageId == 0L;
-
- if (allocated)
- pageId = allocateDataPage(row.partition());
- else
- pageId = PageIdUtils.changePartitionId(pageId, (row.partition()));
-
- DataPageIO init = reuseBucket || allocated ? DataPageIO.VERSIONS.latest() : null;
-
- written = write(pageId, writeRow, init, row, written, FAIL_I);
-
- assert written != FAIL_I; // We can't fail here.
- }
- while (written != COMPLETE);
- }
-
- /** {@inheritDoc} */
- @Override public boolean updateDataRow(long link, CacheDataRow row) throws IgniteCheckedException {
- assert link != 0;
-
- long pageId = PageIdUtils.pageId(link);
- int itemId = PageIdUtils.itemId(link);
-
- Boolean updated = write(pageId, updateRow, row, itemId, null);
-
- assert updated != null; // Can't fail here.
-
- return updated;
- }
-
- /** {@inheritDoc} */
- @Override public void removeDataRowByLink(long link) throws IgniteCheckedException {
- assert link != 0;
-
- long pageId = PageIdUtils.pageId(link);
- int itemId = PageIdUtils.itemId(link);
-
- long nextLink = write(pageId, rmvRow, itemId, FAIL_L);
-
- assert nextLink != FAIL_L; // Can't fail here.
-
- while (nextLink != 0L) {
- memMetrics.decrementLargeEntriesPages();
-
- itemId = PageIdUtils.itemId(nextLink);
- pageId = PageIdUtils.pageId(nextLink);
-
- nextLink = write(pageId, rmvRow, itemId, FAIL_L);
-
- assert nextLink != FAIL_L; // Can't fail here.
- }
- }
-
- /** {@inheritDoc} */
- @Override protected Stripe[] getBucket(int bucket) {
- return buckets.get(bucket);
- }
-
- /** {@inheritDoc} */
- @Override protected boolean casBucket(int bucket, Stripe[] exp, Stripe[] upd) {
- return buckets.compareAndSet(bucket, exp, upd);
- }
-
- /** {@inheritDoc} */
- @Override protected boolean isReuseBucket(int bucket) {
- return bucket == REUSE_BUCKET;
- }
-
- /**
- * @return Number of empty data pages in free list.
- */
- public int emptyDataPages() {
- return bucketsSize[emptyDataPagesBucket].intValue();
- }
-
- /** {@inheritDoc} */
- @Override public void addForRecycle(ReuseBag bag) throws IgniteCheckedException {
- assert reuseList == this: "not allowed to be a reuse list";
-
- put(bag, 0, 0, 0L, REUSE_BUCKET);
- }
-
- /** {@inheritDoc} */
- @Override public long takeRecycledPage() throws IgniteCheckedException {
- assert reuseList == this: "not allowed to be a reuse list";
-
- return takeEmptyPage(REUSE_BUCKET, null);
- }
-
- /** {@inheritDoc} */
- @Override public long recycledPagesCount() throws IgniteCheckedException {
- assert reuseList == this: "not allowed to be a reuse list";
-
- return storedPagesCount(REUSE_BUCKET);
- }
-
- /**
- * @param row Row.
- * @param withCacheId If {@code true} adds cache ID size.
- * @return Entry size on page.
- * @throws IgniteCheckedException If failed.
- */
- public static int getRowSize(CacheDataRow row, boolean withCacheId) throws IgniteCheckedException {
- KeyCacheObject key = row.key();
- CacheObject val = row.value();
-
- int keyLen = key.valueBytesLength(null);
- int valLen = val.valueBytesLength(null);
-
- return keyLen + valLen + CacheVersionIO.size(row.version(), false) + 8 + (withCacheId ? 4 : 0);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "FreeList [name=" + name + ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
index b113c62..54ffb3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
@@ -38,7 +38,7 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetPreviousR
import org.apache.ignite.internal.processors.cache.persistence.DataStructure;
import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
@@ -240,11 +240,10 @@ public abstract class PagesList extends DataStructure {
}
boolean ok = casBucket(bucket, null, tails);
+
assert ok;
bucketsSize[bucket].set(bucketSize);
-
- changed = true;
}
}
}
@@ -695,7 +694,7 @@ public abstract class PagesList extends DataStructure {
if (needWalDeltaRecord(pageId, page, null))
wal.log(new PagesListAddPageRecord(grpId, pageId, dataId));
- DataPageIO dataIO = DataPageIO.VERSIONS.forPage(dataAddr);
+ AbstractDataPageIO dataIO = PageIO.getPageIO(dataAddr);
dataIO.setFreeListPageId(dataAddr, pageId);
if (needWalDeltaRecord(dataId, dataPage, null))
@@ -726,7 +725,7 @@ public abstract class PagesList extends DataStructure {
final long dataAddr,
int bucket
) throws IgniteCheckedException {
- DataPageIO dataIO = DataPageIO.VERSIONS.forPage(dataAddr);
+ AbstractDataPageIO dataIO = PageIO.getPageIO(dataAddr);
// Attempt to add page failed: the node page is full.
if (isReuseBucket(bucket)) {
@@ -1149,7 +1148,7 @@ public abstract class PagesList extends DataStructure {
final long dataId,
final long dataPage,
final long dataAddr,
- DataPageIO dataIO,
+ AbstractDataPageIO dataIO,
int bucket)
throws IgniteCheckedException {
final long pageId = dataIO.getFreeListPageId(dataAddr);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
new file mode 100644
index 0000000..f0b2e16
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
+import org.apache.ignite.internal.processors.cache.IncompleteObject;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.RootPage;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.SimpleDataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.jetbrains.annotations.NotNull;
+
+import static org.apache.ignite.internal.pagemem.PageIdUtils.itemId;
+import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
+
+/**
+ * General purpose key-value local-only storage.
+ */
+public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, ReadWriteMetastorage {
+ /** */
+ public static final String METASTORAGE_CACHE_NAME = "MetaStorage";
+
+ /** */
+ public static final int METASTORAGE_CACHE_ID = CU.cacheId(METASTORAGE_CACHE_NAME);
+
+ /** */
+ private final IgniteWriteAheadLogManager wal;
+
+ /** */
+ private final DataRegion dataRegion;
+
+ /** */
+ private MetastorageTree tree;
+
+ /** */
+ private AtomicLong rmvId = new AtomicLong();
+
+ /** */
+ private DataRegionMetricsImpl regionMetrics;
+
+ /** */
+ private boolean readOnly;
+
+ /** */
+ private boolean empty;
+
+ /** */
+ private RootPage treeRoot;
+
+ /** */
+ private RootPage reuseListRoot;
+
+ /** */
+ private FreeListImpl freeList;
+
+ /** */
+ private Map<String, byte[]> lastUpdates;
+
+ /** */
+ private final Marshaller marshaller = new JdkMarshaller();
+
+ /** */
+ public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
+ boolean readOnly) {
+ this.wal = wal;
+ this.dataRegion = dataRegion;
+ this.regionMetrics = regionMetrics;
+ this.readOnly = readOnly;
+ }
+
+ /** */
+ public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
+ this(wal, memPlc, memMetrics, false);
+ }
+
+ /** */
+ public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedException {
+ getOrAllocateMetas();
+
+ if (!empty) {
+ freeList = new FreeListImpl(METASTORAGE_CACHE_ID, "metastorage",
+ regionMetrics, dataRegion, null, wal, reuseListRoot.pageId().pageId(),
+ reuseListRoot.isAllocated());
+
+ MetastorageRowStore rowStore = new MetastorageRowStore(freeList, db);
+
+ tree = new MetastorageTree(METASTORAGE_CACHE_ID, dataRegion.pageMemory(), wal, rmvId,
+ freeList, rowStore, treeRoot.pageId().pageId(), treeRoot.isAllocated());
+
+ if (!readOnly)
+ ((GridCacheDatabaseSharedManager)db).addCheckpointListener(this);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Serializable read(String key) throws IgniteCheckedException {
+ byte[] data = getData(key);
+
+ Object result = null;
+
+ if (data != null)
+ result = marshaller.unmarshal(data, getClass().getClassLoader());
+
+ return (Serializable) result;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(@NotNull String key, @NotNull Serializable val) throws IgniteCheckedException {
+ assert val != null;
+
+ byte[] data = marshaller.marshal(val);
+
+ putData(key, data);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove(@NotNull String key) throws IgniteCheckedException {
+ removeData(key);
+ }
+
+ /** */
+ public void putData(String key, byte[] data) throws IgniteCheckedException {
+ if (!readOnly) {
+ WALPointer ptr = wal.log(new MetastoreDataRecord(key, data));
+
+ wal.fsync(ptr);
+
+ synchronized (this) {
+ MetastorageDataRow oldRow = tree.findOne(new MetastorageDataRow(key, null));
+
+ if (oldRow != null) {
+ tree.removex(oldRow);
+ tree.rowStore().removeRow(oldRow.link());
+ }
+
+ MetastorageDataRow row = new MetastorageDataRow(key, data);
+ tree.rowStore().addRow(row);
+ tree.put(row);
+ }
+ }
+ }
+
+ /** */
+ public byte[] getData(String key) throws IgniteCheckedException {
+ if (readOnly) {
+ if (lastUpdates != null) {
+ byte[] lastUpdate = lastUpdates.get(key);
+
+ if (lastUpdate != null)
+ return lastUpdate;
+ }
+
+ if (empty)
+ return null;
+ }
+
+ MetastorageDataRow row = tree.findOne(new MetastorageDataRow(key, null));
+
+ if (row == null)
+ return null;
+
+ return row.value();
+ }
+
+ /** */
+ public void removeData(String key) throws IgniteCheckedException {
+ if (!readOnly) {
+ WALPointer ptr = wal.log(new MetastoreDataRecord(key, null));
+
+ wal.fsync(ptr);
+
+ synchronized (this) {
+ MetastorageDataRow row = new MetastorageDataRow(key, null);
+ MetastorageDataRow oldRow = tree.findOne(row);
+
+ if (oldRow != null) {
+ tree.removex(oldRow);
+ tree.rowStore().removeRow(oldRow.link());
+ }
+ }
+ }
+ }
+
+ /** */
+ private void getOrAllocateMetas() throws IgniteCheckedException {
+ PageMemoryEx pageMem = (PageMemoryEx)dataRegion.pageMemory();
+
+ int partId = 0;
+
+ long partMetaId = pageMem.partitionMetaPageId(METASTORAGE_CACHE_ID, partId);
+ long partMetaPage = pageMem.acquirePage(METASTORAGE_CACHE_ID, partMetaId);
+ try {
+ if (readOnly) {
+ long pageAddr = pageMem.readLock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+
+ try {
+ if (PageIO.getType(pageAddr) != PageIO.T_PART_META) {
+ empty = true;
+
+ return;
+ }
+
+ PagePartitionMetaIO io = PageIO.getPageIO(pageAddr);
+
+ treeRoot = new RootPage(new FullPageId(io.getTreeRoot(pageAddr), METASTORAGE_CACHE_ID), false);
+ reuseListRoot = new RootPage(new FullPageId(io.getReuseListRoot(pageAddr), METASTORAGE_CACHE_ID), false);
+ }
+ finally {
+ pageMem.readUnlock(METASTORAGE_CACHE_ID, partId, partMetaPage);
+ }
+ }
+ else {
+ boolean allocated = false;
+ long pageAddr = pageMem.writeLock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+
+ try {
+ long treeRoot, reuseListRoot;
+
+ if (PageIO.getType(pageAddr) != PageIO.T_PART_META) {
+ // Initialize new page.
+ PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.latest();
+
+ io.initNewPage(pageAddr, partMetaId, pageMem.pageSize());
+
+ treeRoot = pageMem.allocatePage(METASTORAGE_CACHE_ID, partId, PageMemory.FLAG_DATA);
+ reuseListRoot = pageMem.allocatePage(METASTORAGE_CACHE_ID, partId, PageMemory.FLAG_DATA);
+
+ assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA;
+ assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA;
+
+ io.setTreeRoot(pageAddr, treeRoot);
+ io.setReuseListRoot(pageAddr, reuseListRoot);
+
+ if (PageHandler.isWalDeltaRecordNeeded(pageMem, METASTORAGE_CACHE_ID, partMetaId, partMetaPage, wal, null))
+ wal.log(new MetaPageInitRecord(
+ METASTORAGE_CACHE_ID,
+ partMetaId,
+ io.getType(),
+ io.getVersion(),
+ treeRoot,
+ reuseListRoot
+ ));
+
+ allocated = true;
+ }
+ else {
+ PagePartitionMetaIO io = PageIO.getPageIO(pageAddr);
+
+ treeRoot = io.getTreeRoot(pageAddr);
+ reuseListRoot = io.getReuseListRoot(pageAddr);
+
+ assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA :
+ U.hexLong(treeRoot) + ", part=" + partId + ", METASTORAGE_CACHE_ID=" + METASTORAGE_CACHE_ID;
+ assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA :
+ U.hexLong(reuseListRoot) + ", part=" + partId + ", METASTORAGE_CACHE_ID=" + METASTORAGE_CACHE_ID;
+ }
+
+ this.treeRoot = new RootPage(new FullPageId(treeRoot, METASTORAGE_CACHE_ID), allocated);
+ this.reuseListRoot = new RootPage(new FullPageId(reuseListRoot, METASTORAGE_CACHE_ID), allocated);
+ }
+ finally {
+ pageMem.writeUnlock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage, null, allocated);
+ }
+ }
+ }
+ finally {
+ pageMem.releasePage(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+ }
+ }
+
+ /**
+ * @return Page memory.
+ */
+ public PageMemory pageMemory() {
+ return dataRegion.pageMemory();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
+ freeList.saveMetadata();
+
+ MetastorageRowStore rowStore = tree.rowStore();
+
+ saveStoreMetadata(rowStore, ctx);
+ }
+
+ /**
+ * @param rowStore Store to save metadata.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void saveStoreMetadata(MetastorageRowStore rowStore, Context ctx) throws IgniteCheckedException {
+ FreeListImpl freeList = (FreeListImpl)rowStore.freeList();
+
+ freeList.saveMetadata();
+ }
+
+ public void applyUpdate(String key, byte[] value) throws IgniteCheckedException {
+ if (readOnly) {
+ if (lastUpdates == null)
+ lastUpdates = new HashMap<>();
+
+ lastUpdates.put(key, value);
+ }
+ else {
+ if (value != null)
+ putData(key, value);
+ else
+ removeData(key);
+ }
+ }
+
+ /** */
+ public static class FreeListImpl extends AbstractFreeList<MetastorageDataRow> {
+ /** {@inheritDoc} */
+ FreeListImpl(int cacheId, String name, DataRegionMetricsImpl regionMetrics, DataRegion dataRegion,
+ ReuseList reuseList,
+ IgniteWriteAheadLogManager wal, long metaPageId, boolean initNew) throws IgniteCheckedException {
+ super(cacheId, name, regionMetrics, dataRegion, reuseList, wal, metaPageId, initNew);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IOVersions<? extends AbstractDataPageIO<MetastorageDataRow>> ioVersions() {
+ return SimpleDataPageIO.VERSIONS;
+ }
+
+ /**
+ * Read row from data pages.
+ */
+ final MetastorageDataRow readRow(String key, long link)
+ throws IgniteCheckedException {
+ assert link != 0 : "link";
+
+ long nextLink = link;
+ IncompleteObject incomplete = null;
+ int size = 0;
+
+ boolean first = true;
+
+ do {
+ final long pageId = pageId(nextLink);
+
+ final long page = pageMem.acquirePage(grpId, pageId);
+
+ try {
+ long pageAddr = pageMem.readLock(grpId, pageId, page); // Non-empty data page must not be recycled.
+
+ assert pageAddr != 0L : nextLink;
+
+ try {
+ SimpleDataPageIO io = (SimpleDataPageIO)ioVersions().forPage(pageAddr);
+
+ DataPagePayload data = io.readPayload(pageAddr, itemId(nextLink), pageMem.pageSize());
+
+ nextLink = data.nextLink();
+
+ if (first) {
+ if (nextLink == 0) {
+ // Fast path for a single page row.
+ return new MetastorageDataRow(link, key, SimpleDataPageIO.readPayload(pageAddr + data.offset()));
+ }
+
+ first = false;
+ }
+
+ ByteBuffer buf = pageMem.pageBuffer(pageAddr);
+
+ buf.position(data.offset());
+ buf.limit(data.offset() + data.payloadSize());
+
+ if (size == 0) {
+ if (buf.remaining() >= 4 && incomplete == null) {
+ // Just read size.
+ size = buf.getInt();
+ incomplete = new IncompleteObject(new byte[size]);
+ }
+ else {
+ if (incomplete == null)
+ incomplete = new IncompleteObject(new byte[4]);
+
+ incomplete.readData(buf);
+
+ if (incomplete.isReady()) {
+ size = ByteBuffer.wrap(incomplete.data()).order(buf.order()).getInt();
+ incomplete = new IncompleteObject(new byte[size]);
+ }
+ }
+ }
+
+ if (size != 0 && buf.remaining() > 0)
+ incomplete.readData(buf);
+ }
+ finally {
+ pageMem.readUnlock(grpId, pageId, page);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, pageId, page);
+ }
+ }
+ while (nextLink != 0);
+
+ assert incomplete.isReady();
+
+ return new MetastorageDataRow(link, key, incomplete.data());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java
new file mode 100644
index 0000000..271efdf
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import org.apache.ignite.internal.processors.cache.persistence.Storable;
+
+/**
+ *
+ */
+public class MetastorageDataRow implements MetastorageSearchRow, Storable {
+ /** */
+ private long link;
+
+ /** */
+ private String key;
+
+ /** */
+ private byte[] value;
+
+ /** */
+ public MetastorageDataRow(long link, String key, byte[] value) {
+ this.link = link;
+ this.key = key;
+ this.value = value;
+ }
+
+ /** */
+ public MetastorageDataRow(String key, byte[] value) {
+ this.key = key;
+ this.value = value;
+ }
+
+ /**
+ * @return Key.
+ */
+ public String key() {
+ return key;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int hash() {
+ return key.hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int partition() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void link(long link) {
+ this.link = link;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long link() {
+ return link;
+ }
+
+ /**
+ * @return Value.
+ */
+ public byte[] value() {
+ return value;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ return "key=" + key;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java
new file mode 100644
index 0000000..8ab418c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Listener for events of metastore lifecycle.
+ *
+ * Database manager is responsible for initializing metastore on node startup
+ * and notifying other components about its readiness.
+ */
+public interface MetastorageLifecycleListener {
+ /**
+ * Is called when metastorage is made ready for read-only operations very early on node startup phase.
+ *
+ * Reference for read-only metastorage should be used only within this method and shouldn't be stored
+ * to any field.
+ *
+ * @param metastorage Read-only meta storage.
+ */
+ public void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException;
+
+ /**
+ * Fully functional metastore capable of performing reading and writing operations.
+ *
+ * Components interested in using metastore are allowed to keep reference passed into the method
+ * in their fields.
+ *
+ * @param metastorage Fully functional meta storage.
+ */
+ public void onReadyForReadWrite(ReadWriteMetastorage metastorage) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageRowStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageRowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageRowStore.java
new file mode 100644
index 0000000..e4bafd5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageRowStore.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+
+/**
+ *
+ */
+public class MetastorageRowStore {
+ /** */
+ private final FreeList freeList;
+
+ /** */
+ protected final IgniteCacheDatabaseSharedManager db;
+
+ /** */
+ public MetastorageRowStore(FreeList freeList, IgniteCacheDatabaseSharedManager db) {
+ this.freeList = freeList;
+ this.db = db;
+ }
+
+ /**
+ * @param link Row link.
+ * @return Data row.
+ */
+ public MetastorageDataRow dataRow(String key, long link) throws IgniteCheckedException {
+ return ((MetaStorage.FreeListImpl)freeList).readRow(key, link);
+ }
+
+ /**
+ * @param link Row link.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void removeRow(long link) throws IgniteCheckedException {
+ assert link != 0;
+ db.checkpointReadLock();
+
+ try {
+ freeList.removeDataRowByLink(link);
+ }
+ finally {
+ db.checkpointReadUnlock();
+ }
+ }
+
+ /**
+ * @param row Row.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void addRow(MetastorageDataRow row) throws IgniteCheckedException {
+ db.checkpointReadLock();
+
+ try {
+ freeList.insertDataRow(row);
+ }
+ finally {
+ db.checkpointReadUnlock();
+ }
+ }
+
+ /**
+ * @param link Row link.
+ * @param row New row data.
+ * @return {@code True} if was able to update row.
+ * @throws IgniteCheckedException If failed.
+ */
+ public boolean updateRow(long link, MetastorageDataRow row) throws IgniteCheckedException {
+ return freeList.updateDataRow(link, row);
+ }
+
+ /**
+ * @return Free list.
+ */
+ public FreeList freeList() {
+ return freeList;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageSearchRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageSearchRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageSearchRow.java
new file mode 100644
index 0000000..601fbc1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageSearchRow.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+/**
+ *
+ */
+public interface MetastorageSearchRow {
+ /**
+ * @return Key.
+ */
+ public String key();
+
+ /**
+ * @return Link for this row.
+ */
+ public long link();
+
+ /**
+ * @return Key hash code.
+ */
+ public int hash();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
new file mode 100644
index 0000000..445522b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+
+/**
+ *
+ */
+public class MetastorageTree extends BPlusTree<MetastorageSearchRow, MetastorageDataRow> {
+ /** Max key length (bytes num) */
+ public static final int MAX_KEY_LEN = 64;
+
+ /** */
+ private MetastorageRowStore rowStore;
+
+ /**
+ * @param pageMem
+ * @param wal
+ * @param globalRmvId
+ * @param metaPageId
+ * @param reuseList
+ * @throws IgniteCheckedException
+ */
+ public MetastorageTree(int cacheId,
+ PageMemory pageMem,
+ IgniteWriteAheadLogManager wal,
+ AtomicLong globalRmvId,
+ ReuseList reuseList,
+ MetastorageRowStore rowStore,
+ long metaPageId,
+ boolean initNew) throws IgniteCheckedException {
+ super("Metastorage", cacheId, pageMem, wal,
+ globalRmvId, metaPageId, reuseList, MetastorageInnerIO.VERSIONS, MetastoreLeafIO.VERSIONS);
+
+ this.rowStore = rowStore;
+
+ initTree(initNew);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int compare(BPlusIO<MetastorageSearchRow> io, long pageAddr, int idx,
+ MetastorageSearchRow row) throws IgniteCheckedException {
+
+ String key = ((DataLinkIO)io).getKey(pageAddr, idx);
+
+ return key.compareTo(row.key());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected MetastorageDataRow getRow(BPlusIO<MetastorageSearchRow> io, long pageAddr, int idx,
+ Object x) throws IgniteCheckedException {
+ long link = ((DataLinkIO)io).getLink(pageAddr, idx);
+ String key = ((DataLinkIO)io).getKey(pageAddr, idx);
+
+ return rowStore.dataRow(key, link);
+ }
+
+ /**
+ * @return RowStore.
+ */
+ public MetastorageRowStore rowStore() {
+ return rowStore;
+ }
+
+ /**
+ *
+ */
+ private interface DataLinkIO {
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @return Row link.
+ */
+ public long getLink(long pageAddr, int idx);
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @return Key size in bytes.
+ */
+ public short getKeySize(long pageAddr, int idx);
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @return Key.
+ */
+ public String getKey(long pageAddr, int idx);
+ }
+
+ /**
+ *
+ */
+ public static class MetastorageInnerIO extends BPlusInnerIO<MetastorageSearchRow> implements DataLinkIO {
+ /** */
+ public static final IOVersions<MetastorageInnerIO> VERSIONS = new IOVersions<>(
+ new MetastorageInnerIO(1)
+ );
+
+ /**
+ * @param ver Page format version.
+ */
+ MetastorageInnerIO(int ver) {
+ super(T_DATA_REF_METASTORAGE_INNER, ver, true, 10 + MAX_KEY_LEN);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void storeByOffset(long pageAddr, int off,
+ MetastorageSearchRow row) throws IgniteCheckedException {
+ assert row.link() != 0;
+
+ PageUtils.putLong(pageAddr, off, row.link());
+
+ byte[] bytes = row.key().getBytes();
+ assert bytes.length <= MAX_KEY_LEN;
+
+ PageUtils.putShort(pageAddr, off + 8, (short)bytes.length);
+ PageUtils.putBytes(pageAddr, off + 10, bytes);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void store(long dstPageAddr, int dstIdx, BPlusIO<MetastorageSearchRow> srcIo, long srcPageAddr,
+ int srcIdx) throws IgniteCheckedException {
+ int srcOff = srcIo.offset(srcIdx);
+ int dstOff = offset(dstIdx);
+
+ long link = ((DataLinkIO)srcIo).getLink(srcPageAddr, srcIdx);
+ short len = ((DataLinkIO)srcIo).getKeySize(srcPageAddr, srcIdx);
+
+ byte[] payload = PageUtils.getBytes(srcPageAddr, srcOff + 10, len);
+
+ PageUtils.putLong(dstPageAddr, dstOff, link);
+ PageUtils.putShort(dstPageAddr, dstOff + 8, len);
+ PageUtils.putBytes(dstPageAddr, dstOff + 10, payload);
+ }
+
+ /** {@inheritDoc} */
+ @Override public MetastorageSearchRow getLookupRow(BPlusTree<MetastorageSearchRow, ?> tree, long pageAddr,
+ int idx) throws IgniteCheckedException {
+ long link = getLink(pageAddr, idx);
+ String key = getKey(pageAddr, idx);
+
+ return new MetsatorageSearchRowImpl(key, link);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLink(long pageAddr, int idx) {
+ assert idx < getCount(pageAddr) : idx;
+
+ return PageUtils.getLong(pageAddr, offset(idx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public short getKeySize(long pageAddr, int idx) {
+ return PageUtils.getShort(pageAddr, offset(idx) + 8);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getKey(long pageAddr, int idx) {
+ int len = PageUtils.getShort(pageAddr, offset(idx) + 8);
+ byte[] bytes = PageUtils.getBytes(pageAddr, offset(idx) + 10, len);
+ return new String(bytes);
+ }
+ }
+
+ /**
+ *
+ */
+ public static class MetastoreLeafIO extends BPlusLeafIO<MetastorageSearchRow> implements DataLinkIO {
+ /** */
+ public static final IOVersions<MetastoreLeafIO> VERSIONS = new IOVersions<>(
+ new MetastoreLeafIO(1)
+ );
+
+ /**
+ * @param ver Page format version.
+ */
+ MetastoreLeafIO(int ver) {
+ super(T_DATA_REF_METASTORAGE_LEAF, ver, 10 + MAX_KEY_LEN);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void storeByOffset(long pageAddr, int off,
+ MetastorageSearchRow row) throws IgniteCheckedException {
+ assert row.link() != 0;
+
+ PageUtils.putLong(pageAddr, off, row.link());
+ byte[] bytes = row.key().getBytes();
+
+ assert bytes.length <= MAX_KEY_LEN;
+
+ PageUtils.putShort(pageAddr, off + 8, (short)bytes.length);
+ PageUtils.putBytes(pageAddr, off + 10, bytes);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void store(long dstPageAddr, int dstIdx, BPlusIO<MetastorageSearchRow> srcIo, long srcPageAddr,
+ int srcIdx) throws IgniteCheckedException {
+ int srcOff = srcIo.offset(srcIdx);
+ int dstOff = offset(dstIdx);
+
+ long link = ((DataLinkIO)srcIo).getLink(srcPageAddr, srcIdx);
+ short len = ((DataLinkIO)srcIo).getKeySize(srcPageAddr, srcIdx);
+
+ byte[] payload = PageUtils.getBytes(srcPageAddr, srcOff + 10, len);
+
+ PageUtils.putLong(dstPageAddr, dstOff, link);
+ PageUtils.putShort(dstPageAddr, dstOff + 8, len);
+ PageUtils.putBytes(dstPageAddr, dstOff + 10, payload);
+ }
+
+ /** {@inheritDoc} */
+ @Override public MetastorageSearchRow getLookupRow(BPlusTree<MetastorageSearchRow, ?> tree, long pageAddr,
+ int idx) throws IgniteCheckedException {
+ long link = getLink(pageAddr, idx);
+ String key = getKey(pageAddr, idx);
+
+ return new MetsatorageSearchRowImpl(key, link);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLink(long pageAddr, int idx) {
+ assert idx < getCount(pageAddr) : idx;
+
+ return PageUtils.getLong(pageAddr, offset(idx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public short getKeySize(long pageAddr, int idx) {
+ return PageUtils.getShort(pageAddr, offset(idx) + 8);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getKey(long pageAddr, int idx) {
+ int len = PageUtils.getShort(pageAddr, offset(idx) + 8);
+ byte[] bytes = PageUtils.getBytes(pageAddr, offset(idx) + 10, len);
+ return new String(bytes);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetsatorageSearchRowImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetsatorageSearchRowImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetsatorageSearchRowImpl.java
new file mode 100644
index 0000000..363c14d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetsatorageSearchRowImpl.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+/**
+ *
+ */
+public class MetsatorageSearchRowImpl implements MetastorageSearchRow {
+ /** */
+ private final String key;
+ /** */
+ private final long link;
+
+ /**
+ * @param key Key.
+ * @param link Link.
+ */
+ public MetsatorageSearchRowImpl(String key, long link) {
+ this.key = key;
+ this.link = link;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String key() {
+ return key;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public long link() {
+ return link;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int hash() {
+ return key.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java
new file mode 100644
index 0000000..9b73676
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadOnlyMetastorage.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ *
+ */
+public interface ReadOnlyMetastorage {
+ /** */
+ Serializable read(String key) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadWriteMetastorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadWriteMetastorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadWriteMetastorage.java
new file mode 100644
index 0000000..ab48afc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/ReadWriteMetastorage.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public interface ReadWriteMetastorage extends ReadOnlyMetastorage {
+ /** */
+ public void write(@NotNull String key, @NotNull Serializable val) throws IgniteCheckedException;
+
+ /** */
+ public void remove(@NotNull String key) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
index 53e21b7..9c37508 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
@@ -33,17 +33,17 @@ import org.jetbrains.annotations.Nullable;
public interface PageMemoryEx extends PageMemory {
/**
*
- * @param cacheId Cache ID.
+ * @param grpId Group ID.
* @param pageId Page ID.
* @param page Page pointer.
* @param restore Determines if the page is locked for restore.
* @return ByteBuffer for modifying the page.
*/
- long writeLock(int cacheId, long pageId, long page, boolean restore);
+ long writeLock(int grpId, long pageId, long page, boolean restore);
/**
*
- * @param cacheId Cache ID.
+ * @param grpId Group ID.
* @param pageId Page ID.
* @param page Page pointer.
* @param walPlc {@code True} if page should be recorded to WAL, {@code false} if the page must not
@@ -51,27 +51,27 @@ public interface PageMemoryEx extends PageMemory {
* @param dirtyFlag Determines whether the page was modified since the last checkpoint.
* @param restore Determines if the page is locked for restore.
*/
- void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
+ void writeUnlock(int grpId, long pageId, long page, Boolean walPlc,
boolean dirtyFlag, boolean restore);
/**
- * Gets or allocates metadata page for specified cacheId.
+ * Gets or allocates metadata page for specified grpId.
*
- * @param cacheId Cache ID.
- * @return Meta page for cacheId.
+ * @param grpId Group ID.
+ * @return Meta page for grpId.
* @throws IgniteCheckedException If failed.
*/
- public long metaPageId(int cacheId) throws IgniteCheckedException;
+ public long metaPageId(int grpId) throws IgniteCheckedException;
/**
- * Gets or allocates partition metadata page for specified cacheId and partId.
+ * Gets or allocates partition metadata page for specified grpId and partId.
*
- * @param cacheId Cache ID.
+ * @param grpId Group ID.
* @param partId Partition ID.
- * @return Meta page for cacheId and partId.
+ * @return Meta page for grpId and partId.
* @throws IgniteCheckedException If failed.
*/
- public long partitionMetaPageId(int cacheId, int partId) throws IgniteCheckedException;
+ public long partitionMetaPageId(int grpId, int partId) throws IgniteCheckedException;
/**
* @see #acquirePage(int, long)
@@ -125,11 +125,11 @@ public interface PageMemoryEx extends PageMemory {
/**
* Marks partition as invalid / outdated.
*
- * @param cacheId Cache ID.
+ * @param grpId Group ID.
* @param partId Partition ID.
* @return New partition tag (growing 1-based partition file version).
*/
- public int invalidate(int cacheId, int partId);
+ public int invalidate(int grpId, int partId);
/**
* Clears internal metadata of destroyed cache group.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 623bf54..90fde31 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.persistence.pagemem;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -58,8 +59,8 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
@@ -77,6 +78,7 @@ import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
+import org.jetbrains.annotations.Nullable;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
@@ -123,7 +125,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Invalid relative pointer value. */
private static final long INVALID_REL_PTR = RELATIVE_PTR_MASK;
- /** */
+ /** Pointer which means that this page is outdated (for example, cache was destroyed, partition eviction'd happened */
private static final long OUTDATED_REL_PTR = INVALID_REL_PTR + 1;
/** Address mask to avoid ABA problem. */
@@ -214,8 +216,11 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Flush dirty page closure. When possible, will be called by evictPage(). */
private final GridInClosure3X<FullPageId, ByteBuffer, Integer> flushDirtyPage;
- /** Flush dirty page closure. When possible, will be called by evictPage(). */
- private final GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
+ /**
+ * Flush dirty page closure. When possible, will be called by evictPage().
+ * {@code Null} if page tracking functionality is disabled
+ * */
+ @Nullable private final GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
/** Pages write throttle. */
private PagesWriteThrottle writeThrottle;
@@ -246,7 +251,7 @@ public class PageMemoryImpl implements PageMemoryEx {
GridCacheSharedContext<?, ?> ctx,
int pageSize,
GridInClosure3X<FullPageId, ByteBuffer, Integer> flushDirtyPage,
- GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker,
+ @Nullable GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker,
CheckpointLockStateChecker stateChecker,
DataRegionMetricsImpl memMetrics,
boolean throttleEnabled
@@ -351,8 +356,10 @@ public class PageMemoryImpl implements PageMemoryEx {
U.shutdownNow(getClass(), asyncRunner, log);
- for (Segment seg : segments)
- seg.close();
+ if (segments != null) {
+ for (Segment seg : segments)
+ seg.close();
+ }
directMemoryProvider.shutdown();
}
@@ -387,8 +394,8 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public long writeLock(int cacheId, long pageId, long page, boolean restore) {
- return writeLockPage(page, new FullPageId(pageId, cacheId), !restore);
+ @Override public long writeLock(int grpId, long pageId, long page, boolean restore) {
+ return writeLockPage(page, new FullPageId(pageId, grpId), !restore);
}
/** {@inheritDoc} */
@@ -403,9 +410,9 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
+ @Override public void writeUnlock(int grpId, long pageId, long page, Boolean walPlc,
boolean dirtyFlag, boolean restore) {
- writeUnlockPage(page, new FullPageId(pageId, cacheId), walPlc, dirtyFlag, restore);
+ writeUnlockPage(page, new FullPageId(pageId, grpId), walPlc, dirtyFlag, restore);
}
/** {@inheritDoc} */
@@ -436,7 +443,7 @@ public class PageMemoryImpl implements PageMemoryEx {
seg.writeLock().lock();
- boolean isTrackingPage = trackingIO.trackingPageFor(pageId, pageSize()) == pageId;
+ boolean isTrackingPage = changeTracker != null && trackingIO.trackingPageFor(pageId, pageSize()) == pageId;
try {
long relPtr = seg.loadedPages.get(
@@ -555,12 +562,12 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public long metaPageId(int cacheId) throws IgniteCheckedException {
- return storeMgr.metaPageId(cacheId);
+ @Override public long metaPageId(int grpId) throws IgniteCheckedException {
+ return storeMgr.metaPageId(grpId);
}
/** {@inheritDoc} */
- @Override public long partitionMetaPageId(int cacheId, int partId) throws IgniteCheckedException {
+ @Override public long partitionMetaPageId(int grpId, int partId) throws IgniteCheckedException {
return PageIdUtils.pageId(partId, PageIdAllocator.FLAG_DATA, 0);
}
@@ -846,9 +853,11 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public boolean safeToUpdate() {
- for (Segment segment : segments)
- if (!segment.safeToUpdate())
- return false;
+ if (segments != null) {
+ for (Segment segment : segments)
+ if (!segment.safeToUpdate())
+ return false;
+ }
return true;
}
@@ -867,6 +876,9 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@Override public GridMultiCollectionWrapper<FullPageId> beginCheckpoint() throws IgniteException {
+ if (segments == null)
+ return new GridMultiCollectionWrapper<>(Collections.<FullPageId>emptyList());
+
Collection[] collections = new Collection[segments.length];
for (int i = 0; i < segments.length; i++) {
@@ -888,6 +900,9 @@ public class PageMemoryImpl implements PageMemoryEx {
/** {@inheritDoc} */
@SuppressWarnings({"unchecked", "TooBroadScope"})
@Override public void finishCheckpoint() {
+ if (segments == null)
+ return;
+
for (Segment seg : segments)
seg.segCheckpointPages = null;
@@ -1073,14 +1088,14 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/** {@inheritDoc} */
- @Override public int invalidate(int cacheId, int partId) {
+ @Override public int invalidate(int grpId, int partId) {
int tag = 0;
for (Segment seg : segments) {
seg.writeLock().lock();
try {
- int newTag = seg.incrementPartTag(cacheId, partId);
+ int newTag = seg.incrementPartTag(grpId, partId);
if (tag == 0)
tag = newTag;
@@ -1322,7 +1337,7 @@ public class PageMemoryImpl implements PageMemoryEx {
boolean dirty = isDirty(page);
//if page is for restore, we shouldn't mark it as changed
- if (!restore && markDirty && !dirty)
+ if (!restore && markDirty && !dirty && changeTracker != null)
changeTracker.apply(page, fullId, this);
boolean pageWalRec = markDirty && walPlc != FALSE && (walPlc == TRUE || !dirty);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f7aba85/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
index e7170c3..f4508ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PagesAllocationRange.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.processors.cache.persistence.partstate;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
/**
* Range of pages allocated.
* Contains currently allocated page count and previously observed page count.
@@ -60,9 +62,6 @@ public class PagesAllocationRange {
/** {@inheritDoc} */
@Override public String toString() {
- return "PagesAllocationRange{" +
- "lastAllocatedPageCnt=" + lastAllocatedPageCnt +
- ", currAllocatedPageCnt=" + currAllocatedPageCnt +
- '}';
+ return S.toString(PagesAllocationRange.class, this);
}
}