You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by nt...@apache.org on 2015/10/22 12:34:27 UTC
[1/9] ignite git commit: IGNITE-426 temp commit.
Repository: ignite
Updated Branches:
refs/heads/ignite-426-2-reb [created] 1725d51f4
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
new file mode 100644
index 0000000..ed856a5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
@@ -0,0 +1,1104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryListenerException;
+import javax.cache.event.CacheEntryUpdatedListener;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.CacheAtomicWriteOrderMode;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.continuous.GridContinuousHandler;
+import org.apache.ignite.internal.processors.continuous.GridContinuousMessage;
+import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.T3;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.CLOCK;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommonAbstractTest {
+ /** */
+ private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final int BACKUP_ACK_THRESHOLD = 100;
+
+ /** */
+ private static volatile boolean err;
+
+ /** */
+ private boolean client;
+
+ /** */
+ private int backups = 1;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setForceServerMode(true);
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+ TestCommunicationSpi commSpi = new TestCommunicationSpi();
+
+ commSpi.setIdleConnectionTimeout(100);
+
+ cfg.setCommunicationSpi(commSpi);
+
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ ccfg.setCacheMode(cacheMode());
+ ccfg.setAtomicityMode(atomicityMode());
+ ccfg.setAtomicWriteOrderMode(writeOrderMode());
+ ccfg.setBackups(backups);
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ cfg.setClientMode(client);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 5 * 60_000;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ err = false;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+ }
+
+ /**
+ * @return Cache mode.
+ */
+ protected abstract CacheMode cacheMode();
+
+ /**
+ * @return Atomicity mode.
+ */
+ protected abstract CacheAtomicityMode atomicityMode();
+
+ /**
+ * @return Write order mode for atomic cache.
+ */
+ protected CacheAtomicWriteOrderMode writeOrderMode() {
+ return CLOCK;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRebalanceVersion() throws Exception {
+ Ignite ignite0 = startGrid(0);
+ GridDhtPartitionTopology top0 = ((IgniteKernal)ignite0).context().cache().context().cacheContext(1).topology();
+
+ assertTrue(top0.rebalanceFinished(new AffinityTopologyVersion(1)));
+ assertFalse(top0.rebalanceFinished(new AffinityTopologyVersion(2)));
+
+ Ignite ignite1 = startGrid(1);
+ GridDhtPartitionTopology top1 = ((IgniteKernal)ignite1).context().cache().context().cacheContext(1).topology();
+
+ waitRebalanceFinished(ignite0, 2);
+ waitRebalanceFinished(ignite1, 2);
+
+ assertFalse(top0.rebalanceFinished(new AffinityTopologyVersion(3)));
+ assertFalse(top1.rebalanceFinished(new AffinityTopologyVersion(3)));
+
+ Ignite ignite2 = startGrid(2);
+ GridDhtPartitionTopology top2 = ((IgniteKernal)ignite2).context().cache().context().cacheContext(1).topology();
+
+ waitRebalanceFinished(ignite0, 3);
+ waitRebalanceFinished(ignite1, 3);
+ waitRebalanceFinished(ignite2, 3);
+
+ assertFalse(top0.rebalanceFinished(new AffinityTopologyVersion(4)));
+ assertFalse(top1.rebalanceFinished(new AffinityTopologyVersion(4)));
+ assertFalse(top2.rebalanceFinished(new AffinityTopologyVersion(4)));
+
+ client = true;
+
+ Ignite ignite3 = startGrid(3);
+ GridDhtPartitionTopology top3 = ((IgniteKernal)ignite3).context().cache().context().cacheContext(1).topology();
+
+ assertTrue(top0.rebalanceFinished(new AffinityTopologyVersion(4)));
+ assertTrue(top1.rebalanceFinished(new AffinityTopologyVersion(4)));
+ assertTrue(top2.rebalanceFinished(new AffinityTopologyVersion(4)));
+ assertTrue(top3.rebalanceFinished(new AffinityTopologyVersion(4)));
+
+ stopGrid(1);
+
+ waitRebalanceFinished(ignite0, 5);
+ waitRebalanceFinished(ignite2, 5);
+ waitRebalanceFinished(ignite3, 5);
+
+ stopGrid(3);
+
+ assertTrue(top0.rebalanceFinished(new AffinityTopologyVersion(6)));
+ assertTrue(top2.rebalanceFinished(new AffinityTopologyVersion(6)));
+
+ stopGrid(0);
+
+ waitRebalanceFinished(ignite2, 7);
+ }
+
+ /**
+ * @param ignite Ignite.
+ * @param topVer Topology version.
+ * @throws Exception If failed.
+ */
+ private void waitRebalanceFinished(Ignite ignite, long topVer) throws Exception {
+ final AffinityTopologyVersion topVer0 = new AffinityTopologyVersion(topVer);
+
+ final GridDhtPartitionTopology top =
+ ((IgniteKernal)ignite).context().cache().context().cacheContext(1).topology();
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return top.rebalanceFinished(topVer0);
+ }
+ }, 5000);
+
+ assertTrue(top.rebalanceFinished(topVer0));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testOneBackup() throws Exception {
+ checkBackupQueue(1, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testOneBackupClientUpdate() throws Exception {
+ checkBackupQueue(1, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testThreeBackups() throws Exception {
+ if (cacheMode() == REPLICATED)
+ return;
+
+ checkBackupQueue(3, false);
+ }
+
+ /**
+ * @param backups Number of backups.
+ * @param updateFromClient If {@code true} executes cache update from client node.
+ * @throws Exception If failed.
+ */
+ private void checkBackupQueue(int backups, boolean updateFromClient) throws Exception {
+ this.backups = backups;
+
+ final int SRV_NODES = 4;
+
+ startGridsMultiThreaded(SRV_NODES);
+
+ client = true;
+
+ Ignite qryClient = startGrid(SRV_NODES);
+
+ client = false;
+
+ IgniteCache<Object, Object> qryClientCache = qryClient.cache(null);
+
+ if (cacheMode() != REPLICATED)
+ assertEquals(backups, qryClientCache.getConfiguration(CacheConfiguration.class).getBackups());
+
+ Affinity<Object> aff = qryClient.affinity(null);
+
+ CacheEventListener1 lsnr = new CacheEventListener1(false);
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ QueryCursor<?> cur = qryClientCache.query(qry);
+
+ int PARTS = 10;
+
+ Map<Object, T2<Object, Object>> updates = new HashMap<>();
+
+ List<T3<Object, Object, Object>> expEvts = new ArrayList<>();
+
+ for (int i = 0; i < SRV_NODES - 1; i++) {
+ log.info("Stop iteration: " + i);
+
+ TestCommunicationSpi spi = (TestCommunicationSpi)ignite(i).configuration().getCommunicationSpi();
+
+ Ignite ignite = ignite(i);
+
+ IgniteCache<Object, Object> cache = ignite.cache(null);
+
+ List<Integer> keys = testKeys(cache, PARTS);
+
+ CountDownLatch latch = new CountDownLatch(keys.size());
+
+ lsnr.latch = latch;
+
+ boolean first = true;
+
+ for (Integer key : keys) {
+ log.info("Put [node=" + ignite.name() + ", key=" + key + ", part=" + aff.partition(key) + ']');
+
+ T2<Object, Object> t = updates.get(key);
+
+ if (t == null) {
+ updates.put(key, new T2<>((Object)key, null));
+
+ expEvts.add(new T3<>((Object)key, (Object)key, null));
+ }
+ else {
+ updates.put(key, new T2<>((Object)key, (Object)key));
+
+ expEvts.add(new T3<>((Object)key, (Object)key, (Object)key));
+ }
+
+ if (updateFromClient)
+ qryClientCache.put(key, key);
+ else
+ cache.put(key, key);
+
+ if (first) {
+ spi.skipMsg = true;
+
+ first = false;
+ }
+ }
+
+ stopGrid(i);
+
+ if (!latch.await(5, SECONDS)) {
+ Set<Integer> keys0 = new HashSet<>(keys);
+
+ keys0.removeAll(lsnr.keys);
+
+ log.info("Missed events for keys: " + keys0);
+
+ fail("Failed to wait for notifications [exp=" + keys.size() + ", left=" + lsnr.latch.getCount() + ']');
+ }
+
+ checkEvents(expEvts, lsnr);
+ }
+
+ for (int i = 0; i < SRV_NODES - 1; i++) {
+ log.info("Start iteration: " + i);
+
+ Ignite ignite = startGrid(i);
+
+ IgniteCache<Object, Object> cache = ignite.cache(null);
+
+ List<Integer> keys = testKeys(cache, PARTS);
+
+ CountDownLatch latch = new CountDownLatch(keys.size());
+
+ lsnr.latch = latch;
+
+ for (Integer key : keys) {
+ log.info("Put [node=" + ignite.name() + ", key=" + key + ", part=" + aff.partition(key) + ']');
+
+ T2<Object, Object> t = updates.get(key);
+
+ if (t == null) {
+ updates.put(key, new T2<>((Object)key, null));
+
+ expEvts.add(new T3<>((Object)key, (Object)key, null));
+ }
+ else {
+ updates.put(key, new T2<>((Object)key, (Object)key));
+
+ expEvts.add(new T3<>((Object)key, (Object)key, (Object)key));
+ }
+
+ if (updateFromClient)
+ qryClientCache.put(key, key);
+ else
+ cache.put(key, key);
+ }
+
+ if (!latch.await(5, SECONDS)) {
+ Set<Integer> keys0 = new HashSet<>(keys);
+
+ keys0.removeAll(lsnr.keys);
+
+ log.info("Missed events for keys: " + keys0);
+
+ fail("Failed to wait for notifications [exp=" + keys.size() + ", left=" + lsnr.latch.getCount() + ']');
+ }
+
+ checkEvents(expEvts, lsnr);
+ }
+
+ cur.close();
+
+ assertFalse("Unexpected error during test, see log for details.", err);
+ }
+
+ /**
+ * @param expEvts Expected events.
+ * @param lsnr Listener.
+ */
+ private void checkEvents(List<T3<Object, Object, Object>> expEvts, CacheEventListener1 lsnr) {
+ for (T3<Object, Object, Object> exp : expEvts) {
+ CacheEntryEvent<?, ?> e = lsnr.evts.get(exp.get1());
+
+ assertNotNull("No event for key: " + exp.get1(), e);
+ assertEquals("Unexpected value: " + e, exp.get2(), e.getValue());
+ assertEquals("Unexpected old value: " + e, exp.get3(), e.getOldValue());
+ }
+
+ expEvts.clear();
+
+ lsnr.evts.clear();
+ }
+
+ /**
+ * @param cache Cache.
+ * @param parts Number of partitions.
+ * @return Keys.
+ */
+ private List<Integer> testKeys(IgniteCache<Object, Object> cache, int parts) {
+ Ignite ignite = cache.unwrap(Ignite.class);
+
+ List<Integer> res = new ArrayList<>();
+
+ Affinity<Object> aff = ignite.affinity(cache.getName());
+
+ ClusterNode node = ignite.cluster().localNode();
+
+ int[] nodeParts = aff.primaryPartitions(node);
+
+ final int KEYS_PER_PART = 3;
+
+ for (int i = 0; i < parts; i++) {
+ int part = nodeParts[i];
+
+ int cnt = 0;
+
+ for (int key = 0; key < 100_000; key++) {
+ if (aff.partition(key) == part && aff.isPrimary(node, key)) {
+ res.add(key);
+
+ if (++cnt == KEYS_PER_PART)
+ break;
+ }
+ }
+
+ assertEquals(KEYS_PER_PART, cnt);
+ }
+
+ assertEquals(parts * KEYS_PER_PART, res.size());
+
+ return res;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testBackupQueueCleanupClientQuery() throws Exception {
+ startGridsMultiThreaded(2);
+
+ client = true;
+
+ Ignite qryClient = startGrid(2);
+
+ CacheEventListener1 lsnr = new CacheEventListener1(false);
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ QueryCursor<?> cur = qryClient.cache(null).query(qry);
+
+ final Collection<Object> backupQueue = backupQueue(ignite(1));
+
+ assertEquals(0, backupQueue.size());
+
+ IgniteCache<Object, Object> cache0 = ignite(0).cache(null);
+
+ List<Integer> keys = primaryKeys(cache0, BACKUP_ACK_THRESHOLD);
+
+ CountDownLatch latch = new CountDownLatch(keys.size());
+
+ lsnr.latch = latch;
+
+ for (Integer key : keys) {
+ log.info("Put: " + key);
+
+ cache0.put(key, key);
+ }
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return backupQueue.isEmpty();
+ }
+ }, 2000);
+
+ assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.size() < BACKUP_ACK_THRESHOLD);
+
+ if (!latch.await(5, SECONDS))
+ fail("Failed to wait for notifications [exp=" + keys.size() + ", left=" + lsnr.latch.getCount() + ']');
+
+ keys = primaryKeys(cache0, BACKUP_ACK_THRESHOLD / 2);
+
+ latch = new CountDownLatch(keys.size());
+
+ lsnr.latch = latch;
+
+ for (Integer key : keys)
+ cache0.put(key, key);
+
+ final long ACK_FREQ = 5000;
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return backupQueue.isEmpty();
+ }
+ }, ACK_FREQ + 2000);
+
+ assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.isEmpty());
+
+ if (!latch.await(5, SECONDS))
+ fail("Failed to wait for notifications [exp=" + keys.size() + ", left=" + lsnr.latch.getCount() + ']');
+
+ cur.close();
+
+ assertFalse("Unexpected error during test, see log for details.", err);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testBackupQueueCleanupServerQuery() throws Exception {
+ Ignite qryClient = startGridsMultiThreaded(2);
+
+ CacheEventListener1 lsnr = new CacheEventListener1(false);
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ IgniteCache<Object, Object> cache = qryClient.cache(null);
+
+ QueryCursor<?> cur = cache.query(qry);
+
+ final Collection<Object> backupQueue = backupQueue(ignite(1));
+
+ assertEquals(0, backupQueue.size());
+
+ List<Integer> keys = primaryKeys(cache, BACKUP_ACK_THRESHOLD);
+
+ CountDownLatch latch = new CountDownLatch(keys.size());
+
+ lsnr.latch = latch;
+
+ for (Integer key : keys) {
+ log.info("Put: " + key);
+
+ cache.put(key, key);
+ }
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return backupQueue.isEmpty();
+ }
+ }, 3000);
+
+ assertTrue("Backup queue is not cleared: " + backupQueue, backupQueue.size() < BACKUP_ACK_THRESHOLD);
+
+ if (!latch.await(5, SECONDS))
+ fail("Failed to wait for notifications [exp=" + keys.size() + ", left=" + lsnr.latch.getCount() + ']');
+
+ cur.close();
+ }
+
+ /**
+ * @param ignite Ignite.
+ * @return Backup queue for test query.
+ */
+ private Collection<Object> backupQueue(Ignite ignite) {
+ GridContinuousProcessor proc = ((IgniteKernal)ignite).context().continuous();
+
+ ConcurrentMap<Object, Object> infos = GridTestUtils.getFieldValue(proc, "rmtInfos");
+
+ Collection<Object> backupQueue = null;
+
+ for (Object info : infos.values()) {
+ GridContinuousHandler hnd = GridTestUtils.getFieldValue(info, "hnd");
+
+ if (hnd.isForQuery() && hnd.cacheName() == null) {
+ backupQueue = GridTestUtils.getFieldValue(hnd, "backupQueue");
+
+ break;
+ }
+ }
+
+ assertNotNull(backupQueue);
+
+ return backupQueue;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testFailover() throws Exception {
+ final int SRV_NODES = 4;
+
+ startGridsMultiThreaded(SRV_NODES);
+
+ client = true;
+
+ Ignite qryClient = startGrid(SRV_NODES);
+
+ client = false;
+
+ IgniteCache<Object, Object> qryClientCache = qryClient.cache(null);
+
+ final CacheEventListener2 lsnr = new CacheEventListener2();
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ QueryCursor<?> cur = qryClientCache.query(qry);
+
+ final AtomicBoolean stop = new AtomicBoolean();
+
+ final AtomicReference<CountDownLatch> checkLatch = new AtomicReference<>();
+
+ IgniteInternalFuture<?> restartFut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ final int idx = SRV_NODES + 1;
+
+ while (!stop.get() && !err) {
+ log.info("Start node: " + idx);
+
+ startGrid(idx);
+
+ Thread.sleep(3000);
+
+ log.info("Stop node: " + idx);
+
+ stopGrid(idx);
+
+ CountDownLatch latch = new CountDownLatch(1);
+
+ assertTrue(checkLatch.compareAndSet(null, latch));
+
+ if (!stop.get()) {
+ log.info("Wait for event check.");
+
+ assertTrue(latch.await(1, MINUTES));
+ }
+ }
+
+ return null;
+ }
+ });
+
+ final Map<Integer, Integer> vals = new HashMap<>();
+
+ final Map<Integer, List<T2<Integer, Integer>>> expEvts = new HashMap<>();
+
+ try {
+ long stopTime = System.currentTimeMillis() + 3 * 60_000;
+
+ final int PARTS = qryClient.affinity(null).partitions();
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ while (System.currentTimeMillis() < stopTime) {
+ Integer key = rnd.nextInt(PARTS);
+
+ Integer prevVal = vals.get(key);
+ Integer val = vals.get(key);
+
+ if (val == null)
+ val = 0;
+ else
+ val = val + 1;
+
+ qryClientCache.put(key, val);
+
+ vals.put(key, val);
+
+ List<T2<Integer, Integer>> keyEvts = expEvts.get(key);
+
+ if (keyEvts == null) {
+ keyEvts = new ArrayList<>();
+
+ expEvts.put(key, keyEvts);
+ }
+
+ keyEvts.add(new T2<>(val, prevVal));
+
+ CountDownLatch latch = checkLatch.get();
+
+ if (latch != null) {
+ log.info("Check events.");
+
+ checkLatch.set(null);
+
+ boolean success = false;
+
+ try {
+ if (err)
+ break;
+
+ boolean check = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return checkEvents(false, expEvts, lsnr);
+ }
+ }, 10_000);
+
+ if (!check)
+ assertTrue(checkEvents(true, expEvts, lsnr));
+
+ success = true;
+
+ log.info("Events checked.");
+ }
+ finally {
+ if (!success)
+ err = true;
+
+ latch.countDown();
+ }
+ }
+ }
+ }
+ finally {
+ stop.set(true);
+ }
+
+ CountDownLatch latch = checkLatch.get();
+
+ if (latch != null)
+ latch.countDown();
+
+ restartFut.get();
+
+ boolean check = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return checkEvents(false, expEvts, lsnr);
+ }
+ }, 10_000);
+
+ if (!check)
+ assertTrue(checkEvents(true, expEvts, lsnr));
+
+ cur.close();
+
+ assertFalse("Unexpected error during test, see log for details.", err);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMultiThreaded() throws Exception {
+ final int SRV_NODES = 3;
+
+ startGridsMultiThreaded(SRV_NODES);
+
+ client = true;
+
+ Ignite qryClient = startGrid(SRV_NODES);
+
+ final IgniteCache<Object, Object> cache = qryClient.cache(null);
+
+ CacheEventListener1 lsnr = new CacheEventListener1(true);
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ QueryCursor<?> cur = cache.query(qry);
+
+ client = false;
+
+ final int SRV_IDX = SRV_NODES - 1;
+
+ List<Integer> keys = primaryKeys(ignite(SRV_IDX).cache(null), 10);
+
+ final int THREADS = 10;
+
+ for (int i = 0; i < keys.size(); i++) {
+ log.info("Iteration: " + i);
+
+ Ignite srv = ignite(SRV_IDX);
+
+ TestCommunicationSpi spi = (TestCommunicationSpi)srv.configuration().getCommunicationSpi();
+
+ spi.sndFirstOnly = new AtomicBoolean(false);
+
+ final Integer key = keys.get(i);
+
+ final AtomicInteger val = new AtomicInteger();
+
+ CountDownLatch latch = new CountDownLatch(THREADS);
+
+ lsnr.latch = latch;
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ Integer val0 = val.getAndIncrement();
+
+ cache.put(key, val0);
+
+ return null;
+ }
+ }, THREADS, "update-thread");
+
+ fut.get();
+
+ stopGrid(SRV_IDX);
+
+ if (!latch.await(5, SECONDS))
+ fail("Failed to wait for notifications [exp=" + THREADS + ", left=" + lsnr.latch.getCount() + ']');
+
+ assertEquals(THREADS, lsnr.allEvts.size());
+
+ Set<Integer> vals = new HashSet<>();
+
+ boolean err = false;
+
+ for (CacheEntryEvent<?, ?> evt : lsnr.allEvts) {
+ assertEquals(key, evt.getKey());
+ assertNotNull(evt.getValue());
+
+ if (!vals.add((Integer)evt.getValue())) {
+ err = true;
+
+ log.info("Extra event: " + evt);
+ }
+ }
+
+ for (int v = 0; v < THREADS; v++) {
+ if (!vals.contains(v)) {
+ err = true;
+
+ log.info("Event for value not received: " + v);
+ }
+ }
+
+ assertFalse("Invalid events, see log for details.", err);
+
+ lsnr.allEvts.clear();
+
+ startGrid(SRV_IDX);
+ }
+
+ cur.close();
+ }
+
+ /**
+ * @param logAll If {@code true} logs all unexpected values.
+ * @param expEvts Expected values.
+ * @param lsnr Listener.
+ * @return Check status.
+ */
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+ private boolean checkEvents(boolean logAll,
+ Map<Integer, List<T2<Integer, Integer>>> expEvts,
+ CacheEventListener2 lsnr) {
+ assertTrue(!expEvts.isEmpty());
+
+ boolean pass = true;
+
+ for (Map.Entry<Integer, List<T2<Integer, Integer>>> e : expEvts.entrySet()) {
+ Integer key = e.getKey();
+ List<T2<Integer, Integer>> exp = e.getValue();
+
+ List<CacheEntryEvent<?, ?>> rcvdEvts = lsnr.evts.get(key);
+
+ if (rcvdEvts == null) {
+ pass = false;
+
+ log.info("No events for key [key=" + key + ", exp=" + e.getValue() + ']');
+
+ if (!logAll)
+ return false;
+ }
+ else {
+ synchronized (rcvdEvts) {
+ if (rcvdEvts.size() != exp.size()) {
+ pass = false;
+
+ log.info("Missed or extra events for key [key=" + key +
+ ", exp=" + e.getValue() +
+ ", rcvd=" + rcvdEvts + ']');
+
+ if (!logAll)
+ return false;
+ }
+
+ int cnt = Math.min(rcvdEvts.size(), exp.size());
+
+ for (int i = 0; i < cnt; i++) {
+ T2<Integer, Integer> expEvt = exp.get(i);
+ CacheEntryEvent<?, ?> rcvdEvt = rcvdEvts.get(i);
+
+ assertEquals(key, rcvdEvt.getKey());
+ assertEquals(expEvt.get1(), rcvdEvt.getValue());
+ assertEquals(expEvt.get2(), rcvdEvt.getOldValue());
+ }
+ }
+ }
+ }
+
+ if (pass) {
+ expEvts.clear();
+ lsnr.evts.clear();
+ }
+
+ return pass;
+ }
+
+ /**
+ *
+ */
+ private static class CacheEventListener1 implements CacheEntryUpdatedListener<Object, Object> {
+ /** */
+ private volatile CountDownLatch latch;
+
+ /** */
+ private GridConcurrentHashSet<Integer> keys = new GridConcurrentHashSet<>();
+
+ /** */
+ private ConcurrentHashMap<Object, CacheEntryEvent<?, ?>> evts = new ConcurrentHashMap<>();
+
+ /** */
+ private List<CacheEntryEvent<?, ?>> allEvts;
+
+ /** */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /**
+ * @param saveAll Save all events flag.
+ */
+ CacheEventListener1(boolean saveAll) {
+ if (saveAll)
+ allEvts = new ArrayList<>();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts) {
+ try {
+ for (CacheEntryEvent<?, ?> evt : evts) {
+ CountDownLatch latch = this.latch;
+
+ log.info("Received cache event [evt=" + evt +
+ ", left=" + (latch != null ? latch.getCount() : null) + ']');
+
+ this.evts.put(evt.getKey(), evt);
+
+ keys.add((Integer) evt.getKey());
+
+ if (allEvts != null)
+ allEvts.add(evt);
+
+ assertTrue(latch != null);
+ assertTrue(latch.getCount() > 0);
+
+ latch.countDown();
+
+ if (latch.getCount() == 0) {
+ this.latch = null;
+
+ keys.clear();
+ }
+ }
+ }
+ catch (Throwable e) {
+ err = true;
+
+ log.error("Unexpected error", e);
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static class CacheEventListener2 implements CacheEntryUpdatedListener<Object, Object> {
+ /** */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** */
+ private final ConcurrentHashMap<Integer, Integer> vals = new ConcurrentHashMap<>();
+
+ /** */
+ private final ConcurrentHashMap<Integer, List<CacheEntryEvent<?, ?>>> evts = new ConcurrentHashMap<>();
+
+ /** {@inheritDoc} */
+ @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts)
+ throws CacheEntryListenerException {
+ try {
+ for (CacheEntryEvent<?, ?> evt : evts) {
+ Integer key = (Integer)evt.getKey();
+ Integer val = (Integer)evt.getValue();
+
+ assertNotNull(key);
+ assertNotNull(val);
+
+ Integer prevVal = vals.get(key);
+
+ boolean dup = false;
+
+ if (prevVal != null) {
+ if (prevVal.equals(val)) // Can get this event with automatic put retry.
+ dup = true;
+ else {
+ assertEquals("Unexpected event: " + evt, (Integer)(prevVal + 1), val);
+ assertEquals("Unexpected event: " + evt, prevVal, evt.getOldValue());
+ }
+ }
+ else {
+ assertEquals("Unexpected event: " + evt, (Object)0, val);
+ assertNull("Unexpected event: " + evt, evt.getOldValue());
+ }
+
+ if (!dup) {
+ vals.put(key, val);
+
+ List<CacheEntryEvent<?, ?>> keyEvts = this.evts.get(key);
+
+ if (keyEvts == null) {
+ keyEvts = Collections.synchronizedList(new ArrayList<CacheEntryEvent<?, ?>>());
+
+ this.evts.put(key, keyEvts);
+ }
+
+ keyEvts.add(evt);
+ }
+ }
+ }
+ catch (Throwable e) {
+ err = true;
+
+ log.error("Unexpected error", e);
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestCommunicationSpi extends TcpCommunicationSpi {
+ /** */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** */
+ private volatile boolean skipMsg;
+
+ /** */
+ private volatile AtomicBoolean sndFirstOnly;
+
+ /** {@inheritDoc} */
+ @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC)
+ throws IgniteSpiException {
+ Object msg0 = ((GridIoMessage)msg).message();
+
+ if (msg0 instanceof GridContinuousMessage) {
+ if (skipMsg) {
+ log.info("Skip continuous message: " + msg0);
+
+ return;
+ }
+ else {
+ AtomicBoolean sndFirstOnly = this.sndFirstOnly;
+
+ if (sndFirstOnly != null && !sndFirstOnly.compareAndSet(false, true)) {
+ log.info("Skip continuous message: " + msg0);
+
+ return;
+ }
+ }
+ }
+
+ super.sendMessage(node, msg, ackC);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest.java
new file mode 100644
index 0000000..4ddcf0d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import org.apache.ignite.cache.CacheAtomicWriteOrderMode;
+
+import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.PRIMARY;
+
+/**
+ *
+ */
+public class CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest extends CacheContinuousQueryFailoverAtomicTest {
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicWriteOrderMode writeOrderMode() {
+ return PRIMARY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicReplicatedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicReplicatedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicReplicatedTest.java
new file mode 100644
index 0000000..8fc58d3
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicReplicatedTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import org.apache.ignite.cache.CacheAtomicWriteOrderMode;
+import org.apache.ignite.cache.CacheMode;
+
+import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.PRIMARY;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+
+/**
+ *
+ */
+public class CacheContinuousQueryFailoverAtomicReplicatedTest extends CacheContinuousQueryFailoverAtomicTest {
+ /** {@inheritDoc} */
+ @Override protected CacheMode cacheMode() {
+ return REPLICATED;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicWriteOrderMode writeOrderMode() {
+ return PRIMARY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicTest.java
new file mode 100644
index 0000000..fb50387
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ *
+ */
+public class CacheContinuousQueryFailoverAtomicTest extends CacheContinuousQueryFailoverAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected CacheMode cacheMode() {
+ return PARTITIONED;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicityMode atomicityMode() {
+ return ATOMIC;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxReplicatedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxReplicatedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxReplicatedTest.java
new file mode 100644
index 0000000..746f0eb
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxReplicatedTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import org.apache.ignite.cache.CacheMode;
+
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+
+/**
+ *
+ */
+public class CacheContinuousQueryFailoverTxReplicatedTest extends CacheContinuousQueryFailoverTxTest {
+ /** {@inheritDoc} */
+ @Override protected CacheMode cacheMode() {
+ return REPLICATED;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxTest.java
new file mode 100644
index 0000000..8e3a575
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverTxTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ *
+ */
+public class CacheContinuousQueryFailoverTxTest extends CacheContinuousQueryFailoverAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected CacheMode cacheMode() {
+ return PARTITIONED;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicityMode atomicityMode() {
+ return TRANSACTIONAL;
+ }
+}
[2/9] ignite git commit: IGNITE-426 temp commit.
Posted by nt...@apache.org.
IGNITE-426 temp commit.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1f6ba6c9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1f6ba6c9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1f6ba6c9
Branch: refs/heads/ignite-426-2-reb
Commit: 1f6ba6c937d4019a028d9fda0df67c9f14bd600e
Parents: c28af3c
Author: nikolay_tikhonov <nt...@gridgain.com>
Authored: Wed Sep 2 15:38:50 2015 +0300
Committer: nikolay_tikhonov <nt...@gridgain.com>
Committed: Wed Oct 21 16:54:21 2015 +0300
----------------------------------------------------------------------
.../dht/GridClientPartitionTopology.java | 40 +-
.../distributed/dht/GridDhtLocalPartition.java | 61 +-
.../dht/GridDhtPartitionTopology.java | 26 +-
.../dht/GridDhtPartitionTopologyImpl.java | 112 +-
.../CacheContinuousQueryBatchAck.java | 156 +++
.../continuous/CacheContinuousQueryEntry.java | 74 +-
.../continuous/CacheContinuousQueryHandler.java | 345 +++++-
.../CacheContinuousQueryListener.java | 34 +-
.../continuous/CacheContinuousQueryManager.java | 88 +-
.../continuous/GridContinuousBatch.java | 7 +
.../continuous/GridContinuousBatchAdapter.java | 7 +
.../continuous/GridContinuousProcessor.java | 173 ++-
...acheContinuousQueryFailoverAbstractTest.java | 1104 ++++++++++++++++++
...ueryFailoverAtomicPrimaryWriteOrderTest.java | 32 +
...inuousQueryFailoverAtomicReplicatedTest.java | 39 +
.../CacheContinuousQueryFailoverAtomicTest.java | 39 +
...ContinuousQueryFailoverTxReplicatedTest.java | 32 +
.../CacheContinuousQueryFailoverTxTest.java | 39 +
18 files changed, 2311 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 162c116..516b7bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -94,6 +94,9 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
/** Lock. */
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ /** Partition update counter. */
+ private Map<Integer, Long> cntrMap = new HashMap<>();
+
/**
* @param cctx Context.
* @param cacheId Cache ID.
@@ -527,7 +530,8 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
/** {@inheritDoc} */
@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
@Nullable @Override public GridDhtPartitionMap update(@Nullable GridDhtPartitionExchangeId exchId,
- GridDhtPartitionFullMap partMap) {
+ GridDhtPartitionFullMap partMap,
+ Map<Integer, Long> cntrMap) {
if (log.isDebugEnabled())
log.debug("Updating full partition map [exchId=" + exchId + ", parts=" + fullMapString() + ']');
@@ -602,6 +606,9 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
part2node = p2n;
+ if (cntrMap != null)
+ this.cntrMap = new HashMap<>(cntrMap);
+
consistencyCheck();
if (log.isDebugEnabled())
@@ -617,7 +624,8 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
/** {@inheritDoc} */
@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
@Nullable @Override public GridDhtPartitionMap update(@Nullable GridDhtPartitionExchangeId exchId,
- GridDhtPartitionMap parts) {
+ GridDhtPartitionMap parts,
+ Map<Integer, Long> cntrMap) {
if (log.isDebugEnabled())
log.debug("Updating single partition map [exchId=" + exchId + ", parts=" + mapString(parts) + ']');
@@ -698,6 +706,15 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
}
}
+ if (cntrMap != null) {
+ for (Map.Entry<Integer, Long> e : cntrMap.entrySet()) {
+ Long cntr = this.cntrMap.get(e.getKey());
+
+ if (cntr == null || cntr < e.getValue())
+ this.cntrMap.put(e.getKey(), e.getValue());
+ }
+ }
+
consistencyCheck();
if (log.isDebugEnabled())
@@ -852,6 +869,25 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
+ @Override public Map<Integer, Long> updateCounters() {
+ lock.readLock().lock();
+
+ try {
+ return new HashMap<>(cntrMap);
+ }
+ finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean rebalanceFinished(AffinityTopologyVersion topVer) {
+ assert false;
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
@Override public void printMemoryStats(int threshold) {
X.println(">>> Cache partition topology stats [grid=" + cctx.gridName() + ", cacheId=" + cacheId + ']');
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 4f124e6..975d76c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -17,6 +17,18 @@
package org.apache.ignite.internal.processors.cache.distributed.dht;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicStampedReference;
+import java.util.concurrent.locks.ReentrantLock;
+import javax.cache.CacheException;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.IgniteInternalFuture;
@@ -111,11 +123,14 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
private final LongAdder8 mapPubSize = new LongAdder8();
/** Remove queue. */
- private final GridCircularBuffer<T2<KeyCacheObject, GridCacheVersion>> rmvQueue;
+ private GridCircularBuffer<T2<KeyCacheObject, GridCacheVersion>> rmvQueue;
/** Group reservations. */
private final CopyOnWriteArrayList<GridDhtPartitionsReservation> reservations = new CopyOnWriteArrayList<>();
+ /** Continuous query update index. */
+ private final AtomicLong contQryUpdIdx = new AtomicLong();
+
/**
* @param cctx Context.
* @param id Partition ID.
@@ -141,7 +156,8 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
int delQueueSize = CU.isSystemCache(cctx.name()) ? 100 :
Math.max(MAX_DELETE_QUEUE_SIZE / cctx.affinity().partitions(), 20);
- rmvQueue = new GridCircularBuffer<>(U.ceilPow2(delQueueSize));
+ if (cctx.deferredDelete())
+ rmvQueue = new GridCircularBuffer<>(U.ceilPow2(delQueueSize));
}
/**
@@ -295,6 +311,8 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
* @throws IgniteCheckedException If failed.
*/
public void onDeferredDelete(KeyCacheObject key, GridCacheVersion ver) throws IgniteCheckedException {
+ assert cctx.deferredDelete();
+
try {
T2<KeyCacheObject, GridCacheVersion> evicted = rmvQueue.add(new T2<>(key, ver));
@@ -496,7 +514,8 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
((GridDhtPreloader)cctx.preloader()).onPartitionEvicted(this, updateSeq);
- clearDeferredDeletes();
+ if (cctx.deferredDelete())
+ clearDeferredDeletes();
return new GridFinishedFuture<>(true);
}
@@ -541,13 +560,16 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
if (cctx.isDrEnabled())
cctx.dr().partitionEvicted(id);
+ cctx.continuousQueries().onPartitionEvicted(id);
+
cctx.dataStructures().onPartitionEvicted(id);
rent.onDone();
((GridDhtPreloader)cctx.preloader()).onPartitionEvicted(this, updateSeq);
- clearDeferredDeletes();
+ if (cctx.deferredDelete())
+ clearDeferredDeletes();
return true;
}
@@ -604,6 +626,35 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
}
/**
+ * @return Next update index.
+ */
+ public long nextContinuousQueryUpdateIndex() {
+ return contQryUpdIdx.incrementAndGet();
+ }
+
+ /**
+ * @return Current update index.
+ */
+ public long continuousQueryUpdateIndex() {
+ return contQryUpdIdx.get();
+ }
+
+ /**
+ * @param val Update index value.
+ */
+ public void continuousQueryUpdateIndex(long val) {
+ while (true) {
+ long val0 = contQryUpdIdx.get();
+
+ if (val0 >= val)
+ break;
+
+ if (contQryUpdIdx.compareAndSet(val0, val))
+ break;
+ }
+ }
+
+ /**
* Clears values for this partition.
*/
private void clearAll() {
@@ -753,6 +804,8 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
*
*/
private void clearDeferredDeletes() {
+ assert cctx.deferredDelete();
+
rmvQueue.forEach(new CI1<T2<KeyCacheObject, GridCacheVersion>>() {
@Override public void apply(T2<KeyCacheObject, GridCacheVersion> t) {
cctx.dht().removeVersionedEntry(t.get1(), t.get2());
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index d642314..3ac2b85 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.distributed.dht;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cluster.ClusterNode;
@@ -51,6 +52,8 @@ public interface GridDhtPartitionTopology {
*
* @param exchId Exchange ID.
* @param exchFut Exchange future.
+ * @param updateSeq Update sequence.
+ * @param stopping Stopping flag.
* @throws IgniteInterruptedCheckedException If interrupted.
*/
public void updateTopologyVersion(
@@ -193,17 +196,27 @@ public interface GridDhtPartitionTopology {
/**
* @param exchId Exchange ID.
* @param partMap Update partition map.
+ * @param cntrMap Partition update counters.
* @return Local partition map if there were evictions or {@code null} otherwise.
*/
- public GridDhtPartitionMap update(@Nullable GridDhtPartitionExchangeId exchId, GridDhtPartitionFullMap partMap);
+ public GridDhtPartitionMap update(@Nullable GridDhtPartitionExchangeId exchId,
+ GridDhtPartitionFullMap partMap,
+ @Nullable Map<Integer, Long> cntrMap);
/**
* @param exchId Exchange ID.
* @param parts Partitions.
+ * @param cntrMap Partition update counters.
* @return Local partition map if there were evictions or {@code null} otherwise.
*/
@Nullable public GridDhtPartitionMap update(@Nullable GridDhtPartitionExchangeId exchId,
- GridDhtPartitionMap parts);
+ GridDhtPartitionMap parts,
+ @Nullable Map<Integer, Long> cntrMap);
+
+ /**
+ * @return Partition update counters.
+ */
+ public Map<Integer, Long> updateCounters();
/**
* @param part Partition to own.
@@ -213,6 +226,7 @@ public interface GridDhtPartitionTopology {
/**
* @param part Evicted partition.
+ * @param updateSeq Update sequence increment flag.
*/
public void onEvicted(GridDhtLocalPartition part, boolean updateSeq);
@@ -228,4 +242,10 @@ public interface GridDhtPartitionTopology {
* @param threshold Threshold for number of entries.
*/
public void printMemoryStats(int threshold);
-}
\ No newline at end of file
+
+ /**
+ * @param topVer Topology version.
+ * @return {@code True} if rebalance process finished.
+ */
+ public boolean rebalanceFinished(AffinityTopologyVersion topVer);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 6bd283a..5d312b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -102,6 +102,12 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
/** Lock. */
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ /** Partition update counter. */
+ private Map<Integer, Long> cntrMap = new HashMap<>();
+
+ /** */
+ private volatile AffinityTopologyVersion rebalancedTopVer = AffinityTopologyVersion.NONE;
+
/**
* @param cctx Context.
*/
@@ -131,6 +137,8 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
topReadyFut = null;
topVer = AffinityTopologyVersion.NONE;
+
+ rebalancedTopVer = AffinityTopologyVersion.NONE;
}
finally {
lock.writeLock().unlock();
@@ -220,6 +228,8 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
updateSeq.setIfGreater(updSeq);
topReadyFut = exchFut;
+
+ rebalancedTopVer = AffinityTopologyVersion.NONE;;
}
finally {
lock.writeLock().unlock();
@@ -525,6 +535,8 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
}
+ updateRebalanceVersion();
+
consistencyCheck();
}
finally {
@@ -732,7 +744,10 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
* @param states Additional partition states.
* @return List of nodes for the partition.
*/
- private List<ClusterNode> nodes(int p, AffinityTopologyVersion topVer, GridDhtPartitionState state, GridDhtPartitionState... states) {
+ private List<ClusterNode> nodes(int p,
+ AffinityTopologyVersion topVer,
+ GridDhtPartitionState state,
+ GridDhtPartitionState... states) {
Collection<UUID> allIds = topVer.topologyVersion() > 0 ? F.nodeIds(CU.affinityNodes(cctx, topVer)) : null;
lock.readLock().lock();
@@ -831,7 +846,8 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
/** {@inheritDoc} */
@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
@Nullable @Override public GridDhtPartitionMap update(@Nullable GridDhtPartitionExchangeId exchId,
- GridDhtPartitionFullMap partMap) {
+ GridDhtPartitionFullMap partMap,
+ @Nullable Map<Integer, Long> cntrMap) {
if (log.isDebugEnabled())
log.debug("Updating full partition map [exchId=" + exchId + ", parts=" + fullMapString() + ']');
@@ -911,8 +927,26 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
part2node = p2n;
+ if (cntrMap != null) {
+ for (Map.Entry<Integer, Long> e : cntrMap.entrySet()) {
+ Long cntr = this.cntrMap.get(e.getKey());
+
+ if (cntr == null || cntr < e.getValue())
+ this.cntrMap.put(e.getKey(), e.getValue());
+ }
+
+ for (GridDhtLocalPartition part : locParts.values()) {
+ Long cntr = cntrMap.get(part.id());
+
+ if (cntr != null)
+ part.continuousQueryUpdateIndex(cntr);
+ }
+ }
+
boolean changed = checkEvictions(updateSeq);
+ updateRebalanceVersion();
+
consistencyCheck();
if (log.isDebugEnabled())
@@ -928,7 +962,8 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
/** {@inheritDoc} */
@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
@Nullable @Override public GridDhtPartitionMap update(@Nullable GridDhtPartitionExchangeId exchId,
- GridDhtPartitionMap parts) {
+ GridDhtPartitionMap parts,
+ @Nullable Map<Integer, Long> cntrMap) {
if (log.isDebugEnabled())
log.debug("Updating single partition map [exchId=" + exchId + ", parts=" + mapString(parts) + ']');
@@ -1006,8 +1041,26 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
}
+ if (cntrMap != null) {
+ for (Map.Entry<Integer, Long> e : cntrMap.entrySet()) {
+ Long cntr = this.cntrMap.get(e.getKey());
+
+ if (cntr == null || cntr < e.getValue())
+ this.cntrMap.put(e.getKey(), e.getValue());
+ }
+
+ for (GridDhtLocalPartition part : locParts.values()) {
+ Long cntr = cntrMap.get(part.id());
+
+ if (cntr != null)
+ part.continuousQueryUpdateIndex(cntr);
+ }
+ }
+
changed |= checkEvictions(updateSeq);
+ updateRebalanceVersion();
+
consistencyCheck();
if (log.isDebugEnabled())
@@ -1204,6 +1257,8 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
if (part.own()) {
updateLocal(part.id(), loc.id(), part.state(), updateSeq.incrementAndGet());
+ updateRebalanceVersion();
+
consistencyCheck();
return true;
@@ -1254,14 +1309,61 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
}
/** {@inheritDoc} */
+ @Override public Map<Integer, Long> updateCounters() {
+ lock.readLock().lock();
+
+ try {
+ Map<Integer, Long> res = new HashMap<>(cntrMap);
+
+ for (GridDhtLocalPartition part : locParts.values()) {
+ Long cntr0 = res.get(part.id());
+ Long cntr1 = part.continuousQueryUpdateIndex();
+
+ if (cntr0 == null || cntr1 > cntr0)
+ res.put(part.id(), cntr1);
+ }
+
+ return res;
+ }
+ finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean rebalanceFinished(AffinityTopologyVersion topVer) {
+ return topVer.equals(rebalancedTopVer);
+ }
+
+ /** {@inheritDoc} */
@Override public void printMemoryStats(int threshold) {
- X.println(">>> Cache partition topology stats [grid=" + cctx.gridName() + ", cache=" + cctx.name() + ']');
+ X.println(">>> Cache partition topology stats [grid=" + cctx.gridName() + ", cache=" + cctx.name() + ']');
for (GridDhtLocalPartition part : locParts.values()) {
int size = part.size();
if (size >= threshold)
- X.println(">>> Local partition [part=" + part.id() + ", size=" + size + ']');
+ X.println(">>> Local partition [part=" + part.id() + ", size=" + size + ']');
+ }
+ }
+
+ /**
+ *
+ */
+ private void updateRebalanceVersion() {
+ if (!rebalancedTopVer.equals(topVer)) {
+ for (int i = 0; i < cctx.affinity().partitions(); i++) {
+ List<ClusterNode> affNodes = cctx.affinity().nodes(i, topVer);
+ List<ClusterNode> owners = owners(i);
+
+ if (affNodes.size() != owners.size() || !owners.containsAll(affNodes))
+ return;
+ }
+
+ rebalancedTopVer = topVer;
+
+ if (log.isDebugEnabled())
+ log.debug("Updated rebalanced version [cache=" + cctx.name() + ", ver=" + rebalancedTopVer + ']');
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java
new file mode 100644
index 0000000..1e9a848
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.GridDirectMap;
+import org.apache.ignite.internal.processors.cache.GridCacheMessage;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ * Batch acknowledgement.
+ */
+public class CacheContinuousQueryBatchAck extends GridCacheMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Routine ID. */
+ private UUID routineId;
+
+ /** Update indexes. */
+ @GridToStringInclude
+ @GridDirectMap(keyType = Integer.class, valueType = Long.class)
+ private Map<Integer, Long> updateIdxs;
+
+ /**
+ * Default constructor.
+ */
+ public CacheContinuousQueryBatchAck() {
+ // No-op.
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param routineId Routine ID.
+ * @param updateIdxs Update indexes.
+ */
+ CacheContinuousQueryBatchAck(int cacheId, UUID routineId, Map<Integer, Long> updateIdxs) {
+ this.cacheId = cacheId;
+ this.routineId = routineId;
+ this.updateIdxs = updateIdxs;
+ }
+
+ /**
+ * @return Routine ID.
+ */
+ UUID routineId() {
+ return routineId;
+ }
+
+ /**
+ * @return Update indexes.
+ */
+ Map<Integer, Long> updateIndexes() {
+ return updateIdxs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!super.writeTo(buf, writer))
+ return false;
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 3:
+ if (!writer.writeUuid("routineId", routineId))
+ return false;
+
+ writer.incrementState();
+
+ case 4:
+ if (!writer.writeMap("updateIdxs", updateIdxs, MessageCollectionItemType.INT, MessageCollectionItemType.LONG))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ if (!super.readFrom(buf, reader))
+ return false;
+
+ switch (reader.state()) {
+ case 3:
+ routineId = reader.readUuid("routineId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 4:
+ updateIdxs = reader.readMap("updateIdxs", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false);
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte directType() {
+ return 114;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 5;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(CacheContinuousQueryBatchAck.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index a4b35eb..9ea9b73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -22,6 +22,7 @@ import javax.cache.event.EventType;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.GridDirectTransient;
import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheDeployable;
@@ -75,6 +76,17 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
@GridDirectTransient
private GridDeploymentInfo depInfo;
+ /** Partition. */
+ private int part;
+
+ /** Update index. */
+ private long updateIdx;
+
+ /** */
+ @GridToStringInclude
+ @GridDirectTransient
+ private AffinityTopologyVersion topVer;
+
/**
* Required by {@link org.apache.ignite.plugin.extensions.communication.Message}.
*/
@@ -88,18 +100,34 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
* @param key Key.
* @param newVal New value.
* @param oldVal Old value.
+ * @param part Partition.
+ * @param updateIdx Update index.
+ * @param topVer Topology version if applicable.
*/
CacheContinuousQueryEntry(
int cacheId,
EventType evtType,
KeyCacheObject key,
@Nullable CacheObject newVal,
- @Nullable CacheObject oldVal) {
+ @Nullable CacheObject oldVal,
+ int part,
+ long updateIdx,
+ @Nullable AffinityTopologyVersion topVer) {
this.cacheId = cacheId;
this.evtType = evtType;
this.key = key;
this.newVal = newVal;
this.oldVal = oldVal;
+ this.part = part;
+ this.updateIdx = updateIdx;
+ this.topVer = topVer;
+ }
+
+ /**
+ * @return Topology version if applicable.
+ */
+ @Nullable AffinityTopologyVersion topologyVersion() {
+ return topVer;
}
/**
@@ -117,6 +145,20 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
}
/**
+ * @return Partition.
+ */
+ int partition() {
+ return part;
+ }
+
+ /**
+ * @return Update index.
+ */
+ long updateIndex() {
+ return updateIdx;
+ }
+
+ /**
* @param cctx Cache context.
* @throws IgniteCheckedException In case of error.
*/
@@ -225,6 +267,18 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
writer.incrementState();
+ case 5:
+ if (!writer.writeInt("part", part))
+ return false;
+
+ writer.incrementState();
+
+ case 6:
+ if (!writer.writeLong("updateIdx", updateIdx))
+ return false;
+
+ writer.incrementState();
+
}
return true;
@@ -282,6 +336,22 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
reader.incrementState();
+ case 5:
+ part = reader.readInt("part");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 6:
+ updateIdx = reader.readLong("updateIdx");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
}
return reader.afterMessageRead(CacheContinuousQueryEntry.class);
@@ -289,7 +359,7 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 5;
+ return 7;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 1990e18..3253dda 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -22,7 +22,12 @@ import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import javax.cache.event.CacheEntryEvent;
import javax.cache.event.CacheEntryUpdatedListener;
import javax.cache.event.EventType;
@@ -35,13 +40,17 @@ import org.apache.ignite.events.CacheQueryReadEvent;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteDeploymentCheckedException;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.managers.deployment.GridDeployment;
import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
import org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheDeploymentManager;
import org.apache.ignite.internal.processors.cache.query.CacheQueryType;
+import org.apache.ignite.internal.processors.continuous.GridContinuousBatch;
+import org.apache.ignite.internal.processors.continuous.GridContinuousBatchAdapter;
import org.apache.ignite.internal.processors.continuous.GridContinuousHandler;
import org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQueryFilter;
import org.apache.ignite.internal.util.typedef.C1;
@@ -49,7 +58,10 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgnitePredicate;
import org.jetbrains.annotations.Nullable;
+import org.jsr166.ConcurrentLinkedDeque8;
import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED;
import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_OBJECT_READ;
@@ -61,6 +73,9 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
/** */
private static final long serialVersionUID = 0L;
+ /** */
+ private static final int BACKUP_ACK_THRESHOLD = 100;
+
/** Cache name. */
private String cacheName;
@@ -97,6 +112,18 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
/** Whether to skip primary check for REPLICATED cache. */
private transient boolean skipPrimaryCheck;
+ /** Backup queue. */
+ private transient Collection<CacheContinuousQueryEntry> backupQueue;
+
+ /** */
+ private transient Map<Integer, Long> rcvCntrs;
+
+ /** */
+ private transient IgnitePredicate<CacheContinuousQueryEntry> dupEvtFilter;
+
+ /** */
+ private transient AcknowledgeBuffer ackBuf;
+
/** */
private transient int cacheId;
@@ -121,6 +148,7 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
* @param ignoreExpired Ignore expired events flag.
* @param skipPrimaryCheck Whether to skip primary check for REPLICATED cache.
* @param taskHash Task name hash code.
+ * @param locCache {@code True} if local cache.
*/
public CacheContinuousQueryHandler(
String cacheName,
@@ -133,7 +161,8 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
boolean sync,
boolean ignoreExpired,
int taskHash,
- boolean skipPrimaryCheck) {
+ boolean skipPrimaryCheck,
+ boolean locCache) {
assert topic != null;
assert locLsnr != null;
@@ -149,6 +178,14 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
this.taskHash = taskHash;
this.skipPrimaryCheck = skipPrimaryCheck;
+ if (locCache)
+ dupEvtFilter = F.alwaysTrue();
+ else {
+ rcvCntrs = new ConcurrentHashMap<>();
+
+ dupEvtFilter = new DuplicateEventFilter();
+ }
+
cacheId = CU.cacheId(cacheName);
}
@@ -185,8 +222,14 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
if (rmtFilter != null)
ctx.resource().injectGeneric(rmtFilter);
+ backupQueue = new ConcurrentLinkedDeque8<>();
+
+ ackBuf = new AcknowledgeBuffer();
+
final boolean loc = nodeId.equals(ctx.localNodeId());
+ assert !skipPrimaryCheck || loc;
+
CacheContinuousQueryListener<K, V> lsnr = new CacheContinuousQueryListener<K, V>() {
@Override public void onExecution() {
if (ctx.event().isRecordable(EVT_CACHE_QUERY_EXECUTED)) {
@@ -207,15 +250,16 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
}
- @Override public void onEntryUpdated(CacheContinuousQueryEvent<K, V> evt, boolean primary,
+ @Override public void onEntryUpdated(CacheContinuousQueryEvent<K, V> evt,
+ boolean primary,
boolean recordIgniteEvt) {
if (ignoreExpired && evt.getEventType() == EventType.EXPIRED)
return;
GridCacheContext<K, V> cctx = cacheContext(ctx);
- if (cctx.isReplicated() && !skipPrimaryCheck && !primary)
- return;
+ // skipPrimaryCheck is set only when listen locally for replicated cache events.
+ assert !skipPrimaryCheck || (cctx.isReplicated() && ctx.localNodeId().equals(nodeId));
boolean notify = true;
@@ -229,32 +273,36 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
if (notify) {
- if (loc)
- locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
- else {
- try {
- if (ctx.config().isPeerClassLoadingEnabled() && ctx.discovery().node(nodeId) != null) {
- evt.entry().prepareMarshal(cctx);
+ try {
+ final CacheContinuousQueryEntry entry = evt.entry();
- GridCacheDeploymentManager depMgr = cctx.deploy();
+ if (primary || skipPrimaryCheck) {
+ if (loc) {
+ if (dupEvtFilter.apply(entry)) {
+ locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
- depMgr.prepare(evt.entry());
+ if (!skipPrimaryCheck)
+ sendBackupAcknowledge(ackBuf.onAcknowledged(entry), routineId, ctx);
+ }
}
- else
- evt.entry().prepareMarshal(cctx);
+ else {
+ prepareEntry(cctx, nodeId, entry);
- ctx.continuous().addNotification(nodeId, routineId, evt.entry(), topic, sync, true);
+ ctx.continuous().addNotification(nodeId, routineId, entry, topic, sync, true);
+ }
}
- catch (ClusterTopologyCheckedException ex) {
- IgniteLogger log = ctx.log(getClass());
+ else
+ backupQueue.add(entry);
+ }
+ catch (ClusterTopologyCheckedException ex) {
+ IgniteLogger log = ctx.log(getClass());
- if (log.isDebugEnabled())
- log.debug("Failed to send event notification to node, node left cluster " +
- "[node=" + nodeId + ", err=" + ex + ']');
- }
- catch (IgniteCheckedException ex) {
- U.error(ctx.log(getClass()), "Failed to send event notification to node: " + nodeId, ex);
- }
+ if (log.isDebugEnabled())
+ log.debug("Failed to send event notification to node, node left cluster " +
+ "[node=" + nodeId + ", err=" + ex + ']');
+ }
+ catch (IgniteCheckedException ex) {
+ U.error(ctx.log(getClass()), "Failed to send event notification to node: " + nodeId, ex);
}
if (recordIgniteEvt) {
@@ -285,6 +333,49 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
((PlatformContinuousQueryFilter)rmtFilter).onQueryUnregister();
}
+ @Override public void cleanupBackupQueue(Map<Integer, Long> updateIdxs) {
+ Iterator<CacheContinuousQueryEntry> it = backupQueue.iterator();
+
+ while (it.hasNext()) {
+ CacheContinuousQueryEntry backupEntry = it.next();
+
+ Long updateIdx = updateIdxs.get(backupEntry.partition());
+
+ if (updateIdx != null && backupEntry.updateIndex() <= updateIdx)
+ it.remove();
+ }
+ }
+
+ @Override public void flushBackupQueue(GridKernalContext ctx, AffinityTopologyVersion topVer) {
+ if (backupQueue.isEmpty())
+ return;
+
+ try {
+ GridCacheContext<K, V> cctx = cacheContext(ctx);
+
+ for (CacheContinuousQueryEntry e : backupQueue)
+ prepareEntry(cctx, nodeId, e);
+
+ ctx.continuous().addBackupNotification(nodeId, routineId, backupQueue, topic);
+
+ backupQueue.clear();
+ }
+ catch (IgniteCheckedException e) {
+ U.error(ctx.log(getClass()), "Failed to send backup event notification to node: " + nodeId, e);
+ }
+ }
+
+ @Override public void acknowledgeBackupOnTimeout(GridKernalContext ctx) {
+ sendBackupAcknowledge(ackBuf.acknowledgeOnTimeout(), routineId, ctx);
+ }
+
+ @Override public void onPartitionEvicted(int part) {
+ for (Iterator<CacheContinuousQueryEntry> it = backupQueue.iterator(); it.hasNext();) {
+ if (it.next().partition() == part)
+ it.remove();
+ }
+ }
+
@Override public boolean oldValueRequired() {
return oldValRequired;
}
@@ -306,6 +397,23 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
return mgr.registerListener(routineId, lsnr, internal);
}
+ /**
+ * @param cctx Context.
+ * @param nodeId ID of the node that started routine.
+ * @param entry Entry.
+ * @throws IgniteCheckedException In case of error.
+ */
+ private void prepareEntry(GridCacheContext cctx, UUID nodeId, CacheContinuousQueryEntry entry)
+ throws IgniteCheckedException {
+ if (cctx.kernalContext().config().isPeerClassLoadingEnabled() && cctx.discovery().node(nodeId) != null) {
+ entry.prepareMarshal(cctx);
+
+ cctx.deploy().prepare(entry);
+ }
+ else
+ entry.prepareMarshal(cctx);
+ }
+
/** {@inheritDoc} */
@Override public void onListenerRegistered(UUID routineId, GridKernalContext ctx) {
// No-op.
@@ -373,12 +481,40 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
@Override public CacheEntryEvent<? extends K, ? extends V> apply(CacheContinuousQueryEntry e) {
return new CacheContinuousQueryEvent<>(cache, cctx, e);
}
- }
+ },
+ dupEvtFilter
);
locLsnr.onUpdated(evts);
}
+ /**
+ * @param e Entry.
+ * @return {@code True} if listener should be notified.
+ */
+ private boolean notifyListener(CacheContinuousQueryEntry e) {
+ Integer part = e.partition();
+
+ Long cntr = rcvCntrs.get(part);
+
+ if (cntr != null) {
+ long cntr0 = cntr;
+
+ if (e.updateIndex() > cntr0) {
+ // TODO IGNITE-426: remove assert.
+ assert e.updateIndex() == cntr0 + 1 : "Invalid entry [cntr=" + cntr + ", e=" + e + ']';
+
+ rcvCntrs.put(part, e.updateIndex());
+ }
+ else
+ return false;
+ }
+ else
+ rcvCntrs.put(part, e.updateIndex());
+
+ return true;
+ }
+
/** {@inheritDoc} */
@Override public void p2pMarshal(GridKernalContext ctx) throws IgniteCheckedException {
assert ctx != null;
@@ -399,6 +535,65 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
/** {@inheritDoc} */
+ @Override public GridContinuousBatch createBatch() {
+ return new GridContinuousBatchAdapter();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onBatchAcknowledged(final UUID routineId,
+ GridContinuousBatch batch,
+ final GridKernalContext ctx) {
+ sendBackupAcknowledge(ackBuf.onAcknowledged(batch), routineId, ctx);
+ }
+
+ /**
+ * @param t Acknowledge information.
+ * @param routineId Routine ID.
+ * @param ctx Context.
+ */
+ private void sendBackupAcknowledge(final IgniteBiTuple<Map<Integer, Long>, Set<AffinityTopologyVersion>> t,
+ final UUID routineId,
+ final GridKernalContext ctx) {
+ if (t != null) {
+ ctx.closure().runLocalSafe(new Runnable() {
+ @Override public void run() {
+ GridCacheContext<K, V> cctx = cacheContext(ctx);
+
+ CacheContinuousQueryBatchAck msg = new CacheContinuousQueryBatchAck(cctx.cacheId(),
+ routineId,
+ t.get1());
+
+ Collection<ClusterNode> nodes = new HashSet<>();
+
+ for (AffinityTopologyVersion topVer : t.get2())
+ nodes.addAll(ctx.discovery().cacheNodes(topVer));
+
+ for (ClusterNode node : nodes) {
+ if (!node.id().equals(ctx.localNodeId())) {
+ try {
+ cctx.io().send(node, msg, GridIoPolicy.SYSTEM_POOL);
+ }
+ catch (ClusterTopologyCheckedException e) {
+ IgniteLogger log = ctx.log(getClass());
+
+ if (log.isDebugEnabled())
+ log.debug("Failed to send acknowledge message, node left " +
+ "[msg=" + msg + ", node=" + node + ']');
+ }
+ catch (IgniteCheckedException e) {
+ IgniteLogger log = ctx.log(getClass());
+
+ U.error(log, "Failed to send acknowledge message " +
+ "[msg=" + msg + ", node=" + node + ']', e);
+ }
+ }
+ }
+ }
+ });
+ }
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public Object orderedTopic() {
return topic;
}
@@ -473,6 +668,106 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
return ctx.cache().<K, V>context().cacheContext(cacheId);
}
+ /** */
+ private static class AcknowledgeBuffer {
+ /** */
+ private int size;
+
+ /** */
+ @GridToStringInclude
+ private Map<Integer, Long> updateIdxs = new HashMap<>();
+
+ /** */
+ @GridToStringInclude
+ private Set<AffinityTopologyVersion> topVers = U.newHashSet(1);
+
+ /**
+ * @param batch Batch.
+ * @return Non-null tuple if acknowledge should be sent to backups.
+ */
+ @SuppressWarnings("unchecked")
+ @Nullable synchronized IgniteBiTuple<Map<Integer, Long>, Set<AffinityTopologyVersion>>
+ onAcknowledged(GridContinuousBatch batch) {
+ size += batch.size();
+
+ Collection<CacheContinuousQueryEntry> entries = (Collection)batch.collect();
+
+ for (CacheContinuousQueryEntry e : entries)
+ addEntry(e);
+
+ return size >= BACKUP_ACK_THRESHOLD ? acknowledgeData() : null;
+ }
+
+ /**
+ * @param e Entry.
+ * @return Non-null tuple if acknowledge should be sent to backups.
+ */
+ @Nullable synchronized IgniteBiTuple<Map<Integer, Long>, Set<AffinityTopologyVersion>>
+ onAcknowledged(CacheContinuousQueryEntry e) {
+ size++;
+
+ addEntry(e);
+
+ return size >= BACKUP_ACK_THRESHOLD ? acknowledgeData() : null;
+ }
+
+ /**
+ * @param e Entry.
+ */
+ private void addEntry(CacheContinuousQueryEntry e) {
+ topVers.add(e.topologyVersion());
+
+ Long cntr0 = updateIdxs.get(e.partition());
+
+ if (cntr0 == null || e.updateIndex() > cntr0)
+ updateIdxs.put(e.partition(), e.updateIndex());
+ }
+
+ /**
+ * @return Non-null tuple if acknowledge should be sent to backups.
+ */
+ @Nullable synchronized IgniteBiTuple<Map<Integer, Long>, Set<AffinityTopologyVersion>>
+ acknowledgeOnTimeout() {
+ return size > 0 ? acknowledgeData() : null;
+ }
+
+ /**
+ * @return Tuple with acknowledge information.
+ */
+ private IgniteBiTuple<Map<Integer, Long>, Set<AffinityTopologyVersion>> acknowledgeData() {
+ assert size > 0;
+
+ Map<Integer, Long> idxs = new HashMap<>(updateIdxs);
+
+ IgniteBiTuple<Map<Integer, Long>, Set<AffinityTopologyVersion>> res =
+ new IgniteBiTuple<>(idxs, topVers);
+
+ topVers = U.newHashSet(1);
+
+ size = 0;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(AcknowledgeBuffer.class, this);
+ }
+ }
+
+ /**
+ *
+ */
+ private class DuplicateEventFilter implements IgnitePredicate<CacheContinuousQueryEntry> {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public boolean apply(CacheContinuousQueryEntry e) {
+ return notifyListener(e);
+ }
+ }
+
/**
* Deployable object.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
index a3c19a9..2f9e111 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
@@ -17,6 +17,11 @@
package org.apache.ignite.internal.processors.cache.query.continuous;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.affinity.*;
+
+import java.util.*;
+
/**
* Continuous query listener.
*/
@@ -33,7 +38,9 @@ interface CacheContinuousQueryListener<K, V> {
* @param primary Primary flag.
* @param recordIgniteEvt Whether to record event.
*/
- public void onEntryUpdated(CacheContinuousQueryEvent<K, V> evt, boolean primary, boolean recordIgniteEvt);
+ public void onEntryUpdated(CacheContinuousQueryEvent<K, V> evt,
+ boolean primary,
+ boolean recordIgniteEvt);
/**
* Listener unregistered callback.
@@ -41,6 +48,31 @@ interface CacheContinuousQueryListener<K, V> {
public void onUnregister();
/**
+ * Cleans backup queue.
+ *
+ * @param updateIdxs Update indexes map.
+ */
+ public void cleanupBackupQueue(Map<Integer, Long> updateIdxs);
+
+ /**
+ * Flushes backup queue.
+ *
+ * @param ctx Context.
+ * @param topVer Topology version.
+ */
+ public void flushBackupQueue(GridKernalContext ctx, AffinityTopologyVersion topVer);
+
+ /**
+ * @param ctx Context.
+ */
+ public void acknowledgeBackupOnTimeout(GridKernalContext ctx);
+
+ /**
+ * @param part Partition.
+ */
+ public void onPartitionEvicted(int part);
+
+ /**
* @return Whether old value is required.
*/
public boolean oldValueRequired();
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 6a151a5..c9fb656 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -52,6 +52,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
import org.apache.ignite.internal.processors.cache.GridCacheManagerAdapter;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.continuous.GridContinuousHandler;
+import org.apache.ignite.internal.util.typedef.CI2;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgnitePredicate;
@@ -82,6 +83,9 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
/** */
private static final byte EXPIRED_FLAG = 0b1000;
+ /** */
+ private static final long BACKUP_ACK_FREQ = 5000;
+
/** Listeners. */
private final ConcurrentMap<UUID, CacheContinuousQueryListener> lsnrs = new ConcurrentHashMap8<>();
@@ -108,6 +112,26 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
@Override protected void start0() throws IgniteCheckedException {
// Append cache name to the topic.
topicPrefix = "CONTINUOUS_QUERY" + (cctx.name() == null ? "" : "_" + cctx.name());
+
+ cctx.io().addHandler(cctx.cacheId(), CacheContinuousQueryBatchAck.class,
+ new CI2<UUID, CacheContinuousQueryBatchAck>() {
+ @Override public void apply(UUID uuid, CacheContinuousQueryBatchAck msg) {
+ CacheContinuousQueryListener lsnr = lsnrs.get(msg.routineId());
+
+ if (lsnr != null)
+ lsnr.cleanupBackupQueue(msg.updateIndexes());
+ }
+ });
+
+ cctx.time().schedule(new Runnable() {
+ @Override public void run() {
+ for (CacheContinuousQueryListener lsnr : lsnrs.values())
+ lsnr.acknowledgeBackupOnTimeout(cctx.kernalContext());
+
+ for (CacheContinuousQueryListener lsnr : intLsnrs.values())
+ lsnr.acknowledgeBackupOnTimeout(cctx.kernalContext());
+ }
+ }, BACKUP_ACK_FREQ, BACKUP_ACK_FREQ);
}
/** {@inheritDoc} */
@@ -141,18 +165,25 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
* @param key Key.
* @param newVal New value.
* @param oldVal Old value.
+ * @param primary {@code True} if called on primary node.
* @param preload Whether update happened during preloading.
+ * @param updateIdx Update index.
+ * @param topVer Topology version.
* @throws IgniteCheckedException In case of error.
*/
public void onEntryUpdated(GridCacheEntryEx e,
KeyCacheObject key,
CacheObject newVal,
CacheObject oldVal,
- boolean preload)
+ boolean primary,
+ boolean preload,
+ long updateIdx,
+ AffinityTopologyVersion topVer)
throws IgniteCheckedException
{
assert e != null;
assert key != null;
+ assert Thread.holdsLock(e) : e;
boolean internal = e.isInternal() || !e.context().userCache();
@@ -179,8 +210,7 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
boolean initialized = false;
- boolean primary = cctx.affinity().primary(cctx.localNode(), key, AffinityTopologyVersion.NONE);
- boolean recordIgniteEvt = !internal && cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ);
+ boolean recordIgniteEvt = primary && !internal && cctx.gridEvents().isRecordable(EVT_CACHE_QUERY_OBJECT_READ);
for (CacheContinuousQueryListener lsnr : lsnrCol.values()) {
if (preload && !lsnr.notifyExisting())
@@ -205,7 +235,10 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
evtType,
key,
newVal,
- lsnr.oldValueRequired() ? oldVal : null);
+ lsnr.oldValueRequired() ? oldVal : null,
+ e.partition(),
+ updateIdx,
+ topVer);
CacheContinuousQueryEvent evt = new CacheContinuousQueryEvent<>(
cctx.kernalContext().cache().jcache(cctx.name()), cctx, e0);
@@ -224,6 +257,7 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
throws IgniteCheckedException {
assert e != null;
assert key != null;
+ assert Thread.holdsLock(e) : e;
if (e.isInternal())
return;
@@ -255,7 +289,10 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
EXPIRED,
key,
null,
- lsnr.oldValueRequired() ? oldVal : null);
+ lsnr.oldValueRequired() ? oldVal : null,
+ e.partition(),
+ 0,
+ null);
CacheContinuousQueryEvent evt = new CacheContinuousQueryEvent(
cctx.kernalContext().cache().jcache(cctx.name()), cctx, e0);
@@ -373,6 +410,30 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
}
/**
+ * @param topVer Topology version.
+ */
+ public void beforeExchange(AffinityTopologyVersion topVer) {
+ for (CacheContinuousQueryListener lsnr : lsnrs.values())
+ lsnr.flushBackupQueue(cctx.kernalContext(), topVer);
+
+ for (CacheContinuousQueryListener lsnr : intLsnrs.values())
+ lsnr.flushBackupQueue(cctx.kernalContext(), topVer);
+ }
+
+ /**
+ * Partition evicted callback.
+ *
+ * @param part Partition number.
+ */
+ public void onPartitionEvicted(int part) {
+ for (CacheContinuousQueryListener lsnr : lsnrs.values())
+ lsnr.onPartitionEvicted(part);
+
+ for (CacheContinuousQueryListener lsnr : intLsnrs.values())
+ lsnr.onPartitionEvicted(part);
+ }
+
+ /**
* @param locLsnr Local listener.
* @param rmtFilter Remote filter.
* @param bufSize Buffer size.
@@ -417,7 +478,8 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
sync,
ignoreExpired,
taskNameHash,
- skipPrimaryCheck);
+ skipPrimaryCheck,
+ cctx.isLocal());
IgnitePredicate<ClusterNode> pred = null;
@@ -471,10 +533,19 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
GridCacheEntryEx e = it.next();
+ CacheContinuousQueryEntry entry = new CacheContinuousQueryEntry(
+ cctx.cacheId(),
+ CREATED,
+ e.key(),
+ e.rawGet(),
+ null,
+ 0,
+ 0,
+ null);
+
next = new CacheContinuousQueryEvent<>(
cctx.kernalContext().cache().jcache(cctx.name()),
- cctx,
- new CacheContinuousQueryEntry(cctx.cacheId(), CREATED, e.key(), e.rawGet(), null));
+ cctx, entry);
if (rmtFilter != null && !rmtFilter.evaluate(next))
next = null;
@@ -639,6 +710,7 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
/**
* @param impl Listener.
+ * @param log Logger.
*/
JCacheQueryLocalListener(CacheEntryListener<K, V> impl, IgniteLogger log) {
assert impl != null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatch.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatch.java
new file mode 100644
index 0000000..2fef161
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatch.java
@@ -0,0 +1,7 @@
+package org.apache.ignite.internal.processors.continuous;
+
+/**
+ * Created by Nikolay on 02.09.2015.
+ */
+public interface GridContinuousBatch {
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatchAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatchAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatchAdapter.java
new file mode 100644
index 0000000..8e29e29
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatchAdapter.java
@@ -0,0 +1,7 @@
+package org.apache.ignite.internal.processors.continuous;
+
+/**
+ * Created by Nikolay on 02.09.2015.
+ */
+public class GridContinuousBatchAdapter {
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f6ba6c9/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index d1cb3a9..15c9dd2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -651,6 +651,30 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
/**
* @param nodeId ID of the node that started routine.
* @param routineId Routine ID.
+ * @param objs Notification objects.
+ * @param orderedTopic Topic for ordered notifications. If {@code null}, non-ordered message will be sent.
+ * @throws IgniteCheckedException In case of error.
+ */
+ public void addBackupNotification(UUID nodeId,
+ final UUID routineId,
+ Collection<?> objs,
+ @Nullable Object orderedTopic)
+ throws IgniteCheckedException {
+ if (processorStopped)
+ return;
+
+ final RemoteRoutineInfo info = rmtInfos.get(routineId);
+
+ if (info != null) {
+ final GridContinuousBatch batch = info.addAll(objs);
+
+ sendNotification(nodeId, routineId, null, batch.collect(), orderedTopic, true, null);
+ }
+ }
+
+ /**
+ * @param nodeId ID of the node that started routine.
+ * @param routineId Routine ID.
* @param obj Notification object.
* @param orderedTopic Topic for ordered notifications. If {@code null}, non-ordered message will be sent.
* @param sync If {@code true} then waits for event acknowledgment.
@@ -658,8 +682,8 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
* @throws IgniteCheckedException In case of error.
*/
public void addNotification(UUID nodeId,
- UUID routineId,
- @Nullable Object obj,
+ final UUID routineId,
+ Object obj,
@Nullable Object orderedTopic,
boolean sync,
boolean msg)
@@ -673,7 +697,7 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
if (processorStopped)
return;
- RemoteRoutineInfo info = rmtInfos.get(routineId);
+ final RemoteRoutineInfo info = rmtInfos.get(routineId);
if (info != null) {
assert info.interval == 0 || !sync;
@@ -686,7 +710,7 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
syncMsgFuts.put(futId, fut);
try {
- sendNotification(nodeId, routineId, futId, F.asList(obj), orderedTopic, msg);
+ sendNotification(nodeId, routineId, futId, F.asList(obj), orderedTopic, msg, null);
}
catch (IgniteCheckedException e) {
syncMsgFuts.remove(futId);
@@ -697,10 +721,18 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
fut.get();
}
else {
- Collection<Object> toSnd = info.add(obj);
+ final GridContinuousBatch batch = info.add(obj);
+
+ if (batch != null) {
+ CI1<IgniteException> ackC = new CI1<IgniteException>() {
+ @Override public void apply(IgniteException e) {
+ if (e == null)
+ info.hnd.onBatchAcknowledged(routineId, batch, ctx);
+ }
+ };
- if (toSnd != null)
- sendNotification(nodeId, routineId, null, toSnd, orderedTopic, msg);
+ sendNotification(nodeId, routineId, null, batch.collect(), orderedTopic, msg, ackC);
+ }
}
}
}
@@ -725,6 +757,7 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
* @param orderedTopic Topic for ordered notifications.
* If {@code null}, non-ordered message will be sent.
* @param msg If {@code true} then sent data is collection of messages.
+ * @param ackC Ack closure.
* @throws IgniteCheckedException In case of error.
*/
private void sendNotification(UUID nodeId,
@@ -732,7 +765,8 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
@Nullable IgniteUuid futId,
Collection<Object> toSnd,
@Nullable Object orderedTopic,
- boolean msg) throws IgniteCheckedException {
+ boolean msg,
+ IgniteInClosure<IgniteException> ackC) throws IgniteCheckedException {
assert nodeId != null;
assert routineId != null;
assert toSnd != null;
@@ -740,7 +774,8 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
sendWithRetries(nodeId,
new GridContinuousMessage(MSG_EVT_NOTIFICATION, routineId, futId, toSnd, msg),
- orderedTopic);
+ orderedTopic,
+ ackC);
}
/**
@@ -859,6 +894,7 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
try {
sendWithRetries(nodeId,
new GridContinuousMessage(MSG_EVT_ACK, null, msg.futureId(), null, false),
+ null,
null);
}
catch (IgniteCheckedException e) {
@@ -922,15 +958,30 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
break;
}
- IgniteBiTuple<Collection<Object>, Long> t = info.checkInterval();
+ IgniteBiTuple<GridContinuousBatch, Long> t = info.checkInterval();
- Collection<Object> toSnd = t.get1();
+ final GridContinuousBatch batch = t.get1();
- if (toSnd != null && !toSnd.isEmpty()) {
+ if (batch != null && batch.size() > 0) {
try {
+ Collection<Object> toSnd = batch.collect();
+
boolean msg = toSnd.iterator().next() instanceof Message;
- sendNotification(nodeId, routineId, null, toSnd, hnd.orderedTopic(), msg);
+ CI1<IgniteException> ackC = new CI1<IgniteException>() {
+ @Override public void apply(IgniteException e) {
+ if (e == null)
+ info.hnd.onBatchAcknowledged(routineId, batch, ctx);
+ }
+ };
+
+ sendNotification(nodeId,
+ routineId,
+ null,
+ toSnd,
+ hnd.orderedTopic(),
+ msg,
+ ackC);
}
catch (ClusterTopologyCheckedException ignored) {
if (log.isDebugEnabled())
@@ -1013,9 +1064,11 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
* @param msg Message.
* @param orderedTopic Topic for ordered notifications.
* If {@code null}, non-ordered message will be sent.
+ * @param ackC Ack closure.
* @throws IgniteCheckedException In case of error.
*/
- private void sendWithRetries(UUID nodeId, GridContinuousMessage msg, @Nullable Object orderedTopic)
+ private void sendWithRetries(UUID nodeId, GridContinuousMessage msg, @Nullable Object orderedTopic,
+ IgniteInClosure<IgniteException> ackC)
throws IgniteCheckedException {
assert nodeId != null;
assert msg != null;
@@ -1023,7 +1076,7 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
ClusterNode node = ctx.discovery().node(nodeId);
if (node != null)
- sendWithRetries(node, msg, orderedTopic);
+ sendWithRetries(node, msg, orderedTopic, ackC);
else
throw new ClusterTopologyCheckedException("Node for provided ID doesn't exist (did it leave the grid?): " + nodeId);
}
@@ -1033,14 +1086,15 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
* @param msg Message.
* @param orderedTopic Topic for ordered notifications.
* If {@code null}, non-ordered message will be sent.
+ * @param ackC Ack closure.
* @throws IgniteCheckedException In case of error.
*/
- private void sendWithRetries(ClusterNode node, GridContinuousMessage msg, @Nullable Object orderedTopic)
- throws IgniteCheckedException {
+ private void sendWithRetries(ClusterNode node, GridContinuousMessage msg, @Nullable Object orderedTopic,
+ IgniteInClosure<IgniteException> ackC) throws IgniteCheckedException {
assert node != null;
assert msg != null;
- sendWithRetries(F.asList(node), msg, orderedTopic);
+ sendWithRetries(F.asList(node), msg, orderedTopic, ackC);
}
/**
@@ -1048,10 +1102,11 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
* @param msg Message.
* @param orderedTopic Topic for ordered notifications.
* If {@code null}, non-ordered message will be sent.
+ * @param ackC Ack closure.
* @throws IgniteCheckedException In case of error.
*/
private void sendWithRetries(Collection<? extends ClusterNode> nodes, GridContinuousMessage msg,
- @Nullable Object orderedTopic) throws IgniteCheckedException {
+ @Nullable Object orderedTopic, IgniteInClosure<IgniteException> ackC) throws IgniteCheckedException {
assert !F.isEmpty(nodes);
assert msg != null;
@@ -1074,10 +1129,11 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
msg,
SYSTEM_POOL,
0,
- true);
+ true,
+ ackC);
}
else
- ctx.io().send(node, TOPIC_CONTINUOUS, msg, SYSTEM_POOL);
+ ctx.io().send(node, TOPIC_CONTINUOUS, msg, SYSTEM_POOL, ackC);
break;
}
@@ -1178,8 +1234,8 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
/** Lock. */
private final ReadWriteLock lock = new ReentrantReadWriteLock();
- /** Buffer. */
- private ConcurrentLinkedDeque8<Object> buf;
+ /** Batch. */
+ private GridContinuousBatch batch;
/** Last send time. */
private long lastSndTime = U.currentTimeMillis();
@@ -1210,7 +1266,7 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
this.interval = interval;
this.autoUnsubscribe = autoUnsubscribe;
- buf = new ConcurrentLinkedDeque8<>();
+ batch = hnd.createBatch();
}
/**
@@ -1238,21 +1294,53 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
}
/**
+ * @param objs Objects to add.
+ * @return Batch to send.
+ */
+ GridContinuousBatch addAll(Collection<?> objs) {
+ assert objs != null;
+ assert objs.size() > 0;
+
+ GridContinuousBatch toSnd = null;
+
+ lock.writeLock().lock();
+
+ try {
+ for (Object obj : objs)
+ batch.add(obj);
+
+ toSnd = batch;
+
+ batch = hnd.createBatch();
+
+ if (interval > 0)
+ lastSndTime = U.currentTimeMillis();
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
+
+ return toSnd;
+ }
+
+ /**
* @param obj Object to add.
- * @return Object to send or {@code null} if there is nothing to send for now.
+ * @return Batch to send or {@code null} if there is nothing to send for now.
*/
- @Nullable Collection<Object> add(@Nullable Object obj) {
- ConcurrentLinkedDeque8 buf0 = null;
+ @Nullable GridContinuousBatch add(Object obj) {
+ assert obj != null;
- if (buf.sizex() >= bufSize - 1) {
+ GridContinuousBatch toSnd = null;
+
+ if (batch.size() >= bufSize - 1) {
lock.writeLock().lock();
try {
- buf.add(obj);
+ batch.add(obj);
- buf0 = buf;
+ toSnd = batch;
- buf = new ConcurrentLinkedDeque8<>();
+ batch = hnd.createBatch();
if (interval > 0)
lastSndTime = U.currentTimeMillis();
@@ -1265,34 +1353,25 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
lock.readLock().lock();
try {
- buf.add(obj);
+ batch.add(obj);
}
finally {
lock.readLock().unlock();
}
}
- Collection<Object> toSnd = null;
-
- if (buf0 != null) {
- toSnd = new ArrayList<>(buf0.sizex());
-
- for (Object o : buf0)
- toSnd.add(o);
- }
-
return toSnd;
}
/**
- * @return Tuple with objects to sleep (or {@code null} if there is nothing to
+ * @return Tuple with batch to send (or {@code null} if there is nothing to
* send for now) and time interval after next check is needed.
*/
@SuppressWarnings("TooBroadScope")
- IgniteBiTuple<Collection<Object>, Long> checkInterval() {
+ IgniteBiTuple<GridContinuousBatch, Long> checkInterval() {
assert interval > 0;
- Collection<Object> toSnd = null;
+ GridContinuousBatch toSnd = null;
long diff;
long now = U.currentTimeMillis();
@@ -1302,10 +1381,10 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
try {
diff = now - lastSndTime;
- if (diff >= interval && !buf.isEmpty()) {
- toSnd = buf;
+ if (diff >= interval && batch.size() > 0) {
+ toSnd = batch;
- buf = new ConcurrentLinkedDeque8<>();
+ batch = hnd.createBatch();
lastSndTime = now;
}
[7/9] ignite git commit: IGNITE-426 Added cache continuos query
probe. Implemented for TX.
Posted by nt...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java
new file mode 100644
index 0000000..e42479a
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/CacheEntryEventProbe.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.cache;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryListenerException;
+import javax.cache.event.CacheEntryUpdatedListener;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkDriver;
+import org.yardstickframework.BenchmarkProbe;
+import org.yardstickframework.BenchmarkProbePoint;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.yardstickframework.BenchmarkUtils.errorHelp;
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Probe which calculate continuous query events.
+ */
+public class CacheEntryEventProbe implements BenchmarkProbe {
+ /** */
+ private BenchmarkConfiguration cfg;
+
+ /** Counter. */
+ private AtomicLong cnt = new AtomicLong(0);
+
+ /** Collected points. */
+ private Collection<BenchmarkProbePoint> collected = new ArrayList<>();
+
+ /** Query cursor. */
+ private QueryCursor qryCur;
+
+ /** Service building probe points. */
+ private ExecutorService buildingService;
+
+ /** {@inheritDoc} */
+ @Override public void start(BenchmarkDriver drv, BenchmarkConfiguration cfg) throws Exception {
+ this.cfg = cfg;
+
+ if (drv instanceof IgniteCacheAbstractBenchmark) {
+ IgniteCacheAbstractBenchmark drv0 = (IgniteCacheAbstractBenchmark)drv;
+
+ if (drv0.cache() != null) {
+ ContinuousQuery<Integer, Integer> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(new CacheEntryUpdatedListener<Integer, Integer>() {
+ @Override public void onUpdated(Iterable<CacheEntryEvent<? extends Integer, ? extends Integer>>
+ events) throws CacheEntryListenerException {
+ int size = 0;
+
+ for (CacheEntryEvent<? extends Integer, ? extends Integer> e : events)
+ ++size;
+
+ cnt.addAndGet(size);
+ }
+ });
+
+ qryCur = drv0.cache().query(qry);
+
+ buildingService = Executors.newSingleThreadExecutor();
+
+ buildingService.submit(new Runnable() {
+ @Override public void run() {
+ try {
+ while (!Thread.currentThread().isInterrupted()) {
+ Thread.sleep(1000);
+
+ long evts = cnt.getAndSet(0);
+
+ BenchmarkProbePoint pnt = new BenchmarkProbePoint(
+ TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()),
+ new double[] {evts});
+
+ collectPoint(pnt);
+ }
+ }
+ catch (InterruptedException e) {
+ // No-op.
+ }
+ }
+ });
+
+ println(cfg, getClass().getSimpleName() + " probe is started.");
+ }
+ }
+
+ if (qryCur == null)
+ errorHelp(cfg, "Can not start " + getClass().getSimpleName()
+ + " probe. Probably, the driver doesn't provide \"cache()\" method.");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws Exception {
+ if (qryCur != null) {
+ qryCur.close();
+
+ qryCur = null;
+
+ buildingService.shutdownNow();
+
+ buildingService.awaitTermination(1, MINUTES);
+
+ println(cfg, getClass().getSimpleName() + " is stopped.");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<String> metaInfo() {
+ return Arrays.asList("Time, sec", "Received events/sec (more is better)");
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized Collection<BenchmarkProbePoint> points() {
+ Collection<BenchmarkProbePoint> ret = collected;
+
+ collected = new ArrayList<>(ret.size() + 5);
+
+ return ret;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void buildPoint(long time) {
+ // No-op.
+ }
+
+ /**
+ * @param pnt Probe point.
+ */
+ private synchronized void collectPoint(BenchmarkProbePoint pnt) {
+ collected.add(pnt);
+ }
+}
[6/9] ignite git commit: IGNITE-426 WIP test and semen reviewed fixes
Posted by nt...@apache.org.
IGNITE-426 WIP test and semen reviewed fixes
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/df45134d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/df45134d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/df45134d
Branch: refs/heads/ignite-426-2-reb
Commit: df45134dd2ba98a9fad698cec82f7cc65d2a9daf
Parents: f262e5f
Author: nikolay_tikhonov <nt...@gridgain.com>
Authored: Thu Oct 8 15:27:28 2015 +0300
Committer: nikolay_tikhonov <nt...@gridgain.com>
Committed: Wed Oct 21 16:54:25 2015 +0300
----------------------------------------------------------------------
.../internal/GridEventConsumeHandler.java | 5 -
.../internal/GridMessageListenHandler.java | 5 -
.../communication/GridIoMessageFactory.java | 6 -
.../processors/cache/GridCacheMapEntry.java | 4 +-
.../distributed/dht/GridDhtLocalPartition.java | 30 +-
.../dht/GridDhtPartitionTopologyImpl.java | 6 +-
.../continuous/CacheContinuousQueryEntry.java | 70 +-
.../continuous/CacheContinuousQueryHandler.java | 263 ++---
.../CacheContinuousQueryListener.java | 15 -
.../CacheContinuousQueryLostPartition.java | 148 ---
.../continuous/CacheContinuousQueryManager.java | 33 +-
.../continuous/GridContinuousHandler.java | 6 -
...acheContinuousQueryFailoverAbstractTest.java | 965 ++++++++++++++++---
13 files changed, 985 insertions(+), 571 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
index 1b9c46c..1eeedeb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
@@ -386,11 +386,6 @@ class GridEventConsumeHandler implements GridContinuousHandler {
}
/** {@inheritDoc} */
- @Override public void partitionLost(String cacheName, int partId) {
- // No-op.
- }
-
- /** {@inheritDoc} */
@Nullable @Override public Object orderedTopic() {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/GridMessageListenHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridMessageListenHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/GridMessageListenHandler.java
index e038794..bddebba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridMessageListenHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridMessageListenHandler.java
@@ -179,11 +179,6 @@ public class GridMessageListenHandler implements GridContinuousHandler {
}
/** {@inheritDoc} */
- @Override public void partitionLost(String cacheName, int partId) {
- // No-op.
- }
-
- /** {@inheritDoc} */
@Nullable @Override public Object orderedTopic() {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 3474f84..6f71d57 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -92,7 +92,6 @@ import org.apache.ignite.internal.processors.cache.query.GridCacheQueryResponse;
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryBatchAck;
import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryEntry;
-import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryLostPartition;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.transactions.TxEntryValueHolder;
@@ -691,11 +690,6 @@ public class GridIoMessageFactory implements MessageFactory {
break;
- case 115:
- msg = new CacheContinuousQueryLostPartition();
-
- break;
-
// [-3..115] - this
// [120..123] - DR
// [-4..-22] - SQL
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 259869e..570172d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -3156,14 +3156,12 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
private long nextPartIndex(AffinityTopologyVersion topVer) {
long updateIdx;
- //U.dumpStack();
-
if (!cctx.isLocal() && !isNear()) {
GridDhtLocalPartition locPart = cctx.topology().localPartition(partition(), topVer, false);
assert locPart != null;
- updateIdx = locPart.nextContinuousQueryUpdateIndex();
+ updateIdx = locPart.nextUpdateIndex();
}
else
updateIdx = 0;
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 975d76c..baa8520 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -58,18 +58,6 @@ import org.jetbrains.annotations.NotNull;
import org.jsr166.ConcurrentHashMap8;
import org.jsr166.LongAdder8;
-import javax.cache.CacheException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.atomic.AtomicStampedReference;
-import java.util.concurrent.locks.ReentrantLock;
-
import static org.apache.ignite.IgniteSystemProperties.IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE;
import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_OBJECT_UNLOADED;
import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
@@ -128,8 +116,8 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
/** Group reservations. */
private final CopyOnWriteArrayList<GridDhtPartitionsReservation> reservations = new CopyOnWriteArrayList<>();
- /** Continuous query update index. */
- private final AtomicLong contQryUpdIdx = new AtomicLong();
+ /** Update index. */
+ private final AtomicLong updIdx = new AtomicLong();
/**
* @param cctx Context.
@@ -628,28 +616,28 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
/**
* @return Next update index.
*/
- public long nextContinuousQueryUpdateIndex() {
- return contQryUpdIdx.incrementAndGet();
+ public long nextUpdateIndex() {
+ return updIdx.incrementAndGet();
}
/**
* @return Current update index.
*/
- public long continuousQueryUpdateIndex() {
- return contQryUpdIdx.get();
+ public long updateIndex() {
+ return updIdx.get();
}
/**
* @param val Update index value.
*/
- public void continuousQueryUpdateIndex(long val) {
+ public void updateIndex(long val) {
while (true) {
- long val0 = contQryUpdIdx.get();
+ long val0 = updIdx.get();
if (val0 >= val)
break;
- if (contQryUpdIdx.compareAndSet(val0, val))
+ if (updIdx.compareAndSet(val0, val))
break;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 5d312b6..098a60d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -939,7 +939,7 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
Long cntr = cntrMap.get(part.id());
if (cntr != null)
- part.continuousQueryUpdateIndex(cntr);
+ part.updateIndex(cntr);
}
}
@@ -1053,7 +1053,7 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
Long cntr = cntrMap.get(part.id());
if (cntr != null)
- part.continuousQueryUpdateIndex(cntr);
+ part.updateIndex(cntr);
}
}
@@ -1317,7 +1317,7 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
for (GridDhtLocalPartition part : locParts.values()) {
Long cntr0 = res.get(part.id());
- Long cntr1 = part.continuousQueryUpdateIndex();
+ Long cntr1 = part.updateIndex();
if (cntr0 == null || cntr1 > cntr0)
res.put(part.id(), cntr1);
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index 9e73142..eefbbae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -43,6 +43,15 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
private static final long serialVersionUID = 0L;
/** */
+ private static final byte BACKUP_ENTRY = 0b0001;
+
+ /** */
+ private static final byte ORDERED_ENTRY = 0b0010;
+
+ /** */
+ private static final byte FILTERED_ENTRY = 0b0100;
+
+ /** */
private static final EventType[] EVT_TYPE_VALS = EventType.values();
/**
@@ -82,8 +91,8 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
/** Update index. */
private long updateIdx;
- /** */
- private boolean filtered;
+ /** Flags. */
+ private byte flags;
/** */
@GridToStringInclude
@@ -91,7 +100,7 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
private AffinityTopologyVersion topVer;
/**
- * Required by {@link org.apache.ignite.plugin.extensions.communication.Message}.
+ * Required by {@link Message}.
*/
public CacheContinuousQueryEntry() {
// No-op.
@@ -134,13 +143,6 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
}
/**
- * @return Cache ID.
- */
- int cacheId() {
- return cacheId;
- }
-
- /**
* @return Event type.
*/
EventType eventType() {
@@ -155,24 +157,52 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
}
/**
- * Mark this event as filtered.
+ * @return Update index.
+ */
+ long updateIndex() {
+ return updateIdx;
+ }
+
+ /**
+ * Mark that entry create on backup.
+ */
+ void markBackup() {
+ flags |= BACKUP_ENTRY;
+ }
+
+ /**
+ * Mark that entry ordered.
+ */
+ void markOrdered() {
+ flags |= ORDERED_ENTRY;
+ }
+
+ /**
+ * Mark that entry filtered.
*/
void markFiltered() {
- filtered = true;
+ flags |= FILTERED_ENTRY;
}
/**
- * @return Update index.
+ * @return {@code True} if entry sent by backup node.
*/
- long updateIndex() {
- return updateIdx;
+ boolean isBackup() {
+ return (flags & BACKUP_ENTRY) != 0;
+ }
+
+ /**
+ * @return {@code True} .
+ */
+ boolean isOrdered() {
+ return (flags & ORDERED_ENTRY) != 0;
}
/**
- * @return Filtered entry.
+ * @return {@code True} if entry was filtered.
*/
boolean filtered() {
- return filtered;
+ return (flags & FILTERED_ENTRY) != 0;
}
/**
@@ -297,7 +327,7 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
writer.incrementState();
case 7:
- if (!writer.writeBoolean("filtered", filtered))
+ if (!writer.writeByte("flags", flags))
return false;
writer.incrementState();
@@ -376,7 +406,7 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
reader.incrementState();
case 7:
- filtered = reader.readBoolean("filtered");
+ flags = reader.readByte("flags");
if (!reader.isLastRead())
return false;
@@ -390,7 +420,7 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 7;
+ return 8;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index bb2558c..4734998 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -63,6 +64,7 @@ import org.apache.ignite.internal.processors.platform.cache.query.PlatformContin
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.C1;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -84,9 +86,6 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
/** */
private static final int BACKUP_ACK_THRESHOLD = 100;
- /** */
- private static final int QUERY_HOLE_THRESHOLD = 5;
-
/** Cache name. */
private String cacheName;
@@ -291,20 +290,7 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
if (primary || skipPrimaryCheck) {
if (loc) {
if (!localCache) {
- PartitionRecovery rcv = rcvs.get(entry.partition());
-
- if (rcv == null) {
- rcv = new PartitionRecovery(ctx.log(getClass()));
-
- PartitionRecovery oldRec = rcvs.putIfAbsent(entry.partition(), rcv);
-
- if (oldRec != null)
- rcv = oldRec;
- }
-
- rcv.add(entry);
-
- Collection<CacheContinuousQueryEntry> entries = rcv.entries();
+ Collection<CacheContinuousQueryEntry> entries = handleEntry(ctx, entry);
if (!entries.isEmpty()) {
final IgniteCache cache = cctx.kernalContext().cache().jcache(cctx.name());
@@ -342,8 +328,11 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
ctx.continuous().addNotification(nodeId, routineId, entry, topic, sync, true);
}
}
- else
+ else {
+ entry.markBackup();
+
backupQueue.add(entry);
+ }
}
catch (ClusterTopologyCheckedException ex) {
IgniteLogger log = ctx.log(getClass());
@@ -378,54 +367,6 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
}
- @Override public void partitionLost(int partId) {
- assert rcvs != null;
-
- PartitionRecovery rcv = rcvs.get(partId);
-
- if (rcv != null)
- rcv.reset();
- }
-
- @Override public void firePartitionLostEvent(String cacheName0, final int partId) {
- GridCacheContext<K, V> cctx = cacheContext(ctx);
-
- // Check that cache stopped.
- if (cctx == null)
- return;
-
- if ((cacheName == null && cacheName0 == null) || // Check default cache.
- (cacheName0 != null && cacheName != null && cacheName0.equals(cacheName))) {
- ctx.closure().runLocalSafe(new Runnable() {
- @Override public void run() {
- GridCacheContext<K, V> cctx = cacheContext(ctx);
-
- CacheContinuousQueryLostPartition msg = new CacheContinuousQueryLostPartition(
- routineId,
- cctx.cacheId(),
- partId);
-
- try {
- cctx.io().send(nodeId, msg, GridIoPolicy.SYSTEM_POOL);
- }
- catch (ClusterTopologyCheckedException e) {
- IgniteLogger log = ctx.log(getClass());
-
- if (log.isDebugEnabled())
- log.debug("Failed to send lost partition message, node left " +
- "[msg=" + msg + ", nodeId=" + routineId + ']');
- }
- catch (IgniteCheckedException e) {
- IgniteLogger log = ctx.log(getClass());
-
- U.error(log, "Failed to send lost partition message " +
- "[msg=" + msg + ", nodeId=" + routineId + ']', e);
- }
- }
- });
- }
- }
-
@Override public void onUnregister() {
if (rmtFilter instanceof PlatformContinuousQueryFilter)
((PlatformContinuousQueryFilter)rmtFilter).onQueryUnregister();
@@ -574,31 +515,10 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
final IgniteCache cache = cctx.kernalContext().cache().jcache(cctx.name());
- Map<Integer, PartitionRecovery> parts = new HashMap<>();
-
- for (CacheContinuousQueryEntry e : entries) {
- PartitionRecovery rec = parts.containsKey(e.partition()) ?
- parts.get(e.partition()) : rcvs.get(e.partition());
-
- if (rec == null) {
- rec = new PartitionRecovery(ctx.log(getClass()));
-
- PartitionRecovery oldRec = rcvs.putIfAbsent(e.partition(), rec);
-
- if (oldRec != null)
- rec = oldRec;
- }
-
- rec.add(e);
-
- if (!parts.containsKey(e.partition()))
- parts.put(e.partition(), rec);
- }
-
Collection<CacheContinuousQueryEntry> entries0 = new ArrayList<>();
- for (PartitionRecovery rec : parts.values())
- entries0.addAll(rec.entries());
+ for (CacheContinuousQueryEntry e : entries)
+ entries0.addAll(handleEntry(ctx, e));
Iterable<CacheEntryEvent<? extends K, ? extends V>> evts = F.viewReadOnly(entries0,
new C1<CacheContinuousQueryEntry, CacheEntryEvent<? extends K, ? extends V>>() {
@@ -617,6 +537,39 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
/**
+ * @param ctx Context.
+ * @param e entry.
+ * @return Entry collection.
+ */
+ private Collection<CacheContinuousQueryEntry> handleEntry(GridKernalContext ctx, CacheContinuousQueryEntry e) {
+ assert e != null;
+
+ // Initial query entry or evicted entry.
+ // This events should be fired immediately.
+ if (e.updateIndex() == -1)
+ return F.asList(e);
+
+ PartitionRecovery rec = rcvs.get(e.partition());
+
+ if (rec == null) {
+ rec = new PartitionRecovery(ctx.log(getClass()));
+
+ PartitionRecovery oldRec = rcvs.putIfAbsent(e.partition(), rec);
+
+ if (oldRec != null)
+ rec = oldRec;
+ }
+
+ Collection<CacheContinuousQueryEntry> entries = rec.collectEntries(e);
+
+ if (CacheContinuousQueryManager.SUPER_DEBUG)
+ ctx.log(getClass()).error("Fire the following event for partition : " + e.partition() +
+ " Entries: " + Arrays.toString(entries.toArray()));
+
+ return entries;
+ }
+
+ /**
*
*/
private static class PartitionRecovery {
@@ -624,11 +577,16 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
private IgniteLogger log;
/** */
- private long lastFiredEvt = 0;
+ private static final long INIT_VALUE = -100;
+
+ /** */
+ private long lastFiredEvt = INIT_VALUE;
/** */
private final Map<Long, CacheContinuousQueryEntry> pendingEnts = new TreeMap<>();
+ private List<T2<Long, CacheContinuousQueryEntry>> firedEvents = new ArrayList<>();
+
/**
* @param log Logger.
*/
@@ -639,99 +597,83 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
/**
* Add continuous entry.
*
- * @param e Cache continuous qeury entry.
+ * @param entry Cache continuous query entry.
+ * @return Collection entries which will be fired.
*/
- public void add(CacheContinuousQueryEntry e) {
- assert e != null;
+ public Collection<CacheContinuousQueryEntry> collectEntries(CacheContinuousQueryEntry entry) {
+ assert entry != null;
- synchronized (pendingEnts) {
- if (!pendingEnts.containsKey(e.updateIndex()) && e.updateIndex() > lastFiredEvt)
- pendingEnts.put(e.updateIndex(), e);
- else if (log.isDebugEnabled())
- log.debug("Skip duplicate continuous query message: " + e);
- }
- }
-
- /**
- * @return Ordered continuous query entries.
- */
- public Collection<CacheContinuousQueryEntry> entries() {
- List<CacheContinuousQueryEntry> entries = new ArrayList<>();
+ List<CacheContinuousQueryEntry> entries;
synchronized (pendingEnts) {
- if (pendingEnts.isEmpty())
- return Collections.emptyList();
+ // Received first event.
+ if (lastFiredEvt == INIT_VALUE) {
+ if (CacheContinuousQueryManager.SUPER_DEBUG)
+ log.error("First event. " + entry);
- Iterator<Map.Entry<Long, CacheContinuousQueryEntry>> iter = pendingEnts.entrySet().iterator();
+ lastFiredEvt = entry.updateIndex();
- boolean fired = false;
+ firedEvents.add(new T2<>(lastFiredEvt, entry));
- // The elements are consistently.
- while (iter.hasNext()) {
- Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
-
- if (e.getKey() == lastFiredEvt + 1) {
- ++lastFiredEvt;
-
- entries.add(e.getValue());
-
- iter.remove();
-
- fired = true;
- }
+ return F.asList(entry);
}
- if (!fired && lastFiredEvt == 0 && pendingEnts.size() >= QUERY_HOLE_THRESHOLD) {
- Long prevCnt = null;
+ // Handle case when nodes owning partition left from topology.
+ if (entry.updateIndex() == 1 && !entry.isBackup()) {
+ pendingEnts.clear();
- int orderedCnt = 0;
+ lastFiredEvt = 1;
- for (Long cnt : pendingEnts.keySet()) {
- if (prevCnt != null) {
- if (prevCnt + 1 != cnt)
- break;
- else
- ++orderedCnt;
- }
+ if (CacheContinuousQueryManager.SUPER_DEBUG)
+ log.error("Lost partition. Start from 1. Entry: " + entry);
- prevCnt = cnt;
- }
+ firedEvents.add(new T2<>(lastFiredEvt, entry));
- if (orderedCnt >= QUERY_HOLE_THRESHOLD) {
- iter = pendingEnts.entrySet().iterator();
+ return F.asList(entry);
+ }
- while (entries.size() < orderedCnt) {
- Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
+ // Check duplicate.
+ if (entry.updateIndex() > lastFiredEvt) {
+ if (CacheContinuousQueryManager.SUPER_DEBUG)
+ log.error("Put message to pending queue. Counter value: " + lastFiredEvt + " Entry: " + entry);
- entries.add(e.getValue());
+ pendingEnts.put(entry.updateIndex(), entry);
+ }
+ else {
+ if (log.isDebugEnabled())
+ log.debug("Skip duplicate continuous query message: " + entry);
- lastFiredEvt = e.getKey();
+ if (CacheContinuousQueryManager.SUPER_DEBUG)
+ log.error("Received duplicate. Counter value: " + lastFiredEvt + " Entry: " + entry
+ + ", Proceed message " + Arrays.toString(firedEvents.toArray()));
- iter.remove();
- }
- }
+ return Collections.emptyList();
}
- }
- return entries;
- }
+ if (pendingEnts.isEmpty())
+ return Collections.emptyList();
- /**
- * Reset internal state.
- */
- public void reset() {
- synchronized (pendingEnts) {
Iterator<Map.Entry<Long, CacheContinuousQueryEntry>> iter = pendingEnts.entrySet().iterator();
+ entries = new ArrayList<>();
+
+ // Elements are consistently.
while (iter.hasNext()) {
Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
- if (e.getKey() >= lastFiredEvt)
+ if (e.getKey() == lastFiredEvt + 1) {
+ ++lastFiredEvt;
+
+ entries.add(e.getValue());
+
+ firedEvents.add(new T2<>(e.getKey(), e.getValue()));
+
iter.remove();
+ }
}
-
- lastFiredEvt = 0;
}
+
+ return entries;
}
}
@@ -766,17 +708,6 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
sendBackupAcknowledge(ackBuf.onAcknowledged(batch), routineId, ctx);
}
- /** {@inheritDoc} */
- @Override public void partitionLost(String cacheName, int partId) {
- if ((this.cacheName == null && cacheName == null) // Check default caches.
- || (cacheName != null && this.cacheName != null && cacheName.equals(this.cacheName))) {
- PartitionRecovery rcv = rcvs.get(partId);
-
- if (rcv != null)
- rcv.reset();
- }
- }
-
/**
* @param t Acknowledge information.
* @param routineId Routine ID.
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
index a706105..2f9e111 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
@@ -55,21 +55,6 @@ interface CacheContinuousQueryListener<K, V> {
public void cleanupBackupQueue(Map<Integer, Long> updateIdxs);
/**
- * Fire event that partition lost.
- *
- * @param cacheName Cache name.
- * @param partId Partition ID.
- */
- public void firePartitionLostEvent(String cacheName, int partId);
-
- /**
- * Handle partition lost event.
- *
- * @param partId Partition ID.
- */
- public void partitionLost(int partId);
-
- /**
* Flushes backup queue.
*
* @param ctx Context.
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java
deleted file mode 100644
index eeb20cc..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.query.continuous;
-
-import java.nio.ByteBuffer;
-import java.util.UUID;
-import org.apache.ignite.internal.processors.cache.GridCacheMessage;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.plugin.extensions.communication.MessageReader;
-import org.apache.ignite.plugin.extensions.communication.MessageWriter;
-
-/**
- * Continuous query entry.
- */
-public class CacheContinuousQueryLostPartition extends GridCacheMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Routine ID. */
- private UUID routineId;
-
- /** Partition. */
- private int part;
-
- /**
- * Required by {@link Message}.
- */
- public CacheContinuousQueryLostPartition() {
- // No-op.
- }
-
- /**
- * @param cacheId Cache ID.
- * @param part Partition ID.
- */
- CacheContinuousQueryLostPartition(UUID routineId, int cacheId, int part) {
- this.routineId = routineId;
- this.cacheId = cacheId;
- this.part = part;
- }
-
- /**
- * @return Partition.
- */
- int partition() {
- return part;
- }
-
- /**
- * @return Routine ID.
- */
- UUID routineId() {
- return routineId;
- }
-
- /** {@inheritDoc} */
- @Override public byte directType() {
- return 115;
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- writer.setBuffer(buf);
-
- if (!super.writeTo(buf, writer))
- return false;
-
- if (!writer.isHeaderWritten()) {
- if (!writer.writeHeader(directType(), fieldsCount()))
- return false;
-
- writer.onHeaderWritten();
- }
-
- switch (writer.state()) {
- case 3:
- if (!writer.writeInt("part", part))
- return false;
-
- writer.incrementState();
-
- case 4:
- if (!writer.writeUuid("routineId", routineId))
- return false;
-
- writer.incrementState();
-
- }
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- reader.setBuffer(buf);
-
- if (!reader.beforeMessageRead())
- return false;
-
- if (!super.readFrom(buf, reader))
- return false;
-
- switch (reader.state()) {
- case 3:
- part = reader.readInt("part");
-
- if (!reader.isLastRead())
- return false;
-
- reader.incrementState();
-
- case 4:
- routineId = reader.readUuid("routineId");
-
- if (!reader.isLastRead())
- return false;
-
- }
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public byte fieldsCount() {
- return 5;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(CacheContinuousQueryLostPartition.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index d0d877d..7c04053 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -46,9 +46,6 @@ import org.apache.ignite.cache.CacheEntryEventSerializableFilter;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.query.ContinuousQuery;
import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.events.CacheRebalancingEvent;
-import org.apache.ignite.events.Event;
-import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
@@ -90,6 +87,8 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
/** */
private static final long BACKUP_ACK_FREQ = 5000;
+ public static final boolean SUPER_DEBUG = false;
+
/** Listeners. */
private final ConcurrentMap<UUID, CacheContinuousQueryListener> lsnrs = new ConcurrentHashMap8<>();
@@ -127,16 +126,6 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
}
});
- cctx.io().addHandler(cctx.cacheId(), CacheContinuousQueryLostPartition.class,
- new CI2<UUID, CacheContinuousQueryLostPartition>() {
- @Override public void apply(UUID uuid, CacheContinuousQueryLostPartition msg) {
- CacheContinuousQueryListener lsnr = lsnrs.get(msg.routineId());
-
- if (lsnr != null)
- lsnr.partitionLost(msg.partition());
- }
- });
-
cctx.time().schedule(new Runnable() {
@Override public void run() {
for (CacheContinuousQueryListener lsnr : lsnrs.values())
@@ -146,20 +135,6 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
lsnr.acknowledgeBackupOnTimeout(cctx.kernalContext());
}
}, BACKUP_ACK_FREQ, BACKUP_ACK_FREQ);
-
- cctx.kernalContext().event().addLocalEventListener(new GridLocalEventListener() {
- @Override public void onEvent(Event evt) {
- assert evt instanceof CacheRebalancingEvent;
-
- CacheRebalancingEvent evt0 = (CacheRebalancingEvent)evt;
-
- for (CacheContinuousQueryListener lsnr : lsnrs.values())
- lsnr.firePartitionLostEvent(evt0.cacheName(), evt0.partition());
-
- for (CacheContinuousQueryListener lsnr : intLsnrs.values())
- lsnr.firePartitionLostEvent(evt0.cacheName(), evt0.partition());
- }
- }, EVT_CACHE_REBALANCE_PART_DATA_LOST);
}
/** {@inheritDoc} */
@@ -319,7 +294,7 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
null,
lsnr.oldValueRequired() ? oldVal : null,
e.partition(),
- 0,
+ -1,
null);
CacheContinuousQueryEvent evt = new CacheContinuousQueryEvent(
@@ -568,7 +543,7 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
e.rawGet(),
null,
0,
- 0,
+ -1,
null);
next = new CacheContinuousQueryEvent<>(
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousHandler.java
index 975cd2f..40fb12a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousHandler.java
@@ -114,12 +114,6 @@ public interface GridContinuousHandler extends Externalizable, Cloneable {
public void onBatchAcknowledged(UUID routineId, GridContinuousBatch batch, GridKernalContext ctx);
/**
- * @param cacheName Cache name.
- * @param partId Partition ID.
- */
- public void partitionLost(String cacheName, int partId);
-
- /**
* @return Topic for ordered notifications. If {@code null}, notifications
* will be sent in non-ordered messages.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/df45134d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
index 61fa6cd..ca754af 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -31,7 +32,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -39,6 +39,8 @@ import javax.cache.Cache;
import javax.cache.event.CacheEntryEvent;
import javax.cache.event.CacheEntryListenerException;
import javax.cache.event.CacheEntryUpdatedListener;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
@@ -47,6 +49,7 @@ import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheAtomicWriteOrderMode;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheEntryEventSerializableFilter;
+import org.apache.ignite.cache.CacheEntryProcessor;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cache.query.ContinuousQuery;
@@ -71,10 +74,10 @@ import org.apache.ignite.internal.util.typedef.PAX;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.T3;
import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.resources.LoggerResource;
import org.apache.ignite.spi.IgniteSpiException;
-import org.apache.ignite.spi.communication.CommunicationSpi;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -279,6 +282,17 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
IgniteCache<Object, Object> clnCache = qryClient.cache(null);
+ IgniteOutClosure<IgniteCache<Integer, Integer>> rndCache =
+ new IgniteOutClosure<IgniteCache<Integer, Integer>>() {
+ int cnt = 0;
+
+ @Override public IgniteCache<Integer, Integer> apply() {
+ ++cnt;
+
+ return grid(cnt % SRV_NODES + 1).cache(null);
+ }
+ };
+
Ignite igniteSrv = ignite(0);
IgniteCache<Object, Object> srvCache = igniteSrv.cache(null);
@@ -290,16 +304,18 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
for (int j = 0; j < 50; ++j) {
ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
- final TestLocalListener lsnr = new TestLocalListener();
+ final CacheEventListener3 lsnr = new CacheEventListener3();
qry.setLocalListener(lsnr);
+ qry.setRemoteFilter(lsnr);
+
int keyIter = 0;
for (; keyIter < keyCnt / 2; keyIter++) {
int key = keys.get(keyIter);
- clnCache.put(key, key);
+ rndCache.apply().put(key, key);
}
assert lsnr.evts.isEmpty();
@@ -312,28 +328,40 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
Affinity<Object> aff = affinity(srvCache);
+ boolean filtered = false;
+
for (; keyIter < keys.size(); keyIter++) {
int key = keys.get(keyIter);
- log.info("Put [key=" + key + ", part=" + aff.partition(key) + ']');
+ int val = filtered ? 1 : 2;
+
+ log.info("Put [key=" + key + ", val=" + val + ", part=" + aff.partition(key) + ']');
T2<Object, Object> t = updates.get(key);
if (t == null) {
- updates.put(key, new T2<>((Object)key, null));
+ // Check filtered.
+ if (!filtered) {
+ updates.put(key, new T2<>((Object)val, null));
- expEvts.add(new T3<>((Object)key, (Object)key, null));
+ expEvts.add(new T3<>((Object)key, (Object)val, null));
+ }
}
else {
- updates.put(key, new T2<>((Object)key, (Object)key));
+ // Check filtered.
+ if (!filtered) {
+ updates.put(key, new T2<>((Object)val, (Object)t.get1()));
- expEvts.add(new T3<>((Object)key, (Object)key, (Object)key));
+ expEvts.add(new T3<>((Object)key, (Object)val, (Object)t.get1()));
+ }
}
- srvCache.put(key, key);
+ rndCache.apply().put(key, val);
+
+ filtered = !filtered;
}
- checkEvents(expEvts, lsnr);
+ checkEvents(expEvts, lsnr, false);
query.close();
}
@@ -357,7 +385,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
- final TestLocalListener lsnr = new TestLocalListener();
+ final CacheEventListener3 lsnr = new CacheEventListener3();
qry.setLocalListener(lsnr);
@@ -421,7 +449,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
filtered = !filtered;
}
- checkEvents(expEvts, lsnr);
+ checkEvents(expEvts, lsnr, false);
List<Thread> stopThreads = new ArrayList<>(3);
@@ -488,7 +516,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
filtered = !filtered;
}
- checkEvents(expEvts, lsnr);
+ checkEvents(expEvts, lsnr, false);
query.close();
}
@@ -518,7 +546,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
- final TestLocalListener lsnr = new TestLocalListener();
+ final CacheEventListener3 lsnr = new CacheEventListener3();
qry.setLocalListener(lsnr);
@@ -599,43 +627,13 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
fail("Failed to wait for notifications [exp=" + keys.size() + ", left=" + keys0.size() + ']');
}
- checkEvents(expEvts, lsnr);
+ checkEvents(expEvts, lsnr, false);
}
cur.close();
}
/**
- *
- */
- public static class TestLocalListener implements CacheEntryUpdatedListener<Object, Object>,
- CacheEntryEventSerializableFilter<Object, Object> {
- /** Keys. */
- GridConcurrentHashSet<Integer> keys = new GridConcurrentHashSet<>();
-
- /** Events. */
- private final ConcurrentHashMap<Object, CacheEntryEvent<?, ?>> evts = new ConcurrentHashMap<>();
-
- /** {@inheritDoc} */
- @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> events) throws CacheEntryListenerException {
- for (CacheEntryEvent<?, ?> e : events) {
- System.err.println("Update entry: " + e);
-
- Integer key = (Integer)e.getKey();
-
- keys.add(key);
-
- evts.put(key, e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean evaluate(CacheEntryEvent<?, ?> e) throws CacheEntryListenerException {
- return (Integer)e.getValue() % 2 == 0;
- }
- }
-
- /**
* @throws Exception If failed.
*/
public void testThreeBackups() throws Exception {
@@ -822,22 +820,145 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
/**
* @param expEvts Expected events.
* @param lsnr Listener.
+ * @param lostAllow If {@code true} than won't assert on lost events.
*/
- private void checkEvents(final List<T3<Object, Object, Object>> expEvts, final TestLocalListener lsnr)
- throws Exception {
- assert GridTestUtils.waitForCondition(new PA() {
+ private void checkEvents(final List<T3<Object, Object, Object>> expEvts, final CacheEventListener2 lsnr,
+ boolean lostAllow) throws Exception {
+ boolean b = GridTestUtils.waitForCondition(new PA() {
@Override public boolean apply() {
- return lsnr.evts.size() == expEvts.size();
+ return expEvts.size() == lsnr.size();
}
}, 2000L);
+ List<T3<Object, Object, Object>> lostEvents = new ArrayList<>();
+
+ for (T3<Object, Object, Object> exp : expEvts) {
+ List<CacheEntryEvent<?, ?>> rcvdEvts = lsnr.evts.get(exp.get1());
+
+ if (rcvdEvts == null || rcvdEvts.isEmpty()) {
+ lostEvents.add(exp);
+
+ continue;
+ }
+
+ Iterator<CacheEntryEvent<?, ?>> iter = rcvdEvts.iterator();
+
+ boolean found = false;
+
+ while (iter.hasNext()) {
+ CacheEntryEvent<?, ?> e = iter.next();
+
+ if ((exp.get2() != null && e.getValue() != null && exp.get2() == e.getValue())
+ && equalOldValue(e, exp)) {
+ found = true;
+
+ iter.remove();
+
+ break;
+ }
+ }
+
+ // Lost event is acceptable.
+ if (!found)
+ lostEvents.add(exp);
+ }
+
+ boolean dup = false;
+
+ // Check duplicate.
+ if (!lsnr.evts.isEmpty()) {
+ for (List<CacheEntryEvent<?, ?>> evts : lsnr.evts.values()) {
+ if (!evts.isEmpty()) {
+ for (CacheEntryEvent<?, ?> e : evts) {
+ boolean found = false;
+
+ for (T3<Object, Object, Object> lostEvt : lostEvents) {
+ if (e.getKey().equals(lostEvt.get1()) && e.getValue().equals(lostEvt.get2())
+ && equalOldValue(e, lostEvt)) {
+ found = true;
+
+ lostEvents.remove(lostEvt);
+
+ break;
+ }
+ }
+
+ if (!found) {
+ dup = true;
+
+ break;
+ }
+ }
+ }
+ }
+
+ if (dup) {
+ for (T3<Object, Object, Object> e : lostEvents)
+ log.error("Lost event: " + e);
+
+ for (List<CacheEntryEvent<?, ?>> e : lsnr.evts.values())
+ if (!e.isEmpty())
+ log.error("Duplicate event: " + e);
+ }
+
+ assertFalse("Received duplicate events, see log for details.", dup);
+ }
+
+ if (!lostAllow && !lostEvents.isEmpty()) {
+ log.error("Lost event cnt: " + lostEvents.size());
+
+ for (T3<Object, Object, Object> e : lostEvents)
+ log.error("Lost event: " + e);
+
+ assertTrue("Lose events, see log for details.", false);
+ }
+
+ log.error("Lost event cnt: " + lostEvents.size());
+
+ expEvts.clear();
+
+ lsnr.evts.clear();
+ }
+
+ /**
+ * @param e Event
+ * @param expVals expected value
+ * @return {@code True} if entries has the same key, value and oldValue. If cache start without backups
+ * than oldValue ignoring in comparison.
+ */
+ private boolean equalOldValue(CacheEntryEvent<?, ?> e, T3<Object, Object, Object> expVals) {
+ return (e.getOldValue() == null && expVals.get3() == null) // Both null
+ || (e.getOldValue() != null && expVals.get3() != null // Equals
+ && e.getOldValue().equals(expVals.get3()))
+ || (backups == 0); // If we start without backup than oldValue might be lose.
+ }
+
+ /**
+ * @param expEvts Expected events.
+ * @param lsnr Listener.
+ */
+ private void checkEvents(final List<T3<Object, Object, Object>> expEvts, final CacheEventListener3 lsnr,
+ boolean allowLoseEvent) throws Exception {
+ if (!allowLoseEvent)
+ assert GridTestUtils.waitForCondition(new PA() {
+ @Override public boolean apply() {
+ return lsnr.evts.size() == expEvts.size();
+ }
+ }, 2000L);
+
for (T3<Object, Object, Object> exp : expEvts) {
CacheEntryEvent<?, ?> e = lsnr.evts.get(exp.get1());
assertNotNull("No event for key: " + exp.get1(), e);
assertEquals("Unexpected value: " + e, exp.get2(), e.getValue());
+
+ if (allowLoseEvent)
+ lsnr.evts.remove(exp.get1());
}
+ if (allowLoseEvent)
+ assert lsnr.evts.isEmpty();
+
expEvts.clear();
lsnr.evts.clear();
@@ -1058,6 +1179,8 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
final AtomicReference<CountDownLatch> checkLatch = new AtomicReference<>();
+ boolean processorPut = false;
+
IgniteInternalFuture<?> restartFut = GridTestUtils.runAsync(new Callable<Void>() {
@Override public Void call() throws Exception {
final int idx = SRV_NODES + 1;
@@ -1093,7 +1216,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
final Map<Integer, List<T2<Integer, Integer>>> expEvts = new HashMap<>();
try {
- long stopTime = System.currentTimeMillis() + 3 * 60_000;
+ long stopTime = System.currentTimeMillis() + 1 * 60_000;
final int PARTS = qryClient.affinity(null).partitions();
@@ -1110,7 +1233,20 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
else
val = val + 1;
- qryClientCache.put(key, val);
+ if (processorPut && prevVal != null) {
+ qryClientCache.invoke(key, new CacheEntryProcessor<Object, Object, Void>() {
+ @Override public Void process(MutableEntry<Object, Object> entry,
+ Object... arguments) throws EntryProcessorException {
+ entry.setValue(arguments[0]);
+
+ return null;
+ }
+ }, val);
+ }
+ else
+ qryClientCache.put(key, val);
+
+ processorPut = !processorPut;
vals.put(key, val);
@@ -1187,8 +1323,8 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
/**
* @throws Exception If failed.
*/
- public void testMultiThreaded() throws Exception {
- final int SRV_NODES = 3;
+ public void testFailoverFilter() throws Exception {
+ final int SRV_NODES = 4;
startGridsMultiThreaded(SRV_NODES);
@@ -1196,142 +1332,633 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
Ignite qryClient = startGrid(SRV_NODES);
- final IgniteCache<Object, Object> cache = qryClient.cache(null);
+ client = false;
- CacheEventListener1 lsnr = new CacheEventListener1(true);
+ IgniteCache<Object, Object> qryClientCache = qryClient.cache(null);
+
+ final CacheEventListener2 lsnr = new CacheEventListener2();
ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
qry.setLocalListener(lsnr);
- QueryCursor<?> cur = cache.query(qry);
-
- client = false;
-
- final int SRV_IDX = SRV_NODES - 1;
+ qry.setRemoteFilter(new CacheEventFilter());
- List<Integer> keys = primaryKeys(ignite(SRV_IDX).cache(null), 10);
+ QueryCursor<?> cur = qryClientCache.query(qry);
- final int THREADS = 10;
+ final AtomicBoolean stop = new AtomicBoolean();
- for (int i = 0; i < keys.size(); i++) {
- log.info("Iteration: " + i);
+ final AtomicReference<CountDownLatch> checkLatch = new AtomicReference<>();
- Ignite srv = ignite(SRV_IDX);
+ IgniteInternalFuture<?> restartFut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ final int idx = SRV_NODES + 1;
- TestCommunicationSpi spi = (TestCommunicationSpi)srv.configuration().getCommunicationSpi();
+ while (!stop.get() && !err) {
+ log.info("Start node: " + idx);
- spi.sndFirstOnly = new AtomicBoolean(false);
+ startGrid(idx);
- final Integer key = keys.get(i);
+ Thread.sleep(3000);
- final AtomicInteger val = new AtomicInteger();
+ log.info("Stop node: " + idx);
- CountDownLatch latch = new CountDownLatch(THREADS);
+ stopGrid(idx);
- lsnr.latch = latch;
+ CountDownLatch latch = new CountDownLatch(1);
- IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
- @Override public Object call() throws Exception {
- Integer val0 = val.getAndIncrement();
+ assertTrue(checkLatch.compareAndSet(null, latch));
- cache.put(key, val0);
+ if (!stop.get()) {
+ log.info("Wait for event check.");
- return null;
+ assertTrue(latch.await(1, MINUTES));
+ }
}
- }, THREADS, "update-thread");
- fut.get();
+ return null;
+ }
+ });
- stopGrid(SRV_IDX);
+ final Map<Integer, Integer> vals = new HashMap<>();
- if (!latch.await(5, SECONDS))
- fail("Failed to wait for notifications [exp=" + THREADS + ", left=" + lsnr.latch.getCount() + ']');
+ final Map<Integer, List<T2<Integer, Integer>>> expEvts = new HashMap<>();
- assertEquals(THREADS, lsnr.allEvts.size());
+ try {
+ long stopTime = System.currentTimeMillis() + 1 * 60_000;
- Set<Integer> vals = new HashSet<>();
+ final int PARTS = qryClient.affinity(null).partitions();
- boolean err = false;
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
- for (CacheEntryEvent<?, ?> evt : lsnr.allEvts) {
- assertEquals(key, evt.getKey());
- assertNotNull(evt.getValue());
+ boolean filtered = false;
- if (!vals.add((Integer)evt.getValue())) {
- err = true;
+ boolean processorPut = false;
- log.info("Extra event: " + evt);
- }
- }
+ while (System.currentTimeMillis() < stopTime) {
+ Integer key = rnd.nextInt(PARTS);
- for (int v = 0; v < THREADS; v++) {
- if (!vals.contains(v)) {
- err = true;
+ Integer prevVal = vals.get(key);
+ Integer val = vals.get(key);
- log.info("Event for value not received: " + v);
- }
- }
+ if (val == null)
+ val = 0;
+ else
+ val = Math.abs(val) + 1;
- assertFalse("Invalid events, see log for details.", err);
+ if (filtered)
+ val = -val;
- lsnr.allEvts.clear();
+ if (processorPut && prevVal != null) {
+ qryClientCache.invoke(key, new CacheEntryProcessor<Object, Object, Void>() {
+ @Override public Void process(MutableEntry<Object, Object> entry,
+ Object... arguments) throws EntryProcessorException {
+ entry.setValue(arguments[0]);
- startGrid(SRV_IDX);
- }
+ return null;
+ }
+ }, val);
+ }
+ else
+ qryClientCache.put(key, val);
- cur.close();
- }
+ processorPut = !processorPut;
- /**
- * @param logAll If {@code true} logs all unexpected values.
- * @param expEvts Expected values.
- * @param lsnr Listener.
- * @return Check status.
- */
- @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
- private boolean checkEvents(boolean logAll,
- Map<Integer, List<T2<Integer, Integer>>> expEvts,
- CacheEventListener2 lsnr) {
- assertTrue(!expEvts.isEmpty());
+ vals.put(key, val);
- boolean pass = true;
+ if (val >= 0) {
+ List<T2<Integer, Integer>> keyEvts = expEvts.get(key);
- for (Map.Entry<Integer, List<T2<Integer, Integer>>> e : expEvts.entrySet()) {
- Integer key = e.getKey();
- List<T2<Integer, Integer>> exp = e.getValue();
+ if (keyEvts == null) {
+ keyEvts = new ArrayList<>();
- List<CacheEntryEvent<?, ?>> rcvdEvts = lsnr.evts.get(key);
+ expEvts.put(key, keyEvts);
+ }
- if (rcvdEvts == null) {
- pass = false;
+ keyEvts.add(new T2<>(val, prevVal));
+ }
- log.info("No events for key [key=" + key + ", exp=" + e.getValue() + ']');
+ filtered = !filtered;
- if (!logAll)
- return false;
- }
- else {
- synchronized (rcvdEvts) {
- if (rcvdEvts.size() != exp.size()) {
- pass = false;
+ CountDownLatch latch = checkLatch.get();
- log.info("Missed or extra events for key [key=" + key +
- ", exp=" + e.getValue() +
- ", rcvd=" + rcvdEvts + ']');
+ if (latch != null) {
+ log.info("Check events.");
- if (!logAll)
- return false;
- }
+ checkLatch.set(null);
- int cnt = Math.min(rcvdEvts.size(), exp.size());
+ boolean success = false;
- for (int i = 0; i < cnt; i++) {
- T2<Integer, Integer> expEvt = exp.get(i);
- CacheEntryEvent<?, ?> rcvdEvt = rcvdEvts.get(i);
+ try {
+ if (err)
+ break;
- assertEquals(key, rcvdEvt.getKey());
- assertEquals(expEvt.get1(), rcvdEvt.getValue());
+ boolean check = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return checkEvents(false, expEvts, lsnr);
+ }
+ }, 10_000);
+
+ if (!check)
+ assertTrue(checkEvents(true, expEvts, lsnr));
+
+ success = true;
+
+ log.info("Events checked.");
+ }
+ finally {
+ if (!success)
+ err = true;
+
+ latch.countDown();
+ }
+ }
+ }
+ }
+ finally {
+ stop.set(true);
+ }
+
+ CountDownLatch latch = checkLatch.get();
+
+ if (latch != null)
+ latch.countDown();
+
+ restartFut.get();
+
+ boolean check = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return checkEvents(false, expEvts, lsnr);
+ }
+ }, 10_000);
+
+ if (!check)
+ assertTrue(checkEvents(true, expEvts, lsnr));
+
+ cur.close();
+
+ assertFalse("Unexpected error during test, see log for details.", err);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testFailoverStartStopWithoutBackup() throws Exception {
+ failoverStartStopFilter(0);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testFailoverStartStopOneBackup() throws Exception {
+ failoverStartStopFilter(1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void _testStartStop() throws Exception {
+ this.backups = 0;
+
+ final int SRV_NODES = 4;
+
+ startGridsMultiThreaded(SRV_NODES);
+
+ client = true;
+
+ Ignite qryClient = startGrid(SRV_NODES);
+
+ client = false;
+
+ IgniteCache<Object, Object> qryClnCache = qryClient.cache(null);
+
+ final CacheEventListener2 lsnr = new CacheEventListener2();
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ qry.setRemoteFilter(new CacheEventFilter());
+
+ QueryCursor<?> cur = qryClnCache.query(qry);
+
+ for (int i = 0; i < 100; i++) {
+ final int idx = i % (SRV_NODES - 1);
+
+ log.info("Stop node: " + idx);
+
+ stopGrid(idx);
+
+ Thread.sleep(200);
+
+ List<T3<Object, Object, Object>> afterRestEvents = new ArrayList<>();
+
+ for (int j = 0; j < 10; j++) {
+ Integer oldVal = (Integer)qryClnCache.get(j);
+
+ qryClnCache.put(j, i);
+
+ afterRestEvents.add(new T3<>((Object)j, (Object)i, (Object)oldVal));
+ }
+
+ checkEvents(new ArrayList<>(afterRestEvents), lsnr, false);
+
+ log.info("Start node: " + idx);
+
+ startGrid(idx);
+ }
+
+ cur.close();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void failoverStartStopFilter(int backups) throws Exception {
+ this.backups = backups;
+
+ final int SRV_NODES = 4;
+
+ startGridsMultiThreaded(SRV_NODES);
+
+ client = true;
+
+ Ignite qryClient = startGrid(SRV_NODES);
+
+ client = false;
+
+ IgniteCache<Object, Object> qryClnCache = qryClient.cache(null);
+
+ final CacheEventListener2 lsnr = new CacheEventListener2();
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ qry.setRemoteFilter(new CacheEventFilter());
+
+ QueryCursor<?> cur = qryClnCache.query(qry);
+
+ CacheEventListener2 dinLsnr = null;
+
+ QueryCursor<?> dinQry = null;
+
+ final AtomicBoolean stop = new AtomicBoolean();
+
+ final AtomicReference<CountDownLatch> checkLatch = new AtomicReference<>();
+
+ IgniteInternalFuture<?> restartFut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while (!stop.get() && !err) {
+ final int idx = ThreadLocalRandom.current().nextInt(SRV_NODES - 1);
+
+ log.info("Stop node: " + idx);
+
+ stopGrid(idx);
+
+ Thread.sleep(100);
+
+ log.info("Start node: " + idx);
+
+ startGrid(idx);
+
+ CountDownLatch latch = new CountDownLatch(1);
+
+ assertTrue(checkLatch.compareAndSet(null, latch));
+
+ if (!stop.get()) {
+ log.info("Wait for event check.");
+
+ assertTrue(latch.await(1, MINUTES));
+ }
+ }
+
+ return null;
+ }
+ });
+
+ final Map<Integer, Integer> vals = new HashMap<>();
+
+ final Map<Integer, List<T2<Integer, Integer>>> expEvts = new HashMap<>();
+
+ final List<T3<Object, Object, Object>> expEvtsNewLsnr = new ArrayList<>();
+
+ final List<T3<Object, Object, Object>> expEvtsLsnr = new ArrayList<>();
+
+ try {
+ long stopTime = System.currentTimeMillis() + 60_000;
+
+ // Start new filter each 5 sec.
+ long startFilterTime = System.currentTimeMillis() + 5_000;
+
+ final int PARTS = qryClient.affinity(null).partitions();
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ boolean filtered = false;
+
+ boolean processorPut = false;
+
+ while (System.currentTimeMillis() < stopTime) {
+ Integer key = rnd.nextInt(PARTS);
+
+ Integer prevVal = vals.get(key);
+ Integer val = vals.get(key);
+
+ if (System.currentTimeMillis() > startFilterTime) {
+ // Stop filter and check events.
+ if (dinQry != null) {
+ dinQry.close();
+
+ log.error("Continuous query listener closed.");
+
+ checkEvents(expEvtsNewLsnr, dinLsnr, backups == 0);
+ }
+
+ dinLsnr = new CacheEventListener2();
+
+ ContinuousQuery<Object, Object> newQry = new ContinuousQuery<>();
+
+ newQry.setLocalListener(dinLsnr);
+
+ newQry.setRemoteFilter(new CacheEventFilter());
+
+ dinQry = qryClnCache.query(newQry);
+
+ log.error("Continuous query listener started.");
+
+ startFilterTime = System.currentTimeMillis() + 5_000;
+ }
+
+ if (val == null)
+ val = 0;
+ else
+ val = Math.abs(val) + 1;
+
+ if (filtered)
+ val = -val;
+
+ if (processorPut && prevVal != null) {
+ qryClnCache.invoke(key, new CacheEntryProcessor<Object, Object, Void>() {
+ @Override public Void process(MutableEntry<Object, Object> entry,
+ Object... arguments) throws EntryProcessorException {
+ entry.setValue(arguments[0]);
+
+ return null;
+ }
+ }, val);
+ }
+ else
+ qryClnCache.put(key, val);
+
+ processorPut = !processorPut;
+
+ vals.put(key, val);
+
+ if (val >= 0) {
+ List<T2<Integer, Integer>> keyEvts = expEvts.get(key);
+
+ if (keyEvts == null) {
+ keyEvts = new ArrayList<>();
+
+ expEvts.put(key, keyEvts);
+ }
+
+ keyEvts.add(new T2<>(val, prevVal));
+
+ T3<Object, Object, Object> tupVal = new T3<>((Object)key, (Object)val, (Object)prevVal);
+
+ expEvtsLsnr.add(tupVal);
+
+ if (dinQry != null)
+ expEvtsNewLsnr.add(tupVal);
+ }
+
+ filtered = !filtered;
+
+ CountDownLatch latch = checkLatch.get();
+
+ if (latch != null) {
+ log.info("Check events.");
+
+ checkLatch.set(null);
+
+ boolean success = false;
+
+ try {
+ if (err)
+ break;
+
+ checkEvents(expEvtsLsnr, lsnr, backups == 0);
+
+ success = true;
+
+ log.info("Events checked.");
+ }
+ finally {
+ if (!success)
+ err = true;
+
+ latch.countDown();
+ }
+ }
+ }
+ }
+ finally {
+ stop.set(true);
+ }
+
+ CountDownLatch latch = checkLatch.get();
+
+ if (latch != null)
+ latch.countDown();
+
+ restartFut.get();
+
+ checkEvents(expEvtsLsnr, lsnr, backups == 0);
+
+ lsnr.evts.clear();
+ lsnr.vals.clear();
+
+ if (dinQry != null) {
+ checkEvents(expEvtsNewLsnr, dinLsnr, backups == 0);
+
+ dinLsnr.evts.clear();
+ dinLsnr.vals.clear();
+
+ dinQry.close();
+ }
+
+ List<T3<Object, Object, Object>> afterRestEvents = new ArrayList<>();
+
+ for (int i = 0; i < 1024; i++) {
+ Integer oldVal = (Integer)qryClnCache.get(i);
+
+ qryClnCache.put(i, i);
+
+ afterRestEvents.add(new T3<>((Object)i, (Object)i, (Object)oldVal));
+ }
+
+ checkEvents(new ArrayList<>(afterRestEvents), lsnr, false);
+
+ //checkEvents(new ArrayList<>(afterRestEvents), dinLsnr, false);
+
+ cur.close();
+
+ if (dinQry != null)
+ dinQry.close();
+
+ assertFalse("Unexpected error during test, see log for details.", err);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMultiThreaded() throws Exception {
+ final int SRV_NODES = 3;
+
+ startGridsMultiThreaded(SRV_NODES);
+
+ client = true;
+
+ Ignite qryClient = startGrid(SRV_NODES);
+
+ final IgniteCache<Object, Object> cache = qryClient.cache(null);
+
+ CacheEventListener1 lsnr = new CacheEventListener1(true);
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ QueryCursor<?> cur = cache.query(qry);
+
+ client = false;
+
+ final int SRV_IDX = SRV_NODES - 1;
+
+ List<Integer> keys = primaryKeys(ignite(SRV_IDX).cache(null), 10);
+
+ final int THREADS = 10;
+
+ for (int i = 0; i < keys.size(); i++) {
+ log.info("Iteration: " + i);
+
+ Ignite srv = ignite(SRV_IDX);
+
+ TestCommunicationSpi spi = (TestCommunicationSpi)srv.configuration().getCommunicationSpi();
+
+ spi.sndFirstOnly = new AtomicBoolean(false);
+
+ final Integer key = keys.get(i);
+
+ final AtomicInteger val = new AtomicInteger();
+
+ CountDownLatch latch = new CountDownLatch(THREADS);
+
+ lsnr.latch = latch;
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ Integer val0 = val.getAndIncrement();
+
+ cache.put(key, val0);
+
+ return null;
+ }
+ }, THREADS, "update-thread");
+
+ fut.get();
+
+ stopGrid(SRV_IDX);
+
+ if (!latch.await(5, SECONDS))
+ fail("Failed to wait for notifications [exp=" + THREADS + ", left=" + lsnr.latch.getCount() + ']');
+
+ assertEquals(THREADS, lsnr.allEvts.size());
+
+ Set<Integer> vals = new HashSet<>();
+
+ boolean err = false;
+
+ for (CacheEntryEvent<?, ?> evt : lsnr.allEvts) {
+ assertEquals(key, evt.getKey());
+ assertNotNull(evt.getValue());
+
+ if (!vals.add((Integer)evt.getValue())) {
+ err = true;
+
+ log.info("Extra event: " + evt);
+ }
+ }
+
+ for (int v = 0; v < THREADS; v++) {
+ if (!vals.contains(v)) {
+ err = true;
+
+ log.info("Event for value not received: " + v);
+ }
+ }
+
+ assertFalse("Invalid events, see log for details.", err);
+
+ lsnr.allEvts.clear();
+
+ startGrid(SRV_IDX);
+ }
+
+ cur.close();
+ }
+
+ /**
+ * @param logAll If {@code true} logs all unexpected values.
+ * @param expEvts Expected values.
+ * @param lsnr Listener.
+ * @return Check status.
+ */
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+ private boolean checkEvents(boolean logAll,
+ Map<Integer, List<T2<Integer, Integer>>> expEvts,
+ CacheEventListener2 lsnr) {
+ assertTrue(!expEvts.isEmpty());
+
+ boolean pass = true;
+
+ for (Map.Entry<Integer, List<T2<Integer, Integer>>> e : expEvts.entrySet()) {
+ Integer key = e.getKey();
+ List<T2<Integer, Integer>> exp = e.getValue();
+
+ List<CacheEntryEvent<?, ?>> rcvdEvts = lsnr.evts.get(key);
+
+ if (rcvdEvts == null) {
+ pass = false;
+
+ log.info("No events for key [key=" + key + ", exp=" + e.getValue() + ']');
+
+ if (!logAll)
+ return false;
+ }
+ else {
+ synchronized (rcvdEvts) {
+ if (rcvdEvts.size() != exp.size()) {
+ pass = false;
+
+ log.info("Missed or extra events for key [key=" + key +
+ ", exp=" + e.getValue() +
+ ", rcvd=" + rcvdEvts + ']');
+
+ if (!logAll)
+ return false;
+ }
+
+ int cnt = Math.min(rcvdEvts.size(), exp.size());
+
+ for (int i = 0; i < cnt; i++) {
+ T2<Integer, Integer> expEvt = exp.get(i);
+ CacheEntryEvent<?, ?> rcvdEvt = rcvdEvts.get(i);
+
+ assertEquals(key, rcvdEvt.getKey());
+ assertEquals(expEvt.get1(), rcvdEvt.getValue());
}
}
}
@@ -1384,7 +2011,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
this.evts.put(evt.getKey(), evt);
- keys.add((Integer) evt.getKey());
+ keys.add((Integer)evt.getKey());
if (allEvts != null)
allEvts.add(evt);
@@ -1423,6 +2050,18 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
/** */
private final ConcurrentHashMap<Integer, List<CacheEntryEvent<?, ?>>> evts = new ConcurrentHashMap<>();
+ /**
+ * @return Count events.
+ */
+ public int size() {
+ int size = 0;
+
+ for (List<CacheEntryEvent<?, ?>> e : evts.values())
+ size += e.size();
+
+ return size;
+ }
+
/** {@inheritDoc} */
@Override public synchronized void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts)
throws CacheEntryListenerException {
@@ -1467,6 +2106,44 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
/**
*
*/
+ public static class CacheEventListener3 implements CacheEntryUpdatedListener<Object, Object>,
+ CacheEntryEventSerializableFilter<Object, Object> {
+ /** Keys. */
+ GridConcurrentHashSet<Integer> keys = new GridConcurrentHashSet<>();
+
+ /** Events. */
+ private final ConcurrentHashMap<Object, CacheEntryEvent<?, ?>> evts = new ConcurrentHashMap<>();
+
+ /** {@inheritDoc} */
+ @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> events) throws CacheEntryListenerException {
+ for (CacheEntryEvent<?, ?> e : events) {
+ Integer key = (Integer)e.getKey();
+
+ keys.add(key);
+
+ assert evts.put(key, e) == null;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean evaluate(CacheEntryEvent<?, ?> e) throws CacheEntryListenerException {
+ return (Integer)e.getValue() % 2 == 0;
+ }
+ }
+
+ /**
+ *
+ */
+ public static class CacheEventFilter implements CacheEntryEventSerializableFilter<Object, Object> {
+ /** {@inheritDoc} */
+ @Override public boolean evaluate(CacheEntryEvent<?, ?> event) throws CacheEntryListenerException {
+ return ((Integer)event.getValue()) >= 0;
+ }
+ }
+
+ /**
+ *
+ */
private static class TestCommunicationSpi extends TcpCommunicationSpi {
/** */
@LoggerResource
[3/9] ignite git commit: IGNITE-426 WIP
Posted by nt...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 3253dda..ca3579e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -21,13 +21,19 @@ import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import javax.cache.event.CacheEntryEvent;
import javax.cache.event.CacheEntryUpdatedListener;
import javax.cache.event.EventType;
@@ -53,6 +59,7 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousBatch;
import org.apache.ignite.internal.processors.continuous.GridContinuousBatchAdapter;
import org.apache.ignite.internal.processors.continuous.GridContinuousHandler;
import org.apache.ignite.internal.processors.platform.cache.query.PlatformContinuousQueryFilter;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.C1;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -119,6 +126,9 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
private transient Map<Integer, Long> rcvCntrs;
/** */
+ private transient ConcurrentMap<Integer, PartitionRecovery> rcvs;
+
+ /** */
private transient IgnitePredicate<CacheContinuousQueryEntry> dupEvtFilter;
/** */
@@ -183,6 +193,8 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
else {
rcvCntrs = new ConcurrentHashMap<>();
+ rcvs = new ConcurrentHashMap<>();
+
dupEvtFilter = new DuplicateEventFilter();
}
@@ -258,6 +270,10 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
GridCacheContext<K, V> cctx = cacheContext(ctx);
+ // Check that cache stopped.
+ if (cctx == null)
+ return;
+
// skipPrimaryCheck is set only when listen locally for replicated cache events.
assert !skipPrimaryCheck || (cctx.isReplicated() && ctx.localNodeId().equals(nodeId));
@@ -272,27 +288,78 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
}
- if (notify) {
- try {
- final CacheContinuousQueryEntry entry = evt.entry();
+ try {
+ final CacheContinuousQueryEntry entry = notify ? evt.entry() :
+ new CacheContinuousQueryFilteredEntry(evt.entry());
- if (primary || skipPrimaryCheck) {
- if (loc) {
- if (dupEvtFilter.apply(entry)) {
- locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
+ if (primary || skipPrimaryCheck) {
+ if (loc) {
+ if (dupEvtFilter.apply(entry)) {
+ locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
- if (!skipPrimaryCheck)
- sendBackupAcknowledge(ackBuf.onAcknowledged(entry), routineId, ctx);
- }
+ if (!skipPrimaryCheck)
+ sendBackupAcknowledge(ackBuf.onAcknowledged(entry), routineId, ctx);
}
- else {
- prepareEntry(cctx, nodeId, entry);
+ }
+ else {
+ prepareEntry(cctx, nodeId, entry);
- ctx.continuous().addNotification(nodeId, routineId, entry, topic, sync, true);
- }
+ ctx.continuous().addNotification(nodeId, routineId, entry, topic, sync, true);
}
- else
- backupQueue.add(entry);
+ }
+ else
+ backupQueue.add(entry);
+ }
+ catch (ClusterTopologyCheckedException ex) {
+ IgniteLogger log = ctx.log(getClass());
+
+ if (log.isDebugEnabled())
+ log.debug("Failed to send event notification to node, node left cluster " +
+ "[node=" + nodeId + ", err=" + ex + ']');
+ }
+ catch (IgniteCheckedException ex) {
+ U.error(ctx.log(getClass()), "Failed to send event notification to node: " + nodeId, ex);
+ }
+
+ if (recordIgniteEvt && notify) {
+ ctx.event().record(new CacheQueryReadEvent<>(
+ ctx.discovery().localNode(),
+ "Continuous query executed.",
+ EVT_CACHE_QUERY_OBJECT_READ,
+ CacheQueryType.CONTINUOUS.name(),
+ cacheName,
+ null,
+ null,
+ null,
+ rmtFilter,
+ null,
+ nodeId,
+ taskName(),
+ evt.getKey(),
+ evt.getValue(),
+ evt.getOldValue(),
+ null
+ ));
+ }
+ }
+
+ @Override public void partitionLost(String cacheName0, int partId) {
+ GridCacheContext<K, V> cctx = cacheContext(ctx);
+
+ // Check that cache stopped.
+ if (cctx == null)
+ return;
+
+ if ((cacheName == null && cacheName0 == null) || // Check default cache.
+ (cacheName0 != null && cacheName != null && cacheName0.equals(cacheName))) {
+
+ final CacheContinuousQueryEntry entry =
+ new CacheContinuousQueryLostPartition(cctx.cacheId(), partId);
+
+ try {
+ prepareEntry(cctx, nodeId, entry);
+
+ ctx.continuous().addNotification(nodeId, routineId, entry, topic, sync, true);
}
catch (ClusterTopologyCheckedException ex) {
IgniteLogger log = ctx.log(getClass());
@@ -304,27 +371,6 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
catch (IgniteCheckedException ex) {
U.error(ctx.log(getClass()), "Failed to send event notification to node: " + nodeId, ex);
}
-
- if (recordIgniteEvt) {
- ctx.event().record(new CacheQueryReadEvent<>(
- ctx.discovery().localNode(),
- "Continuous query executed.",
- EVT_CACHE_QUERY_OBJECT_READ,
- CacheQueryType.CONTINUOUS.name(),
- cacheName,
- null,
- null,
- null,
- rmtFilter,
- null,
- nodeId,
- taskName(),
- evt.getKey(),
- evt.getValue(),
- evt.getOldValue(),
- null
- ));
- }
}
}
@@ -476,13 +522,47 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
final IgniteCache cache = cctx.kernalContext().cache().jcache(cctx.name());
- Iterable<CacheEntryEvent<? extends K, ? extends V>> evts = F.viewReadOnly(entries,
+ Map<Integer, PartitionRecovery> parts = new HashMap<>();
+
+ for (CacheContinuousQueryEntry e : entries) {
+ PartitionRecovery rec = parts.containsKey(e.partition()) ?
+ parts.get(e.partition()) : rcvs.get(e.partition());
+
+ if (rec == null) {
+ rec = new PartitionRecovery(ctx.log(getClass()));
+
+ PartitionRecovery oldRec = rcvs.putIfAbsent(e.partition(), rec);
+
+ if (oldRec != null)
+ rec = oldRec;
+ }
+
+ if (e instanceof CacheContinuousQueryLostPartition)
+ rec.reset();
+ else {
+ rec.add(e);
+
+ if (!parts.containsKey(e.partition()))
+ parts.put(e.partition(), rec);
+ }
+ }
+
+ Collection<CacheContinuousQueryEntry> entries0 = new ArrayList<>();
+
+ for (PartitionRecovery rec : parts.values())
+ entries0.addAll(rec.entries());
+
+ Iterable<CacheEntryEvent<? extends K, ? extends V>> evts = F.viewReadOnly(entries0,
new C1<CacheContinuousQueryEntry, CacheEntryEvent<? extends K, ? extends V>>() {
@Override public CacheEntryEvent<? extends K, ? extends V> apply(CacheContinuousQueryEntry e) {
return new CacheContinuousQueryEvent<>(cache, cctx, e);
}
},
- dupEvtFilter
+ new IgnitePredicate<CacheContinuousQueryEntry>() {
+ @Override public boolean apply(CacheContinuousQueryEntry entry) {
+ return !entry.filtered();
+ }
+ }
);
locLsnr.onUpdated(evts);
@@ -500,12 +580,8 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
if (cntr != null) {
long cntr0 = cntr;
- if (e.updateIndex() > cntr0) {
- // TODO IGNITE-426: remove assert.
- assert e.updateIndex() == cntr0 + 1 : "Invalid entry [cntr=" + cntr + ", e=" + e + ']';
-
+ if (e.updateIndex() > cntr0)
rcvCntrs.put(part, e.updateIndex());
- }
else
return false;
}
@@ -515,6 +591,119 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
return true;
}
+ /**
+ *
+ */
+ private static class PartitionRecovery {
+ /** */
+ private IgniteLogger log;
+
+ /** */
+ private long lastFiredEvt = 0;
+
+ /** */
+ private final Map<Long, CacheContinuousQueryEntry> pendingEnts = new TreeMap<>();
+
+ /**
+ * @param log Logger.
+ */
+ public PartitionRecovery(IgniteLogger log) {
+ this.log = log;
+ }
+
+ /**
+ * Add continuous entry.
+ *
+ * @param e Cache continuous qeury entry.
+ */
+ public void add(CacheContinuousQueryEntry e) {
+ synchronized (pendingEnts) {
+ if (pendingEnts.containsKey(e.updateIndex()) || e.updateIndex() <= lastFiredEvt)
+ e.cacheId();
+ //log.info("Skip duplicate continuous query entry. Entry: " + e);
+ else {
+ //log.info("Added continuous query entry. Entry: " + e);
+
+ pendingEnts.put(e.updateIndex(), e);
+ }
+ }
+ }
+
+ /**
+ * @return Ordered continuous query entries.
+ */
+ public Collection<CacheContinuousQueryEntry> entries() {
+ List<CacheContinuousQueryEntry> entries = new ArrayList<>();
+
+ synchronized (pendingEnts) {
+ if (pendingEnts.isEmpty())
+ return Collections.emptyList();
+
+ Iterator<Map.Entry<Long, CacheContinuousQueryEntry>> iter = pendingEnts.entrySet().iterator();
+
+ Map.Entry<Long, CacheContinuousQueryEntry> prev = null;
+
+ Set<Long> rmvEnts = new HashSet<>();
+
+ while (iter.hasNext()) {
+ Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
+
+ // The elements are consistently.
+ if (e.getKey() == lastFiredEvt + 1) {
+ ++lastFiredEvt;
+
+ entries.add(e.getValue());
+
+ iter.remove();
+ }
+ // Handle hole in sequence.
+ else if (prev != null && prev.getKey() + 1 == e.getKey()) {
+ entries.add(prev.getValue());
+
+ lastFiredEvt = prev.getKey();
+
+ rmvEnts.add(prev.getKey());
+
+ if (!iter.hasNext()) {
+ entries.add(e.getValue());
+
+ lastFiredEvt = e.getKey();
+
+ rmvEnts.add(e.getKey());
+ }
+ }
+ else if (prev != null)
+ break;
+
+ prev = e;
+ }
+
+ for (Long rmKey : rmvEnts)
+ pendingEnts.remove(rmKey);
+ }
+
+ return entries;
+ }
+
+ /**
+ * Reset internal state.
+ */
+ public void reset() {
+ synchronized (pendingEnts) {
+ Iterator<Map.Entry<Long, CacheContinuousQueryEntry>> iter = pendingEnts.entrySet().iterator();
+
+ while (iter.hasNext()) {
+ Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
+
+ if (e.getKey() >= lastFiredEvt)
+ iter.remove();
+ }
+
+ lastFiredEvt = 0;
+ }
+ }
+ }
+
/** {@inheritDoc} */
@Override public void p2pMarshal(GridKernalContext ctx) throws IgniteCheckedException {
assert ctx != null;
@@ -546,6 +735,23 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
sendBackupAcknowledge(ackBuf.onAcknowledged(batch), routineId, ctx);
}
+ /** {@inheritDoc} */
+ @Override public void partitionLost(String cacheName, int partId) {
+ if (this.cacheName == null) {
+ int z = 0;
+
+ ++z;
+ }
+
+ if ((this.cacheName == null && cacheName == null) // Check default caches.
+ || (cacheName != null && this.cacheName != null && cacheName.equals(this.cacheName))) {
+ PartitionRecovery rcv = rcvs.get(partId);
+
+ if (rcv != null)
+ rcv.reset();
+ }
+ }
+
/**
* @param t Acknowledge information.
* @param routineId Routine ID.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
index 2f9e111..735e808 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
@@ -55,6 +55,14 @@ interface CacheContinuousQueryListener<K, V> {
public void cleanupBackupQueue(Map<Integer, Long> updateIdxs);
/**
+ * Fire event that partition lost.
+ *
+ * @param cacheName Cache name.
+ * @param partId Partition ID.
+ */
+ public void partitionLost(String cacheName, int partId);
+
+ /**
* Flushes backup queue.
*
* @param ctx Context.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java
new file mode 100644
index 0000000..734d072
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.nio.ByteBuffer;
+import javax.cache.event.EventType;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Continuous query entry.
+ */
+public class CacheContinuousQueryLostPartition extends CacheContinuousQueryEntry {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Cache name. */
+ private int cacheId;
+
+ /** Partition. */
+ private int part;
+
+ /**
+ * Required by {@link Message}.
+ */
+ public CacheContinuousQueryLostPartition() {
+ // No-op.
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param part Partition ID.
+ */
+ CacheContinuousQueryLostPartition(int cacheId, int part) {
+ this.cacheId = cacheId;
+ this.part = part;
+ }
+
+ /**
+ * @return Cache ID.
+ */
+ int cacheId() {
+ return cacheId;
+ }
+
+ /**
+ * @return Partition.
+ */
+ int partition() {
+ return part;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte directType() {
+ return 116;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeInt("cacheId", cacheId))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeInt("part", part))
+ return false;
+
+ writer.incrementState();
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ cacheId = reader.readInt("cacheId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ part = reader.readInt("part");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+ }
+
+ return reader.afterMessageRead(CacheContinuousQueryLostPartition.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override void prepareMarshal(GridCacheContext cctx) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override void unmarshal(GridCacheContext cctx, @Nullable ClassLoader ldr) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(CacheContinuousQueryLostPartition.class, this);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index c9fb656..35c6696 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -46,6 +46,10 @@ import org.apache.ignite.cache.CacheEntryEventSerializableFilter;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.query.ContinuousQuery;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.cluster.ClusterTopologyException;
+import org.apache.ignite.events.CacheRebalancingEvent;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
@@ -65,6 +69,7 @@ import static javax.cache.event.EventType.EXPIRED;
import static javax.cache.event.EventType.REMOVED;
import static javax.cache.event.EventType.UPDATED;
import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_OBJECT_READ;
+import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE;
/**
@@ -132,6 +137,20 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
lsnr.acknowledgeBackupOnTimeout(cctx.kernalContext());
}
}, BACKUP_ACK_FREQ, BACKUP_ACK_FREQ);
+
+ cctx.kernalContext().event().addLocalEventListener(new GridLocalEventListener() {
+ @Override public void onEvent(Event evt) {
+ assert evt instanceof CacheRebalancingEvent;
+
+ CacheRebalancingEvent evt0 = (CacheRebalancingEvent)evt;
+
+ for (CacheContinuousQueryListener lsnr : lsnrs.values())
+ lsnr.partitionLost(evt0.cacheName(), evt0.partition());
+
+ for (CacheContinuousQueryListener lsnr : intLsnrs.values())
+ lsnr.partitionLost(evt0.cacheName(), evt0.partition());
+ }
+ }, EVT_CACHE_REBALANCE_PART_DATA_LOST);
}
/** {@inheritDoc} */
@@ -666,7 +685,8 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
fltr = (CacheEntryEventFilter) cfg.getCacheEntryEventFilterFactory().create();
if (!(fltr instanceof Serializable))
- throw new IgniteCheckedException("Cache entry event filter must implement java.io.Serializable: " + fltr);
+ throw new IgniteCheckedException("Cache entry event filter must implement java.io.Serializable: "
+ + fltr);
}
CacheEntryEventSerializableFilter rmtFilter = new JCacheQueryRemoteFilter(fltr, types);
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatch.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatch.java
index 2fef161..67b8c82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatch.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatch.java
@@ -1,7 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.ignite.internal.processors.continuous;
+import java.util.Collection;
+
/**
- * Created by Nikolay on 02.09.2015.
+ * Continuous routine batch.
*/
public interface GridContinuousBatch {
+ /**
+ * Adds element to this batch.
+ *
+ * @param obj Element to add.
+ */
+ public void add(Object obj);
+
+ /**
+ * Collects elements that are currently in this batch.
+ *
+ * @return Elements in this batch.
+ */
+ public Collection<Object> collect();
+
+ /**
+ * @return Current batch size.
+ */
+ public int size();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatchAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatchAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatchAdapter.java
index 8e29e29..4540de1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatchAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousBatchAdapter.java
@@ -1,7 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.ignite.internal.processors.continuous;
+import java.util.Collection;
+import org.jsr166.ConcurrentLinkedDeque8;
+
/**
- * Created by Nikolay on 02.09.2015.
+ * Continuous routine batch adapter.
*/
-public class GridContinuousBatchAdapter {
+public class GridContinuousBatchAdapter implements GridContinuousBatch {
+ /** Buffer. */
+ private final ConcurrentLinkedDeque8<Object> buf = new ConcurrentLinkedDeque8<>();
+
+ /** {@inheritDoc} */
+ @Override public void add(Object obj) {
+ assert obj != null;
+
+ buf.add(obj);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<Object> collect() {
+ return buf;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int size() {
+ return buf.sizex();
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousHandler.java
index 30e596a..975cd2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousHandler.java
@@ -98,6 +98,28 @@ public interface GridContinuousHandler extends Externalizable, Cloneable {
public void p2pUnmarshal(UUID nodeId, GridKernalContext ctx) throws IgniteCheckedException;
/**
+ * Creates new batch.
+ *
+ * @return New batch.
+ */
+ public GridContinuousBatch createBatch();
+
+ /**
+ * Called when ack for a batch is received from client.
+ *
+ * @param routineId Routine ID.
+ * @param batch Acknowledged batch.
+ * @param ctx Kernal context.
+ */
+ public void onBatchAcknowledged(UUID routineId, GridContinuousBatch batch, GridKernalContext ctx);
+
+ /**
+ * @param cacheName Cache name.
+ * @param partId Partition ID.
+ */
+ public void partitionLost(String cacheName, int partId);
+
+ /**
* @return Topic for ordered notifications. If {@code null}, notifications
* will be sent in non-ordered messages.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 15c9dd2..c7676d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -35,7 +35,9 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.events.CacheRebalancingEvent;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.GridKernalContext;
@@ -57,12 +59,14 @@ import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.util.worker.GridWorker;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.marshaller.Marshaller;
@@ -72,6 +76,7 @@ import org.jetbrains.annotations.Nullable;
import org.jsr166.ConcurrentHashMap8;
import org.jsr166.ConcurrentLinkedDeque8;
+import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 1fef4d5..26d1f91 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -507,7 +507,9 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
boolean conflictResolve,
boolean intercept,
UUID subjId,
- String taskName) throws IgniteCheckedException,
+ String taskName,
+ @Nullable CacheObject prevVal,
+ @Nullable Long updateIdx) throws IgniteCheckedException,
GridCacheEntryRemovedException {
assert false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
index ed856a5..3bba5e6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
@@ -25,23 +25,28 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import javax.cache.Cache;
import javax.cache.event.CacheEntryEvent;
import javax.cache.event.CacheEntryListenerException;
import javax.cache.event.CacheEntryUpdatedListener;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheAtomicWriteOrderMode;
import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheEntryEventSerializableFilter;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cache.query.ContinuousQuery;
@@ -59,12 +64,17 @@ import org.apache.ignite.internal.processors.continuous.GridContinuousMessage;
import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.C1;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.internal.util.typedef.PAX;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.T3;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.resources.LoggerResource;
import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.CommunicationSpi;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -254,6 +264,302 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
/**
* @throws Exception If failed.
*/
+ public void testLeftPrimaryAndBackupNodes() throws Exception {
+ this.backups = 1;
+
+ final int SRV_NODES = 3;
+
+ startGridsMultiThreaded(SRV_NODES);
+
+ client = true;
+
+ final Ignite qryClient = startGrid(SRV_NODES);
+
+ client = false;
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ final TestLocalListener lsnr = new TestLocalListener();
+
+ qry.setLocalListener(lsnr);
+
+ qry.setRemoteFilter(lsnr);
+
+ IgniteCache<Object, Object> clnCache = qryClient.cache(null);
+
+ QueryCursor<Cache.Entry<Object, Object>> query = clnCache.query(qry);
+
+ Ignite igniteSrv = ignite(0);
+
+ IgniteCache<Object, Object> srvCache = igniteSrv.cache(null);
+
+ Affinity<Object> aff = affinity(srvCache);
+
+ List<Integer> keys = testKeys(srvCache, 1);
+
+ Collection<ClusterNode> nodes = aff.mapPartitionToPrimaryAndBackups(keys.get(0));
+
+ Collection<UUID> ids = F.transform(nodes, new C1<ClusterNode, UUID>() {
+ @Override public UUID apply(ClusterNode node) {
+ return node.id();
+ }
+ });
+
+ int keyIter = 0;
+
+ boolean filtered = false;
+
+ Map<Object, T2<Object, Object>> updates = new HashMap<>();
+
+ final List<T3<Object, Object, Object>> expEvts = new ArrayList<>();
+
+ for (; keyIter < keys.size() / 2; keyIter++) {
+ int key = keys.get(keyIter);
+
+ log.info("Put [key=" + key + ", part=" + aff.partition(key)
+ + ", filtered=" + filtered + ']');
+
+ T2<Object, Object> t = updates.get(key);
+
+ Integer val = filtered ?
+ (key % 2 == 0 ? key + 1 : key) :
+ key * 2;
+
+ if (t == null) {
+ updates.put(key, new T2<>((Object)val, null));
+
+ if (!filtered)
+ expEvts.add(new T3<>((Object)key, (Object)val, null));
+ }
+ else {
+ updates.put(key, new T2<>((Object)val, (Object)key));
+
+ if (!filtered)
+ expEvts.add(new T3<>((Object)key, (Object)val, (Object)key));
+ }
+
+ srvCache.put(key, val);
+
+ filtered = !filtered;
+ }
+
+ checkEvents(expEvts, lsnr);
+
+ List<Thread> stopThreads = new ArrayList<>(3);
+
+ // Stop nodes which owning this partition.
+ for (int i = 0; i < SRV_NODES; i++) {
+ Ignite ignite = ignite(i);
+
+ if (ids.contains(ignite.cluster().localNode().id())) {
+ final int i0 = i;
+
+ TestCommunicationSpi spi = (TestCommunicationSpi)ignite.configuration().getCommunicationSpi();
+
+ spi.skipAllMsg = true;
+
+ stopThreads.add(new Thread() {
+ @Override public void run() {
+ stopGrid(i0, true);
+ }
+ });
+ }
+ }
+
+ // Stop and join threads.
+ for (Thread t : stopThreads)
+ t.start();
+
+ for (Thread t : stopThreads)
+ t.join();
+
+ assert GridTestUtils.waitForCondition(new PA() {
+ @Override public boolean apply() {
+ // (SRV_NODES + 1 client node) - 1 primary - backup nodes.
+ return qryClient.cluster().nodes().size() == (SRV_NODES + 1 /** client node */)
+ - 1 /** Primary node */ - backups;
+ }
+ }, 10000L);
+
+ for (; keyIter < keys.size(); keyIter++) {
+ int key = keys.get(keyIter);
+
+ log.info("Put [key=" + key + ", filtered=" + filtered + ']');
+
+ T2<Object, Object> t = updates.get(key);
+
+ Integer val = filtered ?
+ (key % 2 == 0 ? key + 1 : key) :
+ key * 2;
+
+ if (t == null) {
+ updates.put(key, new T2<>((Object)val, null));
+
+ if (!filtered)
+ expEvts.add(new T3<>((Object)key, (Object)val, null));
+ }
+ else {
+ updates.put(key, new T2<>((Object)val, (Object)key));
+
+ if (!filtered)
+ expEvts.add(new T3<>((Object)key, (Object)val, (Object)key));
+ }
+
+ clnCache.put(key, val);
+
+ filtered = !filtered;
+ }
+
+ checkEvents(expEvts, lsnr);
+
+ query.close();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRemoteFilter() throws Exception {
+ this.backups = 2;
+
+ final int SRV_NODES = 4;
+
+ startGridsMultiThreaded(SRV_NODES);
+
+ client = true;
+
+ Ignite qryClient = startGrid(SRV_NODES);
+
+ client = false;
+
+ IgniteCache<Object, Object> qryClientCache = qryClient.cache(null);
+
+ if (cacheMode() != REPLICATED)
+ assertEquals(backups, qryClientCache.getConfiguration(CacheConfiguration.class).getBackups());
+
+ Affinity<Object> aff = qryClient.affinity(null);
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ final TestLocalListener lsnr = new TestLocalListener();
+
+ qry.setLocalListener(lsnr);
+
+ qry.setRemoteFilter(lsnr);
+
+ int PARTS = 10;
+
+ QueryCursor<?> cur = qryClientCache.query(qry);
+
+ Map<Object, T2<Object, Object>> updates = new HashMap<>();
+
+ final List<T3<Object, Object, Object>> expEvts = new ArrayList<>();
+
+ for (int i = 0; i < SRV_NODES - 1; i++) {
+ log.info("Stop iteration: " + i);
+
+ TestCommunicationSpi spi = (TestCommunicationSpi)ignite(i).configuration().getCommunicationSpi();
+
+ Ignite ignite = ignite(i);
+
+ IgniteCache<Object, Object> cache = ignite.cache(null);
+
+ List<Integer> keys = testKeys(cache, PARTS);
+
+ boolean first = true;
+
+ boolean filtered = false;
+
+ for (Integer key : keys) {
+ log.info("Put [node=" + ignite.name() + ", key=" + key + ", part=" + aff.partition(key)
+ + ", filtered=" + filtered + ']');
+
+ T2<Object, Object> t = updates.get(key);
+
+ Integer val = filtered ?
+ (key % 2 == 0 ? key + 1 : key) :
+ key * 2;
+
+ if (t == null) {
+ updates.put(key, new T2<>((Object)val, null));
+
+ if (!filtered)
+ expEvts.add(new T3<>((Object)key, (Object)val, null));
+ }
+ else {
+ updates.put(key, new T2<>((Object)val, (Object)key));
+
+ if (!filtered)
+ expEvts.add(new T3<>((Object)key, (Object)val, (Object)key));
+ }
+
+ cache.put(key, val);
+
+ if (first) {
+ spi.skipMsg = true;
+
+ first = false;
+ }
+
+ filtered = !filtered;
+ }
+
+ stopGrid(i);
+
+ boolean check = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() throws IgniteCheckedException {
+ return expEvts.size() == lsnr.keys.size();
+ }
+ }, 5000L);
+
+ if (!check) {
+ Set<Integer> keys0 = new HashSet<>(keys);
+
+ keys0.removeAll(lsnr.keys);
+
+ log.info("Missed events for keys: " + keys0);
+
+ fail("Failed to wait for notifications [exp=" + keys.size() + ", left=" + keys0.size() + ']');
+ }
+
+ checkEvents(expEvts, lsnr);
+ }
+
+ cur.close();
+ }
+
+ /**
+ *
+ */
+ public static class TestLocalListener implements CacheEntryUpdatedListener<Object, Object>,
+ CacheEntryEventSerializableFilter<Object, Object> {
+ /** Keys. */
+ GridConcurrentHashSet<Integer> keys = new GridConcurrentHashSet<>();
+
+ /** Events. */
+ private final ConcurrentHashMap<Object, CacheEntryEvent<?, ?>> evts = new ConcurrentHashMap<>();
+
+ /** {@inheritDoc} */
+ @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> events) throws CacheEntryListenerException {
+ for (CacheEntryEvent<?, ?> e : events) {
+ System.err.println("Update entry: " + e);
+
+ Integer key = (Integer)e.getKey();
+
+ keys.add(key);
+
+ evts.put(key, e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean evaluate(CacheEntryEvent<?, ?> e) throws CacheEntryListenerException {
+ return (Integer)e.getValue() % 2 == 0;
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testThreeBackups() throws Exception {
if (cacheMode() == REPLICATED)
return;
@@ -261,6 +567,11 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
checkBackupQueue(3, false);
}
+ /** {@inheritDoc} */
+ @Override public boolean isDebug() {
+ return true;
+ }
+
/**
* @param backups Number of backups.
* @param updateFromClient If {@code true} executes cache update from client node.
@@ -423,7 +734,6 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
assertNotNull("No event for key: " + exp.get1(), e);
assertEquals("Unexpected value: " + e, exp.get2(), e.getValue());
- assertEquals("Unexpected old value: " + e, exp.get3(), e.getOldValue());
}
expEvts.clear();
@@ -432,6 +742,26 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
}
/**
+ * @param expEvts Expected events.
+ * @param lsnr Listener.
+ */
+ private void checkEvents(List<T3<Object, Object, Object>> expEvts, TestLocalListener lsnr) {
+ assert lsnr.evts.size() == expEvts.size();
+
+ for (T3<Object, Object, Object> exp : expEvts) {
+ CacheEntryEvent<?, ?> e = lsnr.evts.get(exp.get1());
+
+ assertNotNull("No event for key: " + exp.get1(), e);
+ assertEquals("Unexpected value: " + e, exp.get2(), e.getValue());
+ }
+
+ expEvts.clear();
+
+ lsnr.evts.clear();
+ lsnr.keys.clear();
+ }
+
+ /**
* @param cache Cache.
* @param parts Number of partitions.
* @return Keys.
@@ -447,7 +777,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
int[] nodeParts = aff.primaryPartitions(node);
- final int KEYS_PER_PART = 3;
+ final int KEYS_PER_PART = 50;
for (int i = 0; i < parts; i++) {
int part = nodeParts[i];
@@ -919,7 +1249,6 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
assertEquals(key, rcvdEvt.getKey());
assertEquals(expEvt.get1(), rcvdEvt.getValue());
- assertEquals(expEvt.get2(), rcvdEvt.getOldValue());
}
}
}
@@ -1012,7 +1341,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
private final ConcurrentHashMap<Integer, List<CacheEntryEvent<?, ?>>> evts = new ConcurrentHashMap<>();
/** {@inheritDoc} */
- @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts)
+ @Override public synchronized void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts)
throws CacheEntryListenerException {
try {
for (CacheEntryEvent<?, ?> evt : evts) {
@@ -1026,18 +1355,8 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
boolean dup = false;
- if (prevVal != null) {
- if (prevVal.equals(val)) // Can get this event with automatic put retry.
- dup = true;
- else {
- assertEquals("Unexpected event: " + evt, (Integer)(prevVal + 1), val);
- assertEquals("Unexpected event: " + evt, prevVal, evt.getOldValue());
- }
- }
- else {
- assertEquals("Unexpected event: " + evt, (Object)0, val);
- assertNull("Unexpected event: " + evt, evt.getOldValue());
- }
+ if (prevVal != null && prevVal.equals(val))
+ dup = true;
if (!dup) {
vals.put(key, val);
@@ -1074,6 +1393,9 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
private volatile boolean skipMsg;
/** */
+ private volatile boolean skipAllMsg;
+
+ /** */
private volatile AtomicBoolean sndFirstOnly;
/** {@inheritDoc} */
@@ -1081,6 +1403,9 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
throws IgniteSpiException {
Object msg0 = ((GridIoMessage)msg).message();
+ if (skipAllMsg)
+ return;
+
if (msg0 instanceof GridContinuousMessage) {
if (skipMsg) {
log.info("Skip continuous message: " + msg0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index d133a84..503b992 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -1232,7 +1232,7 @@ public abstract class GridAbstractTest extends TestCase {
if (isDebug()) {
discoSpi.setMaxMissedHeartbeats(Integer.MAX_VALUE);
- cfg.setNetworkTimeout(Long.MAX_VALUE);
+ cfg.setNetworkTimeout(Long.MAX_VALUE / 3);
}
else {
// Set network timeout to 10 sec to avoid unexpected p2p class loading errors.
[5/9] ignite git commit: IGNITE-426 WIP
Posted by nt...@apache.org.
IGNITE-426 WIP
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f262e5f6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f262e5f6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f262e5f6
Branch: refs/heads/ignite-426-2-reb
Commit: f262e5f6722f52080dc15c5bbb27df9d9a6b98a0
Parents: 1a05948
Author: nikolay_tikhonov <nt...@gridgain.com>
Authored: Tue Sep 29 16:12:46 2015 +0300
Committer: nikolay_tikhonov <nt...@gridgain.com>
Committed: Wed Oct 21 16:54:24 2015 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 8 +-
.../continuous/CacheContinuousQueryEntry.java | 26 ++-
.../CacheContinuousQueryFilteredEntry.java | 228 -------------------
.../continuous/CacheContinuousQueryHandler.java | 226 +++++++++---------
.../CacheContinuousQueryListener.java | 9 +-
.../CacheContinuousQueryLostPartition.java | 72 +++---
.../continuous/CacheContinuousQueryManager.java | 15 +-
...acheContinuousQueryFailoverAbstractTest.java | 87 ++++++-
8 files changed, 282 insertions(+), 389 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f262e5f6/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 6eb9e17..3474f84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -92,7 +92,6 @@ import org.apache.ignite.internal.processors.cache.query.GridCacheQueryResponse;
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryBatchAck;
import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryEntry;
-import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFilteredEntry;
import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryLostPartition;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
@@ -693,16 +692,11 @@ public class GridIoMessageFactory implements MessageFactory {
break;
case 115:
- msg = new CacheContinuousQueryFilteredEntry();
-
- break;
-
- case 116:
msg = new CacheContinuousQueryLostPartition();
break;
- // [-3..112] - this
+ // [-3..115] - this
// [120..123] - DR
// [-4..-22] - SQL
default:
http://git-wip-us.apache.org/repos/asf/ignite/blob/f262e5f6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index 470aa09..9e73142 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -83,6 +83,9 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
private long updateIdx;
/** */
+ private boolean filtered;
+
+ /** */
@GridToStringInclude
@GridDirectTransient
private AffinityTopologyVersion topVer;
@@ -152,6 +155,13 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
}
/**
+ * Mark this event as filtered.
+ */
+ void markFiltered() {
+ filtered = true;
+ }
+
+ /**
* @return Update index.
*/
long updateIndex() {
@@ -162,7 +172,7 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
* @return Filtered entry.
*/
boolean filtered() {
- return false;
+ return filtered;
}
/**
@@ -286,6 +296,12 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
writer.incrementState();
+ case 7:
+ if (!writer.writeBoolean("filtered", filtered))
+ return false;
+
+ writer.incrementState();
+
}
return true;
@@ -359,6 +375,14 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
reader.incrementState();
+ case 7:
+ filtered = reader.readBoolean("filtered");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
}
return reader.afterMessageRead(CacheContinuousQueryEntry.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f262e5f6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFilteredEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFilteredEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFilteredEntry.java
deleted file mode 100644
index 14d8f51..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFilteredEntry.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.query.continuous;
-
-import java.nio.ByteBuffer;
-import javax.cache.event.EventType;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.GridDirectTransient;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.plugin.extensions.communication.MessageReader;
-import org.apache.ignite.plugin.extensions.communication.MessageWriter;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * Continuous query entry.
- */
-public class CacheContinuousQueryFilteredEntry extends CacheContinuousQueryEntry {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private EventType evtType;
-
- /** Cache name. */
- private int cacheId;
-
- /** Partition. */
- private int part;
-
- /** Update index. */
- private long updateIdx;
-
- /** */
- @GridToStringInclude
- @GridDirectTransient
- private AffinityTopologyVersion topVer;
-
- /**
- * Required by {@link Message}.
- */
- public CacheContinuousQueryFilteredEntry() {
- // No-op.
- }
-
- /**
- * @param e Cache continuous query entry.
- */
- CacheContinuousQueryFilteredEntry(CacheContinuousQueryEntry e) {
- this.cacheId = e.cacheId();
- this.evtType = e.eventType();
- this.part = e.partition();
- this.updateIdx = e.updateIndex();
- this.topVer = e.topologyVersion();
- }
-
- /**
- * @return Topology version if applicable.
- */
- @Nullable AffinityTopologyVersion topologyVersion() {
- return topVer;
- }
-
- /**
- * @return Cache ID.
- */
- int cacheId() {
- return cacheId;
- }
-
- /**
- * @return Event type.
- */
- EventType eventType() {
- return evtType;
- }
-
- /**
- * @return Partition.
- */
- int partition() {
- return part;
- }
-
- /**
- * @return Update index.
- */
- long updateIndex() {
- return updateIdx;
- }
-
- /** {@inheritDoc} */
- @Override boolean filtered() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public byte directType() {
- return 115;
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- writer.setBuffer(buf);
-
- if (!writer.isHeaderWritten()) {
- if (!writer.writeHeader(directType(), fieldsCount()))
- return false;
-
- writer.onHeaderWritten();
- }
-
- switch (writer.state()) {
- case 0:
- if (!writer.writeInt("cacheId", cacheId))
- return false;
-
- writer.incrementState();
-
- case 1:
- if (!writer.writeByte("evtType", evtType != null ? (byte)evtType.ordinal() : -1))
- return false;
-
- writer.incrementState();
-
- case 2:
- if (!writer.writeInt("part", part))
- return false;
-
- writer.incrementState();
-
- case 3:
- if (!writer.writeLong("updateIdx", updateIdx))
- return false;
-
- writer.incrementState();
-
- }
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- reader.setBuffer(buf);
-
- if (!reader.beforeMessageRead())
- return false;
-
- switch (reader.state()) {
- case 0:
- cacheId = reader.readInt("cacheId");
-
- if (!reader.isLastRead())
- return false;
-
- reader.incrementState();
-
- case 1:
- byte evtTypeOrd;
-
- evtTypeOrd = reader.readByte("evtType");
-
- if (!reader.isLastRead())
- return false;
-
- evtType = CacheContinuousQueryEntry.eventTypeFromOrdinal(evtTypeOrd);
-
- reader.incrementState();
-
- case 2:
- part = reader.readInt("part");
-
- if (!reader.isLastRead())
- return false;
-
- reader.incrementState();
-
- case 3:
- updateIdx = reader.readLong("updateIdx");
-
- if (!reader.isLastRead())
- return false;
-
- reader.incrementState();
- }
-
- return reader.afterMessageRead(CacheContinuousQueryFilteredEntry.class);
- }
-
- /** {@inheritDoc} */
- @Override void prepareMarshal(GridCacheContext cctx) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override void unmarshal(GridCacheContext cctx, @Nullable ClassLoader ldr) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public byte fieldsCount() {
- return 4;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(CacheContinuousQueryFilteredEntry.class, this);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/f262e5f6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index ca3579e..bb2558c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -41,6 +41,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheEntryEventSerializableFilter;
+import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.CacheQueryExecutedEvent;
import org.apache.ignite.events.CacheQueryReadEvent;
import org.apache.ignite.internal.GridKernalContext;
@@ -83,6 +84,9 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
/** */
private static final int BACKUP_ACK_THRESHOLD = 100;
+ /** */
+ private static final int QUERY_HOLE_THRESHOLD = 5;
+
/** Cache name. */
private String cacheName;
@@ -123,15 +127,12 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
private transient Collection<CacheContinuousQueryEntry> backupQueue;
/** */
- private transient Map<Integer, Long> rcvCntrs;
+ private boolean localCache;
/** */
private transient ConcurrentMap<Integer, PartitionRecovery> rcvs;
/** */
- private transient IgnitePredicate<CacheContinuousQueryEntry> dupEvtFilter;
-
- /** */
private transient AcknowledgeBuffer ackBuf;
/** */
@@ -187,16 +188,9 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
this.ignoreExpired = ignoreExpired;
this.taskHash = taskHash;
this.skipPrimaryCheck = skipPrimaryCheck;
+ this.localCache = locCache;
- if (locCache)
- dupEvtFilter = F.alwaysTrue();
- else {
- rcvCntrs = new ConcurrentHashMap<>();
-
- rcvs = new ConcurrentHashMap<>();
-
- dupEvtFilter = new DuplicateEventFilter();
- }
+ rcvs = new ConcurrentHashMap<>();
cacheId = CU.cacheId(cacheName);
}
@@ -268,7 +262,7 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
if (ignoreExpired && evt.getEventType() == EventType.EXPIRED)
return;
- GridCacheContext<K, V> cctx = cacheContext(ctx);
+ final GridCacheContext<K, V> cctx = cacheContext(ctx);
// Check that cache stopped.
if (cctx == null)
@@ -289,12 +283,53 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
try {
- final CacheContinuousQueryEntry entry = notify ? evt.entry() :
- new CacheContinuousQueryFilteredEntry(evt.entry());
+ final CacheContinuousQueryEntry entry = evt.entry();
+
+ if (!notify)
+ entry.markFiltered();
if (primary || skipPrimaryCheck) {
if (loc) {
- if (dupEvtFilter.apply(entry)) {
+ if (!localCache) {
+ PartitionRecovery rcv = rcvs.get(entry.partition());
+
+ if (rcv == null) {
+ rcv = new PartitionRecovery(ctx.log(getClass()));
+
+ PartitionRecovery oldRec = rcvs.putIfAbsent(entry.partition(), rcv);
+
+ if (oldRec != null)
+ rcv = oldRec;
+ }
+
+ rcv.add(entry);
+
+ Collection<CacheContinuousQueryEntry> entries = rcv.entries();
+
+ if (!entries.isEmpty()) {
+ final IgniteCache cache = cctx.kernalContext().cache().jcache(cctx.name());
+
+ Iterable<CacheEntryEvent<? extends K, ? extends V>> evts = F.viewReadOnly(entries,
+ new C1<CacheContinuousQueryEntry, CacheEntryEvent<? extends K, ? extends V>>() {
+ @Override public CacheEntryEvent<? extends K, ? extends V> apply(
+ CacheContinuousQueryEntry e) {
+ return new CacheContinuousQueryEvent<>(cache, cctx, e);
+ }
+ },
+ new IgnitePredicate<CacheContinuousQueryEntry>() {
+ @Override public boolean apply(CacheContinuousQueryEntry entry) {
+ return !entry.filtered();
+ }
+ }
+ );
+
+ locLsnr.onUpdated(evts);
+
+ if (!skipPrimaryCheck)
+ sendBackupAcknowledge(ackBuf.onAcknowledged(entry), routineId, ctx);
+ }
+ }
+ else {
locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
if (!skipPrimaryCheck)
@@ -343,7 +378,16 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
}
- @Override public void partitionLost(String cacheName0, int partId) {
+ @Override public void partitionLost(int partId) {
+ assert rcvs != null;
+
+ PartitionRecovery rcv = rcvs.get(partId);
+
+ if (rcv != null)
+ rcv.reset();
+ }
+
+ @Override public void firePartitionLostEvent(String cacheName0, final int partId) {
GridCacheContext<K, V> cctx = cacheContext(ctx);
// Check that cache stopped.
@@ -352,25 +396,33 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
if ((cacheName == null && cacheName0 == null) || // Check default cache.
(cacheName0 != null && cacheName != null && cacheName0.equals(cacheName))) {
+ ctx.closure().runLocalSafe(new Runnable() {
+ @Override public void run() {
+ GridCacheContext<K, V> cctx = cacheContext(ctx);
- final CacheContinuousQueryEntry entry =
- new CacheContinuousQueryLostPartition(cctx.cacheId(), partId);
+ CacheContinuousQueryLostPartition msg = new CacheContinuousQueryLostPartition(
+ routineId,
+ cctx.cacheId(),
+ partId);
- try {
- prepareEntry(cctx, nodeId, entry);
+ try {
+ cctx.io().send(nodeId, msg, GridIoPolicy.SYSTEM_POOL);
+ }
+ catch (ClusterTopologyCheckedException e) {
+ IgniteLogger log = ctx.log(getClass());
- ctx.continuous().addNotification(nodeId, routineId, entry, topic, sync, true);
- }
- catch (ClusterTopologyCheckedException ex) {
- IgniteLogger log = ctx.log(getClass());
+ if (log.isDebugEnabled())
+ log.debug("Failed to send lost partition message, node left " +
+ "[msg=" + msg + ", nodeId=" + routineId + ']');
+ }
+ catch (IgniteCheckedException e) {
+ IgniteLogger log = ctx.log(getClass());
- if (log.isDebugEnabled())
- log.debug("Failed to send event notification to node, node left cluster " +
- "[node=" + nodeId + ", err=" + ex + ']');
- }
- catch (IgniteCheckedException ex) {
- U.error(ctx.log(getClass()), "Failed to send event notification to node: " + nodeId, ex);
- }
+ U.error(log, "Failed to send lost partition message " +
+ "[msg=" + msg + ", nodeId=" + routineId + ']', e);
+ }
+ }
+ });
}
}
@@ -537,14 +589,10 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
rec = oldRec;
}
- if (e instanceof CacheContinuousQueryLostPartition)
- rec.reset();
- else {
- rec.add(e);
+ rec.add(e);
- if (!parts.containsKey(e.partition()))
- parts.put(e.partition(), rec);
- }
+ if (!parts.containsKey(e.partition()))
+ parts.put(e.partition(), rec);
}
Collection<CacheContinuousQueryEntry> entries0 = new ArrayList<>();
@@ -569,29 +617,6 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
/**
- * @param e Entry.
- * @return {@code True} if listener should be notified.
- */
- private boolean notifyListener(CacheContinuousQueryEntry e) {
- Integer part = e.partition();
-
- Long cntr = rcvCntrs.get(part);
-
- if (cntr != null) {
- long cntr0 = cntr;
-
- if (e.updateIndex() > cntr0)
- rcvCntrs.put(part, e.updateIndex());
- else
- return false;
- }
- else
- rcvCntrs.put(part, e.updateIndex());
-
- return true;
- }
-
- /**
*
*/
private static class PartitionRecovery {
@@ -617,15 +642,13 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
* @param e Cache continuous qeury entry.
*/
public void add(CacheContinuousQueryEntry e) {
- synchronized (pendingEnts) {
- if (pendingEnts.containsKey(e.updateIndex()) || e.updateIndex() <= lastFiredEvt)
- e.cacheId();
- //log.info("Skip duplicate continuous query entry. Entry: " + e);
- else {
- //log.info("Added continuous query entry. Entry: " + e);
+ assert e != null;
+ synchronized (pendingEnts) {
+ if (!pendingEnts.containsKey(e.updateIndex()) && e.updateIndex() > lastFiredEvt)
pendingEnts.put(e.updateIndex(), e);
- }
+ else if (log.isDebugEnabled())
+ log.debug("Skip duplicate continuous query message: " + e);
}
}
@@ -641,45 +664,53 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
Iterator<Map.Entry<Long, CacheContinuousQueryEntry>> iter = pendingEnts.entrySet().iterator();
- Map.Entry<Long, CacheContinuousQueryEntry> prev = null;
-
- Set<Long> rmvEnts = new HashSet<>();
+ boolean fired = false;
+ // The elements are consistently.
while (iter.hasNext()) {
Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
- // The elements are consistently.
if (e.getKey() == lastFiredEvt + 1) {
++lastFiredEvt;
entries.add(e.getValue());
iter.remove();
+
+ fired = true;
+ }
+ }
+
+ if (!fired && lastFiredEvt == 0 && pendingEnts.size() >= QUERY_HOLE_THRESHOLD) {
+ Long prevCnt = null;
+
+ int orderedCnt = 0;
+
+ for (Long cnt : pendingEnts.keySet()) {
+ if (prevCnt != null) {
+ if (prevCnt + 1 != cnt)
+ break;
+ else
+ ++orderedCnt;
+ }
+
+ prevCnt = cnt;
}
- // Handle hole in sequence.
- else if (prev != null && prev.getKey() + 1 == e.getKey()) {
- entries.add(prev.getValue());
- lastFiredEvt = prev.getKey();
+ if (orderedCnt >= QUERY_HOLE_THRESHOLD) {
+ iter = pendingEnts.entrySet().iterator();
- rmvEnts.add(prev.getKey());
+ while (entries.size() < orderedCnt) {
+ Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
- if (!iter.hasNext()) {
entries.add(e.getValue());
lastFiredEvt = e.getKey();
- rmvEnts.add(e.getKey());
+ iter.remove();
}
}
- else if (prev != null)
- break;
-
- prev = e;
}
-
- for (Long rmKey : rmvEnts)
- pendingEnts.remove(rmKey);
}
return entries;
@@ -737,12 +768,6 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
/** {@inheritDoc} */
@Override public void partitionLost(String cacheName, int partId) {
- if (this.cacheName == null) {
- int z = 0;
-
- ++z;
- }
-
if ((this.cacheName == null && cacheName == null) // Check default caches.
|| (cacheName != null && this.cacheName != null && cacheName.equals(this.cacheName))) {
PartitionRecovery rcv = rcvs.get(partId);
@@ -962,19 +987,6 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
/**
- *
- */
- private class DuplicateEventFilter implements IgnitePredicate<CacheContinuousQueryEntry> {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** {@inheritDoc} */
- @Override public boolean apply(CacheContinuousQueryEntry e) {
- return notifyListener(e);
- }
- }
-
- /**
* Deployable object.
*/
private static class DeployableObject implements Externalizable {
http://git-wip-us.apache.org/repos/asf/ignite/blob/f262e5f6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
index 735e808..a706105 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
@@ -60,7 +60,14 @@ interface CacheContinuousQueryListener<K, V> {
* @param cacheName Cache name.
* @param partId Partition ID.
*/
- public void partitionLost(String cacheName, int partId);
+ public void firePartitionLostEvent(String cacheName, int partId);
+
+ /**
+ * Handle partition lost event.
+ *
+ * @param partId Partition ID.
+ */
+ public void partitionLost(int partId);
/**
* Flushes backup queue.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f262e5f6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java
index 734d072..eeb20cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryLostPartition.java
@@ -18,27 +18,22 @@
package org.apache.ignite.internal.processors.cache.query.continuous;
import java.nio.ByteBuffer;
-import javax.cache.event.EventType;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.GridDirectTransient;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import java.util.UUID;
+import org.apache.ignite.internal.processors.cache.GridCacheMessage;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.plugin.extensions.communication.MessageReader;
import org.apache.ignite.plugin.extensions.communication.MessageWriter;
-import org.jetbrains.annotations.Nullable;
/**
* Continuous query entry.
*/
-public class CacheContinuousQueryLostPartition extends CacheContinuousQueryEntry {
+public class CacheContinuousQueryLostPartition extends GridCacheMessage {
/** */
private static final long serialVersionUID = 0L;
- /** Cache name. */
- private int cacheId;
+ /** Routine ID. */
+ private UUID routineId;
/** Partition. */
private int part;
@@ -54,34 +49,38 @@ public class CacheContinuousQueryLostPartition extends CacheContinuousQueryEntry
* @param cacheId Cache ID.
* @param part Partition ID.
*/
- CacheContinuousQueryLostPartition(int cacheId, int part) {
+ CacheContinuousQueryLostPartition(UUID routineId, int cacheId, int part) {
+ this.routineId = routineId;
this.cacheId = cacheId;
this.part = part;
}
/**
- * @return Cache ID.
+ * @return Partition.
*/
- int cacheId() {
- return cacheId;
+ int partition() {
+ return part;
}
/**
- * @return Partition.
+ * @return Routine ID.
*/
- int partition() {
- return part;
+ UUID routineId() {
+ return routineId;
}
/** {@inheritDoc} */
@Override public byte directType() {
- return 116;
+ return 115;
}
/** {@inheritDoc} */
@Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
writer.setBuffer(buf);
+ if (!super.writeTo(buf, writer))
+ return false;
+
if (!writer.isHeaderWritten()) {
if (!writer.writeHeader(directType(), fieldsCount()))
return false;
@@ -90,17 +89,18 @@ public class CacheContinuousQueryLostPartition extends CacheContinuousQueryEntry
}
switch (writer.state()) {
- case 0:
- if (!writer.writeInt("cacheId", cacheId))
+ case 3:
+ if (!writer.writeInt("part", part))
return false;
writer.incrementState();
- case 1:
- if (!writer.writeInt("part", part))
+ case 4:
+ if (!writer.writeUuid("routineId", routineId))
return false;
writer.incrementState();
+
}
return true;
@@ -113,44 +113,36 @@ public class CacheContinuousQueryLostPartition extends CacheContinuousQueryEntry
if (!reader.beforeMessageRead())
return false;
+ if (!super.readFrom(buf, reader))
+ return false;
+
switch (reader.state()) {
- case 0:
- cacheId = reader.readInt("cacheId");
+ case 3:
+ part = reader.readInt("part");
if (!reader.isLastRead())
return false;
reader.incrementState();
- case 1:
- part = reader.readInt("part");
+ case 4:
+ routineId = reader.readUuid("routineId");
if (!reader.isLastRead())
return false;
- reader.incrementState();
}
- return reader.afterMessageRead(CacheContinuousQueryLostPartition.class);
- }
-
- /** {@inheritDoc} */
- @Override void prepareMarshal(GridCacheContext cctx) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override void unmarshal(GridCacheContext cctx, @Nullable ClassLoader ldr) throws IgniteCheckedException {
- // No-op.
+ return true;
}
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 2;
+ return 5;
}
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(CacheContinuousQueryLostPartition.class, this);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f262e5f6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 35c6696..d0d877d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -46,7 +46,6 @@ import org.apache.ignite.cache.CacheEntryEventSerializableFilter;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.query.ContinuousQuery;
import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.cluster.ClusterTopologyException;
import org.apache.ignite.events.CacheRebalancingEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
@@ -128,6 +127,16 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
}
});
+ cctx.io().addHandler(cctx.cacheId(), CacheContinuousQueryLostPartition.class,
+ new CI2<UUID, CacheContinuousQueryLostPartition>() {
+ @Override public void apply(UUID uuid, CacheContinuousQueryLostPartition msg) {
+ CacheContinuousQueryListener lsnr = lsnrs.get(msg.routineId());
+
+ if (lsnr != null)
+ lsnr.partitionLost(msg.partition());
+ }
+ });
+
cctx.time().schedule(new Runnable() {
@Override public void run() {
for (CacheContinuousQueryListener lsnr : lsnrs.values())
@@ -145,10 +154,10 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
CacheRebalancingEvent evt0 = (CacheRebalancingEvent)evt;
for (CacheContinuousQueryListener lsnr : lsnrs.values())
- lsnr.partitionLost(evt0.cacheName(), evt0.partition());
+ lsnr.firePartitionLostEvent(evt0.cacheName(), evt0.partition());
for (CacheContinuousQueryListener lsnr : intLsnrs.values())
- lsnr.partitionLost(evt0.cacheName(), evt0.partition());
+ lsnr.firePartitionLostEvent(evt0.cacheName(), evt0.partition());
}
}, EVT_CACHE_REBALANCE_PART_DATA_LOST);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f262e5f6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
index 3bba5e6..61fa6cd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
@@ -264,6 +264,84 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
/**
* @throws Exception If failed.
*/
+ public void testStartStopQuery() throws Exception {
+ this.backups = 1;
+
+ final int SRV_NODES = 3;
+
+ startGridsMultiThreaded(SRV_NODES);
+
+ client = true;
+
+ final Ignite qryClient = startGrid(SRV_NODES);
+
+ client = false;
+
+ IgniteCache<Object, Object> clnCache = qryClient.cache(null);
+
+ Ignite igniteSrv = ignite(0);
+
+ IgniteCache<Object, Object> srvCache = igniteSrv.cache(null);
+
+ List<Integer> keys = testKeys(srvCache, 1);
+
+ int keyCnt = keys.size();
+
+ for (int j = 0; j < 50; ++j) {
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ final TestLocalListener lsnr = new TestLocalListener();
+
+ qry.setLocalListener(lsnr);
+
+ int keyIter = 0;
+
+ for (; keyIter < keyCnt / 2; keyIter++) {
+ int key = keys.get(keyIter);
+
+ clnCache.put(key, key);
+ }
+
+ assert lsnr.evts.isEmpty();
+
+ QueryCursor<Cache.Entry<Object, Object>> query = clnCache.query(qry);
+
+ Map<Object, T2<Object, Object>> updates = new HashMap<>();
+
+ final List<T3<Object, Object, Object>> expEvts = new ArrayList<>();
+
+ Affinity<Object> aff = affinity(srvCache);
+
+ for (; keyIter < keys.size(); keyIter++) {
+ int key = keys.get(keyIter);
+
+ log.info("Put [key=" + key + ", part=" + aff.partition(key) + ']');
+
+ T2<Object, Object> t = updates.get(key);
+
+ if (t == null) {
+ updates.put(key, new T2<>((Object)key, null));
+
+ expEvts.add(new T3<>((Object)key, (Object)key, null));
+ }
+ else {
+ updates.put(key, new T2<>((Object)key, (Object)key));
+
+ expEvts.add(new T3<>((Object)key, (Object)key, (Object)key));
+ }
+
+ srvCache.put(key, key);
+ }
+
+ checkEvents(expEvts, lsnr);
+
+ query.close();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testLeftPrimaryAndBackupNodes() throws Exception {
this.backups = 1;
@@ -745,8 +823,13 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
* @param expEvts Expected events.
* @param lsnr Listener.
*/
- private void checkEvents(List<T3<Object, Object, Object>> expEvts, TestLocalListener lsnr) {
- assert lsnr.evts.size() == expEvts.size();
+ private void checkEvents(final List<T3<Object, Object, Object>> expEvts, final TestLocalListener lsnr)
+ throws Exception {
+ assert GridTestUtils.waitForCondition(new PA() {
+ @Override public boolean apply() {
+ return lsnr.evts.size() == expEvts.size();
+ }
+ }, 2000L);
for (T3<Object, Object, Object> exp : expEvts) {
CacheEntryEvent<?, ?> e = lsnr.evts.get(exp.get1());
[4/9] ignite git commit: IGNITE-426 WIP
Posted by nt...@apache.org.
IGNITE-426 WIP
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1a05948f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1a05948f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1a05948f
Branch: refs/heads/ignite-426-2-reb
Commit: 1a05948f43ce3e1fc55b9b28123f323131297777
Parents: 1f6ba6c
Author: nikolay_tikhonov <nt...@gridgain.com>
Authored: Fri Sep 11 18:57:59 2015 +0300
Committer: nikolay_tikhonov <nt...@gridgain.com>
Committed: Wed Oct 21 16:54:23 2015 +0300
----------------------------------------------------------------------
.../internal/GridEventConsumeHandler.java | 21 +-
.../internal/GridMessageListenHandler.java | 17 +
.../communication/GridIoMessageFactory.java | 18 +
.../processors/cache/GridCacheEntryEx.java | 4 +-
.../processors/cache/GridCacheMapEntry.java | 142 ++++++--
.../GridCachePartitionExchangeManager.java | 4 +-
.../cache/GridCacheUpdateAtomicResult.java | 15 +-
.../dht/atomic/GridDhtAtomicCache.java | 83 +++--
.../dht/atomic/GridDhtAtomicUpdateFuture.java | 22 +-
.../dht/atomic/GridDhtAtomicUpdateRequest.java | 124 +++++--
.../GridDhtPartitionsExchangeFuture.java | 35 +-
.../preloader/GridDhtPartitionsFullMessage.java | 57 ++-
.../GridDhtPartitionsSingleMessage.java | 49 ++-
.../distributed/near/GridNearAtomicCache.java | 14 +-
.../continuous/CacheContinuousQueryEntry.java | 7 +
.../CacheContinuousQueryFilteredEntry.java | 228 ++++++++++++
.../continuous/CacheContinuousQueryHandler.java | 294 ++++++++++++---
.../CacheContinuousQueryListener.java | 8 +
.../CacheContinuousQueryLostPartition.java | 156 ++++++++
.../continuous/CacheContinuousQueryManager.java | 22 +-
.../continuous/GridContinuousBatch.java | 39 +-
.../continuous/GridContinuousBatchAdapter.java | 43 ++-
.../continuous/GridContinuousHandler.java | 22 ++
.../continuous/GridContinuousProcessor.java | 5 +
.../processors/cache/GridCacheTestEntryEx.java | 4 +-
...acheContinuousQueryFailoverAbstractTest.java | 357 ++++++++++++++++++-
.../testframework/junits/GridAbstractTest.java | 2 +-
27 files changed, 1612 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
index 599d301..1b9c46c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
@@ -37,6 +37,8 @@ import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheDeployable;
import org.apache.ignite.internal.processors.cache.GridCacheDeploymentManager;
+import org.apache.ignite.internal.processors.continuous.GridContinuousBatch;
+import org.apache.ignite.internal.processors.continuous.GridContinuousBatchAdapter;
import org.apache.ignite.internal.processors.continuous.GridContinuousHandler;
import org.apache.ignite.internal.processors.platform.PlatformEventFilterListener;
import org.apache.ignite.internal.util.typedef.F;
@@ -210,8 +212,8 @@ class GridEventConsumeHandler implements GridContinuousHandler {
}
}
- ctx.continuous().addNotification(t3.get1(), t3.get2(), wrapper, null, false,
- false);
+ ctx.continuous().addNotification(t3.get1(), t3.get2(), wrapper, null,
+ false, false);
}
catch (ClusterTopologyCheckedException ignored) {
// No-op.
@@ -374,6 +376,21 @@ class GridEventConsumeHandler implements GridContinuousHandler {
}
/** {@inheritDoc} */
+ @Override public GridContinuousBatch createBatch() {
+ return new GridContinuousBatchAdapter();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onBatchAcknowledged(UUID routineId, GridContinuousBatch batch, GridKernalContext ctx) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void partitionLost(String cacheName, int partId) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public Object orderedTopic() {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/GridMessageListenHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridMessageListenHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/GridMessageListenHandler.java
index ff38949..e038794 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridMessageListenHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridMessageListenHandler.java
@@ -26,6 +26,8 @@ import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.managers.deployment.GridDeployment;
import org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean;
+import org.apache.ignite.internal.processors.continuous.GridContinuousBatch;
+import org.apache.ignite.internal.processors.continuous.GridContinuousBatchAdapter;
import org.apache.ignite.internal.processors.continuous.GridContinuousHandler;
import org.apache.ignite.internal.util.lang.GridPeerDeployAware;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -167,6 +169,21 @@ public class GridMessageListenHandler implements GridContinuousHandler {
}
/** {@inheritDoc} */
+ @Override public GridContinuousBatch createBatch() {
+ return new GridContinuousBatchAdapter();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onBatchAcknowledged(UUID routineId, GridContinuousBatch batch, GridKernalContext ctx) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void partitionLost(String cacheName, int partId) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public Object orderedTopic() {
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 079015c..6eb9e17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -90,7 +90,10 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearUnlo
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryRequest;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryResponse;
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryBatchAck;
import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryEntry;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFilteredEntry;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryLostPartition;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.transactions.TxEntryValueHolder;
@@ -684,6 +687,21 @@ public class GridIoMessageFactory implements MessageFactory {
break;
+ case 114:
+ msg = new CacheContinuousQueryBatchAck();
+
+ break;
+
+ case 115:
+ msg = new CacheContinuousQueryFilteredEntry();
+
+ break;
+
+ case 116:
+ msg = new CacheContinuousQueryLostPartition();
+
+ break;
+
// [-3..112] - this
// [120..123] - DR
// [-4..-22] - SQL
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index 430590a..a64752b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -450,7 +450,9 @@ public interface GridCacheEntryEx {
boolean conflictResolve,
boolean intercept,
@Nullable UUID subjId,
- String taskName
+ String taskName,
+ @Nullable CacheObject prevVal,
+ @Nullable Long updateIdx
) throws IgniteCheckedException, GridCacheEntryRemovedException;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 4bf0aa1..259869e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
import org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.extras.GridCacheEntryExtras;
import org.apache.ignite.internal.processors.cache.extras.GridCacheMvccEntryExtras;
import org.apache.ignite.internal.processors.cache.extras.GridCacheObsoleteEntryExtras;
@@ -1032,6 +1033,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
Object key0 = null;
Object val0 = null;
+ long updateIdx0;
+
synchronized (this) {
checkObsolete();
@@ -1101,6 +1104,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
deletedUnlocked(false);
}
+ updateIdx0 = nextPartIndex(topVer);
+
update(val, expireTime, ttl, newVer);
drReplicate(drType, val, newVer);
@@ -1126,8 +1131,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
subjId, null, taskName);
}
- if (cctx.isLocal() || cctx.isReplicated() || (tx != null && tx.local() && !isNear()))
- cctx.continuousQueries().onEntryUpdated(this, key, val, old, false);
+ if (!isNear())
+ cctx.continuousQueries().onEntryUpdated(this, key, val, old, tx.local(), false, updateIdx0, topVer);
cctx.dataStructures().onEntryUpdated(key, false);
}
@@ -1191,11 +1196,13 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
Cache.Entry entry0 = null;
+ Long updateIdx0;
+
synchronized (this) {
checkObsolete();
assert tx == null || (!tx.local() && tx.onePhaseCommit()) || tx.ownsLock(this) :
- "Transaction does not own lock for remove[entry=" + this + ", tx=" + tx + ']';
+ "Transaction does not own lock for remove[entry=" + this + ", tx=" + tx + ']';
boolean startVer = isStartVersion();
@@ -1252,6 +1259,11 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
}
+ updateIdx0 = nextPartIndex(topVer);
+
+// if (updateIdx != null)
+// updateIdx0 = updateIdx;
+
drReplicate(drType, null, newVer);
if (metrics && cctx.cache().configuration().isStatisticsEnabled())
@@ -1284,8 +1296,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
taskName);
}
- if (cctx.isLocal() || cctx.isReplicated() || (tx != null && tx.local() && !isNear()))
- cctx.continuousQueries().onEntryUpdated(this, key, null, old, false);
+ if (!isNear())
+ cctx.continuousQueries().onEntryUpdated(this, key, null, old, tx.local(), false, updateIdx0, topVer);
cctx.dataStructures().onEntryUpdated(key, true);
}
@@ -1627,7 +1639,12 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (res)
updateMetrics(op, metrics);
- cctx.continuousQueries().onEntryUpdated(this, key, val, old, false);
+ if (!isNear()) {
+ long updateIdx = nextPartIndex(AffinityTopologyVersion.NONE);
+
+ cctx.continuousQueries().onEntryUpdated(this, key, val, old, true, false, updateIdx,
+ AffinityTopologyVersion.NONE);
+ }
cctx.dataStructures().onEntryUpdated(key, op == GridCacheOperation.DELETE);
@@ -1670,7 +1687,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
boolean conflictResolve,
boolean intercept,
@Nullable UUID subjId,
- String taskName
+ String taskName,
+ @Nullable CacheObject prevVal,
+ @Nullable Long updateIdx
) throws IgniteCheckedException, GridCacheEntryRemovedException, GridClosureException {
assert cctx.atomic();
@@ -1679,7 +1698,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CacheObject oldVal;
CacheObject updated;
- GridCacheVersion enqueueVer = null;
+ GridCacheVersion rmvVer = null;
GridCacheVersionConflictContext<?, ?> conflictCtx = null;
@@ -1696,6 +1715,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
Object key0 = null;
Object updated0 = null;
+ Long updateIdx0 = null;
+
synchronized (this) {
boolean needVal = intercept || retval || op == GridCacheOperation.TRANSFORM || !F.isEmptyOrNulls(filter);
@@ -1803,7 +1824,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- false);
+ false,
+ updateIdx0);
}
// Will update something.
else {
@@ -1852,6 +1874,38 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
"[entry=" + this + ", newVer=" + newVer + ']');
}
+ if (!cctx.isNear()) {
+ CacheObject evtVal;
+
+ if (op == GridCacheOperation.TRANSFORM) {
+ EntryProcessor<Object, Object, ?> entryProcessor =
+ (EntryProcessor<Object, Object, ?>)writeObj;
+
+ CacheInvokeEntry<Object, Object> entry =
+ new CacheInvokeEntry<>(cctx, key, prevVal, version());
+
+ try {
+ entryProcessor.process(entry, invokeArgs);
+
+ evtVal = entry.modified() ?
+ cctx.toCacheObject(cctx.unwrapTemporary(entry.getValue())) : prevVal;
+ }
+ catch (Exception e) {
+ evtVal = prevVal;
+ }
+ }
+ else
+ evtVal = (CacheObject)writeObj;
+
+ updateIdx0 = nextPartIndex(topVer);
+
+ if (updateIdx != null)
+ updateIdx0 = updateIdx;
+
+ cctx.continuousQueries().onEntryUpdated(this, key, evtVal, prevVal, primary, false,
+ updateIdx0, topVer);
+ }
+
return new GridCacheUpdateAtomicResult(false,
retval ? rawGetOrUnmarshalUnlocked(false) : null,
null,
@@ -1860,7 +1914,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- false);
+ false,
+ updateIdx0);
}
}
else
@@ -1936,7 +1991,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- false);
+ false,
+ updateIdx0);
}
}
@@ -1983,7 +2039,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- false);
+ false,
+ updateIdx0);
}
}
else
@@ -2083,7 +2140,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- false);
+ false,
+ updateIdx0);
else if (interceptorVal != updated0) {
updated0 = cctx.unwrapTemporary(interceptorVal);
@@ -2120,6 +2178,11 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
update(updated, newExpireTime, newTtl, newVer);
+ updateIdx0 = nextPartIndex(topVer);
+
+ if (updateIdx != null)
+ updateIdx0 = updateIdx;
+
drReplicate(drType, updated, newVer);
recordNodeId(affNodeId, topVer);
@@ -2159,7 +2222,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CU.EXPIRE_TIME_ETERNAL,
null,
null,
- false);
+ false,
+ updateIdx0);
}
if (writeThrough)
@@ -2191,7 +2255,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
}
- enqueueVer = newVer;
+ rmvVer = newVer;
boolean hasValPtr = hasOffHeapPointer();
@@ -2211,6 +2275,11 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
recordNodeId(affNodeId, topVer);
+ updateIdx0 = nextPartIndex(topVer);
+
+ if (updateIdx != null)
+ updateIdx0 = updateIdx;
+
drReplicate(drType, null, newVer);
if (evt) {
@@ -2240,8 +2309,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (res)
updateMetrics(op, metrics);
- if (cctx.isReplicated() || primary)
- cctx.continuousQueries().onEntryUpdated(this, key, val, oldVal, false);
+ if (!isNear())
+ cctx.continuousQueries().onEntryUpdated(this, key, val, oldVal, primary, false, updateIdx0, topVer);
cctx.dataStructures().onEntryUpdated(key, op == GridCacheOperation.DELETE);
@@ -2265,9 +2334,10 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
invokeRes,
newSysTtl,
newSysExpireTime,
- enqueueVer,
+ rmvVer,
conflictCtx,
- true);
+ true,
+ updateIdx0);
}
/**
@@ -3056,9 +3126,13 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
drReplicate(drType, val, ver);
+ long updateIdx = -1;
+
+ if (!preload)
+ updateIdx = nextPartIndex(topVer);
+
if (!skipQryNtf) {
- if (cctx.isLocal() || cctx.isReplicated() || cctx.affinity().primary(cctx.localNode(), key, topVer))
- cctx.continuousQueries().onEntryUpdated(this, key, val, null, preload);
+ cctx.continuousQueries().onEntryUpdated(this, key, val, null, true, preload, updateIdx, topVer);
cctx.dataStructures().onEntryUpdated(key, false);
}
@@ -3075,6 +3149,28 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
}
+ /**
+ * @param topVer Topology version.
+ * @return Update index.
+ */
+ private long nextPartIndex(AffinityTopologyVersion topVer) {
+ long updateIdx;
+
+ //U.dumpStack();
+
+ if (!cctx.isLocal() && !isNear()) {
+ GridDhtLocalPartition locPart = cctx.topology().localPartition(partition(), topVer, false);
+
+ assert locPart != null;
+
+ updateIdx = locPart.nextContinuousQueryUpdateIndex();
+ }
+ else
+ updateIdx = 0;
+
+ return updateIdx;
+ }
+
/** {@inheritDoc} */
@Override public synchronized boolean initialValue(KeyCacheObject key, GridCacheSwapEntry unswapped) throws
IgniteCheckedException,
@@ -3942,7 +4038,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
*/
protected void deletedUnlocked(boolean deleted) {
assert Thread.holdsLock(this);
- assert cctx.deferredDelete();
+
+ if (!cctx.deferredDelete())
+ return;
if (deleted) {
assert !deletedUnlocked() : this;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index adc2174..0065403 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -892,7 +892,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
top = cacheCtx.topology();
if (top != null)
- updated |= top.update(null, entry.getValue()) != null;
+ updated |= top.update(null, entry.getValue(), null) != null;
}
if (!cctx.kernalContext().clientNode() && updated)
@@ -935,7 +935,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
top = cacheCtx.topology();
if (top != null)
- updated |= top.update(null, entry.getValue()) != null;
+ updated |= top.update(null, entry.getValue(), null) != null;
}
if (updated)
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateAtomicResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateAtomicResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateAtomicResult.java
index 3674284..092d990 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateAtomicResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateAtomicResult.java
@@ -57,6 +57,9 @@ public class GridCacheUpdateAtomicResult {
/** Whether update should be propagated to DHT node. */
private final boolean sndToDht;
+ /** */
+ private final Long updateIdx;
+
/** Value computed by entry processor. */
private IgniteBiTuple<Object, Exception> res;
@@ -72,6 +75,7 @@ public class GridCacheUpdateAtomicResult {
* @param rmvVer Version for deferred delete.
* @param conflictRes DR resolution result.
* @param sndToDht Whether update should be propagated to DHT node.
+ * @param updateIdx Partition update counter.
*/
public GridCacheUpdateAtomicResult(boolean success,
@Nullable CacheObject oldVal,
@@ -81,7 +85,8 @@ public class GridCacheUpdateAtomicResult {
long conflictExpireTime,
@Nullable GridCacheVersion rmvVer,
@Nullable GridCacheVersionConflictContext<?, ?> conflictRes,
- boolean sndToDht) {
+ boolean sndToDht,
+ long updateIdx) {
this.success = success;
this.oldVal = oldVal;
this.newVal = newVal;
@@ -91,6 +96,7 @@ public class GridCacheUpdateAtomicResult {
this.rmvVer = rmvVer;
this.conflictRes = conflictRes;
this.sndToDht = sndToDht;
+ this.updateIdx = updateIdx;
}
/**
@@ -129,6 +135,13 @@ public class GridCacheUpdateAtomicResult {
}
/**
+ * @return Partition update index.
+ */
+ public Long updateIdx() {
+ return updateIdx;
+ }
+
+ /**
* @return Explicit conflict expire time (if any). Set only if it is necessary to propagate concrete expire time
* value to DHT node. Otherwise set to {@link GridCacheUtils#EXPIRE_TIME_CALCULATE}.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 854a83d..a487593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -65,6 +65,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheA
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedGetFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearAtomicCache;
@@ -1111,7 +1112,9 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted = null;
try {
- topology().readLock();
+ GridDhtPartitionTopology top = topology();
+
+ top.readLock();
try {
if (topology().stopping()) {
@@ -1128,7 +1131,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
// Also do not check topology version if topology was locked on near node by
// external transaction or explicit lock.
if ((req.fastMap() && !req.clientRequest()) || req.topologyLocked() ||
- !needRemap(req.topologyVersion(), topology().topologyVersion())) {
+ !needRemap(req.topologyVersion(), top.topologyVersion())) {
ClusterNode node = ctx.discovery().node(nodeId);
if (node == null) {
@@ -1143,7 +1146,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
if (ver == null) {
// Assign next version for update inside entries lock.
- ver = ctx.versions().next(topology().topologyVersion());
+ ver = ctx.versions().next(top.topologyVersion());
if (hasNear)
res.nearVersion(ver);
@@ -1155,6 +1158,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
log.debug("Using cache version for update request on primary node [ver=" + ver +
", req=" + req + ']');
+ boolean sndPrevVal = !top.rebalanceFinished(req.topologyVersion());
+
dhtFut = createDhtFuture(ver, req, res, completionCb, false);
expiry = expiryPolicy(req.expiry());
@@ -1177,7 +1182,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
completionCb,
ctx.isDrEnabled(),
taskName,
- expiry);
+ expiry,
+ sndPrevVal);
deleted = updRes.deleted();
dhtFut = updRes.dhtFuture();
@@ -1196,7 +1202,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
completionCb,
ctx.isDrEnabled(),
taskName,
- expiry);
+ expiry,
+ sndPrevVal);
retVal = updRes.returnValue();
deleted = updRes.deleted();
@@ -1216,7 +1223,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
remap = true;
}
finally {
- topology().readUnlock();
+ top.readUnlock();
}
}
catch (GridCacheEntryRemovedException e) {
@@ -1291,6 +1298,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
* @param replicate Whether replication is enabled.
* @param taskName Task name.
* @param expiry Expiry policy.
+ * @param sndPrevVal If {@code true} sends previous value to backups.
* @return Deleted entries.
* @throws GridCacheEntryRemovedException Should not be thrown.
*/
@@ -1306,7 +1314,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
CI2<GridNearAtomicUpdateRequest, GridNearAtomicUpdateResponse> completionCb,
boolean replicate,
String taskName,
- @Nullable IgniteCacheExpiryPolicy expiry
+ @Nullable IgniteCacheExpiryPolicy expiry,
+ boolean sndPrevVal
) throws GridCacheEntryRemovedException {
assert !ctx.dr().receiveEnabled(); // Cannot update in batches during DR due to possible conflicts.
assert !req.returnValue() || req.operation() == TRANSFORM; // Should not request return values for putAll.
@@ -1453,7 +1462,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
replicate,
updRes,
taskName,
- expiry);
+ expiry,
+ sndPrevVal);
firstEntryIdx = i;
@@ -1501,7 +1511,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
replicate,
updRes,
taskName,
- expiry);
+ expiry,
+ sndPrevVal);
firstEntryIdx = i;
@@ -1620,7 +1631,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
replicate,
updRes,
taskName,
- expiry);
+ expiry,
+ sndPrevVal);
}
else
assert filtered.isEmpty();
@@ -1696,6 +1708,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
* @param replicate Whether DR is enabled for that cache.
* @param taskName Task name.
* @param expiry Expiry policy.
+ * @param sndPrevVal If {@code true} sends previous value to backups.
* @return Return value.
* @throws GridCacheEntryRemovedException Should be never thrown.
*/
@@ -1710,7 +1723,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
CI2<GridNearAtomicUpdateRequest, GridNearAtomicUpdateResponse> completionCb,
boolean replicate,
String taskName,
- @Nullable IgniteCacheExpiryPolicy expiry
+ @Nullable IgniteCacheExpiryPolicy expiry,
+ boolean sndPrevVal
) throws GridCacheEntryRemovedException {
GridCacheReturn retVal = null;
Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted = null;
@@ -1767,7 +1781,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
req.invokeArguments(),
primary && writeThrough() && !req.skipStore(),
!req.skipStore(),
- req.returnValue(),
+ sndPrevVal || req.returnValue(),
expiry,
true,
true,
@@ -1782,7 +1796,9 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
true,
intercept,
req.subjectId(),
- taskName);
+ taskName,
+ null,
+ null);
if (dhtFut == null && !F.isEmpty(filteredReaders)) {
dhtFut = createDhtFuture(ver, req, res, completionCb, true);
@@ -1799,22 +1815,22 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
else if (conflictCtx.isMerge())
newConflictVer = null; // Conflict version is discarded in case of merge.
- EntryProcessor<Object, Object, Object> entryProcessor = null;
-
if (!readersOnly) {
dhtFut.addWriteEntry(entry,
updRes.newValue(),
- entryProcessor,
+ op == TRANSFORM ? req.entryProcessor(i) : null,
updRes.newTtl(),
updRes.conflictExpireTime(),
- newConflictVer);
+ newConflictVer,
+ sndPrevVal,
+ updRes.oldValue(),
+ updRes.updateIdx());
}
if (!F.isEmpty(filteredReaders))
dhtFut.addNearWriteEntries(filteredReaders,
entry,
updRes.newValue(),
- entryProcessor,
updRes.newTtl(),
updRes.conflictExpireTime());
}
@@ -1914,6 +1930,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
* @param batchRes Batch update result.
* @param taskName Task name.
* @param expiry Expiry policy.
+ * @param sndPrevVal If {@code true} sends previous value to backups.
* @return Deleted entries.
*/
@SuppressWarnings("ForLoopReplaceableByForEach")
@@ -1934,7 +1951,8 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
boolean replicate,
UpdateBatchResult batchRes,
String taskName,
- @Nullable IgniteCacheExpiryPolicy expiry
+ @Nullable IgniteCacheExpiryPolicy expiry,
+ boolean sndPrevVal
) {
assert putMap == null ^ rmvKeys == null;
@@ -2036,7 +2054,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
null,
/*write-through*/false,
/*read-through*/false,
- /*retval*/false,
+ /*retval*/sndPrevVal,
expiry,
/*event*/true,
/*metrics*/true,
@@ -2051,7 +2069,9 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
/*conflict resolve*/false,
/*intercept*/false,
req.subjectId(),
- taskName);
+ taskName,
+ null,
+ null);
assert !updRes.success() || updRes.newTtl() == CU.TTL_NOT_CHANGED || expiry != null :
"success=" + updRes.success() + ", newTtl=" + updRes.newTtl() + ", expiry=" + expiry;
@@ -2081,22 +2101,21 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
}
if (dhtFut != null) {
- EntryProcessor<Object, Object, Object> entryProcessor =
- entryProcessorMap == null ? null : entryProcessorMap.get(entry.key());
-
if (!batchRes.readersOnly())
dhtFut.addWriteEntry(entry,
writeVal,
- entryProcessor,
+ entryProcessorMap == null ? null : entryProcessorMap.get(entry.key()),
updRes.newTtl(),
CU.EXPIRE_TIME_CALCULATE,
- null);
+ null,
+ sndPrevVal,
+ updRes.oldValue(),
+ updRes.updateIdx());
if (!F.isEmpty(filteredReaders))
dhtFut.addNearWriteEntries(filteredReaders,
entry,
writeVal,
- entryProcessor,
updRes.newTtl(),
CU.EXPIRE_TIME_CALCULATE);
}
@@ -2499,7 +2518,9 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
entry = entryExx(key);
CacheObject val = req.value(i);
+ CacheObject prevVal = req.previousValue(i);
EntryProcessor<Object, Object, Object> entryProcessor = req.entryProcessor(i);
+ Long updateIdx = req.updateIdx(i);
GridCacheOperation op = entryProcessor != null ? TRANSFORM :
(val != null) ? UPDATE : DELETE;
@@ -2521,7 +2542,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
/*event*/true,
/*metrics*/true,
/*primary*/false,
- /*check version*/!req.forceTransformBackups(),
+ /*check version*/op != TRANSFORM || !req.forceTransformBackups(),
req.topologyVersion(),
CU.empty0(),
replicate ? DR_BACKUP : DR_NONE,
@@ -2531,7 +2552,9 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
false,
intercept,
req.subjectId(),
- taskName);
+ taskName,
+ prevVal,
+ updateIdx);
if (updRes.removeVersion() != null)
ctx.onDeferredDelete(entry, updRes.removeVersion());
@@ -2573,7 +2596,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
}
catch (ClusterTopologyCheckedException ignored) {
U.warn(log, "Failed to send DHT atomic update response to node because it left grid: " +
- req.nodeId());
+ nodeId);
}
catch (IgniteCheckedException e) {
U.error(log, "Failed to send DHT atomic update response (did node leave grid?) [nodeId=" + nodeId +
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
index 35b8e27..4019579 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
@@ -36,6 +36,7 @@ import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheAtomicFuture;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -132,6 +133,9 @@ public class GridDhtAtomicUpdateFuture extends GridFutureAdapter<Void>
boolean topLocked = updateReq.topologyLocked() || (updateReq.fastMap() && !updateReq.clientRequest());
waitForExchange = !topLocked;
+
+ // We can send entry processor instead of value to backup if updates are ordered.
+ forceTransformBackups = updateReq.operation() == GridCacheOperation.TRANSFORM;
}
/** {@inheritDoc} */
@@ -198,16 +202,22 @@ public class GridDhtAtomicUpdateFuture extends GridFutureAdapter<Void>
* @param ttl TTL (optional).
* @param conflictExpireTime Conflict expire time (optional).
* @param conflictVer Conflict version (optional).
+ * @param updateIdx Partition update index.
*/
public void addWriteEntry(GridDhtCacheEntry entry,
@Nullable CacheObject val,
EntryProcessor<Object, Object, Object> entryProcessor,
long ttl,
long conflictExpireTime,
- @Nullable GridCacheVersion conflictVer) {
+ @Nullable GridCacheVersion conflictVer,
+ boolean addPrevVal,
+ @Nullable CacheObject prevVal,
+ @Nullable Long updateIdx) {
AffinityTopologyVersion topVer = updateReq.topologyVersion();
- Collection<ClusterNode> dhtNodes = cctx.dht().topology().nodes(entry.partition(), topVer);
+ int part = entry.partition();
+
+ Collection<ClusterNode> dhtNodes = cctx.dht().topology().nodes(part, topVer);
if (log.isDebugEnabled())
log.debug("Mapping entry to DHT nodes [nodes=" + U.nodeIds(dhtNodes) + ", entry=" + entry + ']');
@@ -243,7 +253,10 @@ public class GridDhtAtomicUpdateFuture extends GridFutureAdapter<Void>
entryProcessor,
ttl,
conflictExpireTime,
- conflictVer);
+ conflictVer,
+ addPrevVal,
+ prevVal,
+ updateIdx);
}
}
}
@@ -252,14 +265,12 @@ public class GridDhtAtomicUpdateFuture extends GridFutureAdapter<Void>
* @param readers Entry readers.
* @param entry Entry.
* @param val Value.
- * @param entryProcessor Entry processor..
* @param ttl TTL for near cache update (optional).
* @param expireTime Expire time for near cache update (optional).
*/
public void addNearWriteEntries(Iterable<UUID> readers,
GridDhtCacheEntry entry,
@Nullable CacheObject val,
- EntryProcessor<Object, Object, Object> entryProcessor,
long ttl,
long expireTime) {
CacheWriteSynchronizationMode syncMode = updateReq.writeSynchronizationMode();
@@ -300,7 +311,6 @@ public class GridDhtAtomicUpdateFuture extends GridFutureAdapter<Void>
updateReq.addNearWriteValue(entry.key(),
val,
- entryProcessor,
ttl,
expireTime);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
index f5231ef..380b194 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
@@ -78,6 +78,11 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
@GridDirectCollection(CacheObject.class)
private List<CacheObject> vals;
+ /** Previous values. */
+ @GridToStringInclude
+ @GridDirectCollection(CacheObject.class)
+ private List<CacheObject> prevVals;
+
/** Conflict versions. */
@GridDirectCollection(GridCacheVersion.class)
private List<GridCacheVersion> conflictVers;
@@ -139,6 +144,9 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
/** Task name hash. */
private int taskNameHash;
+ /** Partition. */
+ private GridLongList updateCntrs;
+
/**
* Empty constructor required by {@link Externalizable}.
*/
@@ -209,13 +217,18 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
* @param ttl TTL (optional).
* @param conflictExpireTime Conflict expire time (optional).
* @param conflictVer Conflict version (optional).
+ * @param addPrevVal If {@code true} adds previous value.
+ * @param prevVal Previous value.
*/
public void addWriteValue(KeyCacheObject key,
@Nullable CacheObject val,
EntryProcessor<Object, Object, Object> entryProcessor,
long ttl,
long conflictExpireTime,
- @Nullable GridCacheVersion conflictVer) {
+ @Nullable GridCacheVersion conflictVer,
+ boolean addPrevVal,
+ @Nullable CacheObject prevVal,
+ @Nullable Long updateIdx) {
keys.add(key);
if (forceTransformBackups) {
@@ -226,6 +239,20 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
else
vals.add(val);
+ if (addPrevVal) {
+ if (prevVals == null)
+ prevVals = new ArrayList<>();
+
+ prevVals.add(prevVal);
+ }
+
+ if (updateIdx != null) {
+ if (updateCntrs == null)
+ updateCntrs = new GridLongList();
+
+ updateCntrs.add(updateIdx);
+ }
+
// In case there is no conflict, do not create the list.
if (conflictVer != null) {
if (conflictVers == null) {
@@ -268,36 +295,21 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
/**
* @param key Key to add.
* @param val Value, {@code null} if should be removed.
- * @param entryProcessor Entry processor.
* @param ttl TTL.
* @param expireTime Expire time.
*/
public void addNearWriteValue(KeyCacheObject key,
@Nullable CacheObject val,
- EntryProcessor<Object, Object, Object> entryProcessor,
long ttl,
long expireTime)
{
if (nearKeys == null) {
nearKeys = new ArrayList<>();
-
- if (forceTransformBackups) {
- nearEntryProcessors = new ArrayList<>();
- nearEntryProcessorsBytes = new ArrayList<>();
- }
- else
- nearVals = new ArrayList<>();
+ nearVals = new ArrayList<>();
}
nearKeys.add(key);
-
- if (forceTransformBackups) {
- assert entryProcessor != null;
-
- nearEntryProcessors.add(entryProcessor);
- }
- else
- nearVals.add(val);
+ nearVals.add(val);
if (ttl >= 0) {
if (nearTtls == null) {
@@ -408,6 +420,17 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
}
/**
+ * @param idx Counter index.
+ * @return Update counter.
+ */
+ public Long updateIdx(int idx) {
+ if (idx < updateCntrs.size())
+ return updateCntrs.get(idx);
+
+ return null;
+ }
+
+ /**
* @param idx Near key index.
* @return Key.
*/
@@ -428,6 +451,17 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
/**
* @param idx Key index.
+ * @return Value.
+ */
+ @Nullable public CacheObject previousValue(int idx) {
+ if (prevVals != null)
+ return prevVals.get(idx);
+
+ return null;
+ }
+
+ /**
+ * @param idx Key index.
* @return Entry processor.
*/
@Nullable public EntryProcessor<Object, Object, Object> entryProcessor(int idx) {
@@ -673,42 +707,54 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
writer.incrementState();
case 16:
- if (!writer.writeUuid("subjId", subjId))
+ if (!writer.writeMessage("updateCntrs", updateCntrs))
return false;
writer.incrementState();
case 17:
- if (!writer.writeByte("syncMode", syncMode != null ? (byte)syncMode.ordinal() : -1))
+ if (!writer.writeCollection("prevVals", prevVals, MessageCollectionItemType.MSG))
return false;
writer.incrementState();
case 18:
- if (!writer.writeInt("taskNameHash", taskNameHash))
+ if (!writer.writeUuid("subjId", subjId))
return false;
writer.incrementState();
case 19:
- if (!writer.writeMessage("topVer", topVer))
+ if (!writer.writeByte("syncMode", syncMode != null ? (byte)syncMode.ordinal() : -1))
return false;
writer.incrementState();
case 20:
- if (!writer.writeMessage("ttls", ttls))
+ if (!writer.writeInt("taskNameHash", taskNameHash))
return false;
writer.incrementState();
case 21:
- if (!writer.writeCollection("vals", vals, MessageCollectionItemType.MSG))
+ if (!writer.writeMessage("topVer", topVer))
return false;
writer.incrementState();
case 22:
+ if (!writer.writeMessage("ttls", ttls))
+ return false;
+
+ writer.incrementState();
+
+ case 23:
+ if (!writer.writeCollection("vals", vals, MessageCollectionItemType.MSG))
+ return false;
+
+ writer.incrementState();
+
+ case 24:
if (!writer.writeMessage("writeVer", writeVer))
return false;
@@ -835,7 +881,7 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
reader.incrementState();
case 16:
- subjId = reader.readUuid("subjId");
+ updateCntrs = reader.readMessage("updateCntrs");
if (!reader.isLastRead())
return false;
@@ -843,6 +889,22 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
reader.incrementState();
case 17:
+ prevVals = reader.readCollection("prevVals", MessageCollectionItemType.MSG);
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 18:
+ subjId = reader.readUuid("subjId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 19:
byte syncModeOrd;
syncModeOrd = reader.readByte("syncMode");
@@ -854,7 +916,7 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
reader.incrementState();
- case 18:
+ case 20:
taskNameHash = reader.readInt("taskNameHash");
if (!reader.isLastRead())
@@ -862,7 +924,7 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
reader.incrementState();
- case 19:
+ case 21:
topVer = reader.readMessage("topVer");
if (!reader.isLastRead())
@@ -870,7 +932,7 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
reader.incrementState();
- case 20:
+ case 22:
ttls = reader.readMessage("ttls");
if (!reader.isLastRead())
@@ -878,7 +940,7 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
reader.incrementState();
- case 21:
+ case 23:
vals = reader.readCollection("vals", MessageCollectionItemType.MSG);
if (!reader.isLastRead())
@@ -886,7 +948,7 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
reader.incrementState();
- case 22:
+ case 24:
writeVer = reader.readMessage("writeVer");
if (!reader.isLastRead())
@@ -906,7 +968,7 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 23;
+ return 25;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 77e47a7..cb4bb4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -613,7 +613,9 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
if (updateTop) {
for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) {
if (top.cacheId() == cacheCtx.cacheId()) {
- cacheCtx.topology().update(exchId, top.partitionMap(true));
+ cacheCtx.topology().update(exchId,
+ top.partitionMap(true),
+ top.updateCounters());
break;
}
@@ -811,6 +813,8 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
}
}
+ boolean topChanged = discoEvt.type() != EVT_DISCOVERY_CUSTOM_EVT;
+
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal())
continue;
@@ -821,6 +825,9 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
if (drCacheCtx.isDrEnabled())
drCacheCtx.dr().beforeExchange(topVer, exchId.isLeft());
+ if (topChanged)
+ cacheCtx.continuousQueries().beforeExchange(exchId.topologyVersion());
+
// Partition release future is done so we can flush the write-behind store.
cacheCtx.store().forceFlush();
@@ -954,14 +961,18 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
* @param id ID.
* @throws IgniteCheckedException If failed.
*/
- private void sendLocalPartitions(ClusterNode node, @Nullable GridDhtPartitionExchangeId id) throws IgniteCheckedException {
+ private void sendLocalPartitions(ClusterNode node, @Nullable GridDhtPartitionExchangeId id)
+ throws IgniteCheckedException {
GridDhtPartitionsSingleMessage m = new GridDhtPartitionsSingleMessage(id,
clientOnlyExchange,
cctx.versions().last());
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
- if (!cacheCtx.isLocal())
+ if (!cacheCtx.isLocal()) {
m.addLocalPartitionMap(cacheCtx.cacheId(), cacheCtx.topology().localPartitionMap());
+
+ m.partitionUpdateCounters(cacheCtx.cacheId(), cacheCtx.topology().updateCounters());
+ }
}
if (log.isDebugEnabled())
@@ -987,15 +998,21 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
boolean ready = startTopVer == null || startTopVer.compareTo(id.topologyVersion()) <= 0;
- if (ready)
+ if (ready) {
m.addFullPartitionsMap(cacheCtx.cacheId(), cacheCtx.topology().partitionMap(true));
+
+ m.addPartitionUpdateCounters(cacheCtx.cacheId(), cacheCtx.topology().updateCounters());
+ }
}
}
// It is important that client topologies be added after contexts.
- for (GridClientPartitionTopology top : cctx.exchange().clientTopologies())
+ for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) {
m.addFullPartitionsMap(top.cacheId(), top.partitionMap(true));
+ m.addPartitionUpdateCounters(top.cacheId(), top.updateCounters());
+ }
+
if (log.isDebugEnabled())
log.debug("Sending full partition map [nodeIds=" + F.viewReadOnly(nodes, F.node2id()) +
", exchId=" + exchId + ", msg=" + m + ']');
@@ -1332,15 +1349,17 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
for (Map.Entry<Integer, GridDhtPartitionFullMap> entry : msg.partitions().entrySet()) {
Integer cacheId = entry.getKey();
+ Map<Integer, Long> cntrMap = msg.partitionUpdateCounters(cacheId);
+
GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
if (cacheCtx != null)
- cacheCtx.topology().update(exchId, entry.getValue());
+ cacheCtx.topology().update(exchId, entry.getValue(), cntrMap);
else {
ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, AffinityTopologyVersion.NONE);
if (oldest != null && oldest.isLocal())
- cctx.exchange().clientTopology(cacheId, this).update(exchId, entry.getValue());
+ cctx.exchange().clientTopology(cacheId, this).update(exchId, entry.getValue(), cntrMap);
}
}
}
@@ -1358,7 +1377,7 @@ public class GridDhtPartitionsExchangeFuture extends GridFutureAdapter<AffinityT
GridDhtPartitionTopology top = cacheCtx != null ? cacheCtx.topology() :
cctx.exchange().clientTopology(cacheId, this);
- top.update(exchId, entry.getValue());
+ top.update(exchId, entry.getValue(), msg.partitionUpdateCounters(cacheId));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index c06d773..758818d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
import java.io.Externalizable;
import java.nio.ByteBuffer;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.apache.ignite.IgniteCheckedException;
@@ -48,6 +49,14 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
/** */
private byte[] partsBytes;
+ /** Partitions update counters. */
+ @GridToStringInclude
+ @GridDirectTransient
+ private Map<Integer, Map<Integer, Long>> partCntrs = new HashMap<>();
+
+ /** Serialized partitions counters. */
+ private byte[] partCntrsBytes;
+
/** Topology version. */
private AffinityTopologyVersion topVer;
@@ -92,13 +101,34 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
parts.put(cacheId, fullMap);
}
- /** {@inheritDoc}
- * @param ctx*/
+ /**
+ * @param cacheId Cache ID.
+ * @param cntrMap Partition update counters.
+ */
+ public void addPartitionUpdateCounters(int cacheId, Map<Integer, Long> cntrMap) {
+ if (!partCntrs.containsKey(cacheId))
+ partCntrs.put(cacheId, cntrMap);
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @return Partition update counters.
+ */
+ public Map<Integer, Long> partitionUpdateCounters(int cacheId) {
+ Map<Integer, Long> res = partCntrs.get(cacheId);
+
+ return res != null ? res : Collections.<Integer, Long>emptyMap();
+ }
+
+ /** {@inheritDoc} */
@Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
super.prepareMarshal(ctx);
if (parts != null && partsBytes == null)
partsBytes = ctx.marshaller().marshal(parts);
+
+ if (partCntrs != null)
+ partCntrsBytes = ctx.marshaller().marshal(partCntrs);
}
/**
@@ -121,6 +151,9 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
if (partsBytes != null && parts == null)
parts = ctx.marshaller().unmarshal(partsBytes, ldr);
+
+ if (partCntrsBytes != null)
+ partCntrs = ctx.marshaller().unmarshal(partCntrsBytes, ldr);
}
/** {@inheritDoc} */
@@ -139,12 +172,18 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
switch (writer.state()) {
case 5:
- if (!writer.writeByteArray("partsBytes", partsBytes))
+ if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
return false;
writer.incrementState();
case 6:
+ if (!writer.writeByteArray("partsBytes", partsBytes))
+ return false;
+
+ writer.incrementState();
+
+ case 7:
if (!writer.writeMessage("topVer", topVer))
return false;
@@ -167,7 +206,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
switch (reader.state()) {
case 5:
- partsBytes = reader.readByteArray("partsBytes");
+ partCntrsBytes = reader.readByteArray("partCntrsBytes");
if (!reader.isLastRead())
return false;
@@ -175,6 +214,14 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
reader.incrementState();
case 6:
+ partsBytes = reader.readByteArray("partsBytes");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 7:
topVer = reader.readMessage("topVer");
if (!reader.isLastRead())
@@ -194,7 +241,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 7;
+ return 8;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 83fbb1a..547c0f6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
import java.io.Externalizable;
import java.nio.ByteBuffer;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.apache.ignite.IgniteCheckedException;
@@ -46,6 +47,14 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
/** Serialized partitions. */
private byte[] partsBytes;
+ /** Partitions update counters. */
+ @GridToStringInclude
+ @GridDirectTransient
+ private Map<Integer, Map<Integer, Long>> partCntrs = new HashMap<>();
+
+ /** Serialized partitions counters. */
+ private byte[] partCntrsBytes;
+
/** */
private boolean client;
@@ -90,6 +99,24 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
}
/**
+ * @param cacheId Cache ID.
+ * @param cntrMap Partition update counters.
+ */
+ public void partitionUpdateCounters(int cacheId, Map<Integer, Long> cntrMap) {
+ partCntrs.put(cacheId, cntrMap);
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @return Partition update counters.
+ */
+ public Map<Integer, Long> partitionUpdateCounters(int cacheId) {
+ Map<Integer, Long> res = partCntrs.get(cacheId);
+
+ return res != null ? res : Collections.<Integer, Long>emptyMap();
+ }
+
+ /**
* @return Local partitions.
*/
public Map<Integer, GridDhtPartitionMap> partitions() {
@@ -103,6 +130,9 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
if (partsBytes == null && parts != null)
partsBytes = ctx.marshaller().marshal(parts);
+
+ if (partCntrs != null)
+ partCntrsBytes = ctx.marshaller().marshal(partCntrs);
}
/** {@inheritDoc} */
@@ -111,6 +141,9 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
if (partsBytes != null && parts == null)
parts = ctx.marshaller().unmarshal(partsBytes, ldr);
+
+ if (partCntrsBytes != null)
+ partCntrs = ctx.marshaller().unmarshal(partCntrsBytes, ldr);
}
/** {@inheritDoc} */
@@ -135,6 +168,12 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
writer.incrementState();
case 6:
+ if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
+ return false;
+
+ writer.incrementState();
+
+ case 7:
if (!writer.writeByteArray("partsBytes", partsBytes))
return false;
@@ -165,6 +204,14 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
reader.incrementState();
case 6:
+ partCntrsBytes = reader.readByteArray("partCntrsBytes");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 7:
partsBytes = reader.readByteArray("partsBytes");
if (!reader.isLastRead())
@@ -184,7 +231,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 7;
+ return 8;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index 82054d9..eaeb5f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -249,7 +249,7 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
/*write-through*/false,
/*read-through*/false,
/*retval*/false,
- /**expiry policy*/null,
+ /*expiry policy*/null,
/*event*/true,
/*metrics*/true,
/*primary*/false,
@@ -263,7 +263,9 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
false,
false,
subjId,
- taskName);
+ taskName,
+ null,
+ null);
if (updRes.removeVersion() != null)
ctx.onDeferredDelete(entry, updRes.removeVersion());
@@ -351,7 +353,7 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
/*event*/true,
/*metrics*/true,
/*primary*/false,
- /*check version*/!req.forceTransformBackups(),
+ /*check version*/op != TRANSFORM || !req.forceTransformBackups(),
req.topologyVersion(),
CU.empty0(),
DR_NONE,
@@ -359,9 +361,11 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
expireTime,
null,
false,
- intercept,
+ /*intercept*/false,
req.subjectId(),
- taskName);
+ taskName,
+ null,
+ null);
if (updRes.removeVersion() != null)
ctx.onDeferredDelete(entry, updRes.removeVersion());
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index 9ea9b73..470aa09 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -159,6 +159,13 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
}
/**
+ * @return Filtered entry.
+ */
+ boolean filtered() {
+ return false;
+ }
+
+ /**
* @param cctx Cache context.
* @throws IgniteCheckedException In case of error.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a05948f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFilteredEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFilteredEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFilteredEntry.java
new file mode 100644
index 0000000..14d8f51
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFilteredEntry.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.nio.ByteBuffer;
+import javax.cache.event.EventType;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Continuous query entry.
+ */
+public class CacheContinuousQueryFilteredEntry extends CacheContinuousQueryEntry {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private EventType evtType;
+
+ /** Cache name. */
+ private int cacheId;
+
+ /** Partition. */
+ private int part;
+
+ /** Update index. */
+ private long updateIdx;
+
+ /** */
+ @GridToStringInclude
+ @GridDirectTransient
+ private AffinityTopologyVersion topVer;
+
+ /**
+ * Required by {@link Message}.
+ */
+ public CacheContinuousQueryFilteredEntry() {
+ // No-op.
+ }
+
+ /**
+ * @param e Cache continuous query entry.
+ */
+ CacheContinuousQueryFilteredEntry(CacheContinuousQueryEntry e) {
+ this.cacheId = e.cacheId();
+ this.evtType = e.eventType();
+ this.part = e.partition();
+ this.updateIdx = e.updateIndex();
+ this.topVer = e.topologyVersion();
+ }
+
+ /**
+ * @return Topology version if applicable.
+ */
+ @Nullable AffinityTopologyVersion topologyVersion() {
+ return topVer;
+ }
+
+ /**
+ * @return Cache ID.
+ */
+ int cacheId() {
+ return cacheId;
+ }
+
+ /**
+ * @return Event type.
+ */
+ EventType eventType() {
+ return evtType;
+ }
+
+ /**
+ * @return Partition.
+ */
+ int partition() {
+ return part;
+ }
+
+ /**
+ * @return Update index.
+ */
+ long updateIndex() {
+ return updateIdx;
+ }
+
+ /** {@inheritDoc} */
+ @Override boolean filtered() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte directType() {
+ return 115;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 0:
+ if (!writer.writeInt("cacheId", cacheId))
+ return false;
+
+ writer.incrementState();
+
+ case 1:
+ if (!writer.writeByte("evtType", evtType != null ? (byte)evtType.ordinal() : -1))
+ return false;
+
+ writer.incrementState();
+
+ case 2:
+ if (!writer.writeInt("part", part))
+ return false;
+
+ writer.incrementState();
+
+ case 3:
+ if (!writer.writeLong("updateIdx", updateIdx))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ switch (reader.state()) {
+ case 0:
+ cacheId = reader.readInt("cacheId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 1:
+ byte evtTypeOrd;
+
+ evtTypeOrd = reader.readByte("evtType");
+
+ if (!reader.isLastRead())
+ return false;
+
+ evtType = CacheContinuousQueryEntry.eventTypeFromOrdinal(evtTypeOrd);
+
+ reader.incrementState();
+
+ case 2:
+ part = reader.readInt("part");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 3:
+ updateIdx = reader.readLong("updateIdx");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+ }
+
+ return reader.afterMessageRead(CacheContinuousQueryFilteredEntry.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override void prepareMarshal(GridCacheContext cctx) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override void unmarshal(GridCacheContext cctx, @Nullable ClassLoader ldr) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 4;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(CacheContinuousQueryFilteredEntry.class, this);
+ }
+}
\ No newline at end of file
[8/9] ignite git commit: IGNITE-426 Added cache continuos query
probe. Implemented for TX.
Posted by nt...@apache.org.
IGNITE-426 Added cache continuos query probe. Implemented for TX.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/318607ba
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/318607ba
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/318607ba
Branch: refs/heads/ignite-426-2-reb
Commit: 318607baa1a8d9d43dc4f1dc687a1c2ac6d1f66b
Parents: df45134
Author: nikolay_tikhonov <nt...@gridgain.com>
Authored: Wed Oct 21 15:56:36 2015 +0300
Committer: nikolay_tikhonov <nt...@gridgain.com>
Committed: Wed Oct 21 17:41:21 2015 +0300
----------------------------------------------------------------------
.../processors/cache/GridCacheEntryEx.java | 8 +-
.../processors/cache/GridCacheMapEntry.java | 39 ++--
.../cache/GridCacheUpdateTxResult.java | 30 ++-
.../GridDistributedTxRemoteAdapter.java | 20 +-
.../dht/GridDhtPartitionTopologyImpl.java | 2 +
.../distributed/dht/GridDhtTxFinishFuture.java | 12 +-
.../distributed/dht/GridDhtTxFinishRequest.java | 89 +++++++-
.../continuous/CacheContinuousQueryHandler.java | 30 +--
.../continuous/CacheContinuousQueryManager.java | 3 -
.../cache/transactions/IgniteTxEntry.java | 34 ++-
.../cache/transactions/IgniteTxHandler.java | 3 +
.../transactions/IgniteTxLocalAdapter.java | 18 +-
.../cache/transactions/IgniteTxRemoteEx.java | 7 +-
.../continuous/GridContinuousProcessor.java | 3 -
.../processors/cache/GridCacheTestEntryEx.java | 8 +-
...acheContinuousQueryFailoverAbstractTest.java | 209 ++++++++++++++-----
...ueryFailoverAtomicPrimaryWriteOrderTest.java | 14 +-
...inuousQueryFailoverAtomicReplicatedTest.java | 3 +-
.../CacheContinuousQueryFailoverAtomicTest.java | 39 ----
...CacheContinuousQueryClientReconnectTest.java | 187 +++++++++++++++++
.../IgniteCacheContinuousQueryClientTest.java | 157 ++++++++++++--
...cheContinuousQueryClientTxReconnectTest.java | 32 +++
.../IgniteCacheQuerySelfTestSuite.java | 14 +-
.../yardstick/cache/CacheEntryEventProbe.java | 156 ++++++++++++++
24 files changed, 942 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index a64752b..28c6e86 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -355,7 +355,8 @@ public interface GridCacheEntryEx {
long drExpireTime,
@Nullable GridCacheVersion explicitVer,
@Nullable UUID subjId,
- String taskName
+ String taskName,
+ @Nullable Long updateIdx
) throws IgniteCheckedException, GridCacheEntryRemovedException;
/**
@@ -390,7 +391,8 @@ public interface GridCacheEntryEx {
GridDrType drType,
@Nullable GridCacheVersion explicitVer,
@Nullable UUID subjId,
- String taskName
+ String taskName,
+ @Nullable Long updatePartIdx
) throws IgniteCheckedException, GridCacheEntryRemovedException;
/**
@@ -972,4 +974,4 @@ public interface GridCacheEntryEx {
* Calls {@link GridDhtLocalPartition#onUnlock()} for this entry's partition.
*/
public void onUnlock();
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 570172d..c550e7c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -1016,7 +1016,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
long drExpireTime,
@Nullable GridCacheVersion explicitVer,
@Nullable UUID subjId,
- String taskName
+ String taskName,
+ @Nullable Long updateIdx
) throws IgniteCheckedException, GridCacheEntryRemovedException {
CacheObject old;
@@ -1106,6 +1107,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
updateIdx0 = nextPartIndex(topVer);
+ if (updateIdx != null && updateIdx != 0)
+ updateIdx0 = updateIdx;
+
update(val, expireTime, ttl, newVer);
drReplicate(drType, val, newVer);
@@ -1131,7 +1135,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
subjId, null, taskName);
}
- if (!isNear())
+ if (!isNear() &&
+ // Ignore events on backups for one phase commit.
+ !(tx.onePhaseCommit() && updateIdx != null && updateIdx == 0))
cctx.continuousQueries().onEntryUpdated(this, key, val, old, tx.local(), false, updateIdx0, topVer);
cctx.dataStructures().onEntryUpdated(key, false);
@@ -1148,7 +1154,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (intercept)
cctx.config().getInterceptor().onAfterPut(new CacheLazyEntry(cctx, key, key0, val, val0));
- return valid ? new GridCacheUpdateTxResult(true, retval ? old : null) :
+ return valid ? new GridCacheUpdateTxResult(true, retval ? old : null, updateIdx0) :
new GridCacheUpdateTxResult(false, null);
}
@@ -1174,7 +1180,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
GridDrType drType,
@Nullable GridCacheVersion explicitVer,
@Nullable UUID subjId,
- String taskName
+ String taskName,
+ @Nullable Long updateIdx
) throws IgniteCheckedException, GridCacheEntryRemovedException {
assert cctx.transactional();
@@ -1261,8 +1268,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
updateIdx0 = nextPartIndex(topVer);
-// if (updateIdx != null)
-// updateIdx0 = updateIdx;
+ if (updateIdx != null && updateIdx != 0)
+ updateIdx0 = updateIdx;
drReplicate(drType, null, newVer);
@@ -1296,7 +1303,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
taskName);
}
- if (!isNear())
+ if (!isNear() &&
+ // Ignore events on backups for one phase commit.
+ !(tx.onePhaseCommit() && updateIdx != null && updateIdx == 0))
cctx.continuousQueries().onEntryUpdated(this, key, null, old, tx.local(), false, updateIdx0, topVer);
cctx.dataStructures().onEntryUpdated(key, true);
@@ -1347,7 +1356,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
else
ret = old;
- return new GridCacheUpdateTxResult(true, ret);
+ return new GridCacheUpdateTxResult(true, ret, updateIdx0);
}
else
return new GridCacheUpdateTxResult(false, null);
@@ -1915,7 +1924,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
null,
null,
false,
- updateIdx0);
+ updateIdx0 == null ? 0 : updateIdx0);
}
}
else
@@ -1992,7 +2001,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
null,
null,
false,
- updateIdx0);
+ -1);
}
}
@@ -2040,7 +2049,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
null,
null,
false,
- updateIdx0);
+ updateIdx0 == null ? 0 : updateIdx);
}
}
else
@@ -3124,13 +3133,13 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
else if (deletedUnlocked())
deletedUnlocked(false);
- drReplicate(drType, val, ver);
-
- long updateIdx = -1;
+ long updateIdx = 0;
if (!preload)
updateIdx = nextPartIndex(topVer);
+ drReplicate(drType, val, ver);
+
if (!skipQryNtf) {
cctx.continuousQueries().onEntryUpdated(this, key, val, null, true, preload, updateIdx, topVer);
@@ -4228,4 +4237,4 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
return "IteratorEntry [key=" + key + ']';
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
index ffda7a2..0f63777 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
@@ -32,6 +32,9 @@ public class GridCacheUpdateTxResult {
@GridToStringInclude
private final CacheObject oldVal;
+ /** Partition idx. */
+ private long partIdx;
+
/**
* Constructor.
*
@@ -44,6 +47,31 @@ public class GridCacheUpdateTxResult {
}
/**
+ * Constructor.
+ *
+ * @param success Success flag.
+ * @param oldVal Old value (if any),
+ */
+ GridCacheUpdateTxResult(boolean success, @Nullable CacheObject oldVal, long partIdx) {
+ this.success = success;
+ this.oldVal = oldVal;
+ this.partIdx = partIdx;
+ }
+
+ /**
+ * Sets partition idx.
+ *
+ * @param partIdx Partition idx.
+ */
+ public void partIdx(long partIdx) {
+ this.partIdx = partIdx;
+ }
+
+ public long partIdx() {
+ return partIdx;
+ }
+
+ /**
* @return Success flag.
*/
public boolean success() {
@@ -61,4 +89,4 @@ public class GridCacheUpdateTxResult {
@Override public String toString() {
return S.toString(GridCacheUpdateTxResult.class, this);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index f969737..b24ff10 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -276,6 +276,19 @@ public class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
}
}
+ /** {@inheritDoc} */
+ @Override public void setPartitionUpdateIdx(long[] idxs) {
+ if (writeMap != null && !writeMap.isEmpty() && idxs != null && idxs.length > 0) {
+ int i = 0;
+
+ for (IgniteTxEntry txEntry : writeMap.values()) {
+ txEntry.partIdx(idxs[i]);
+
+ ++i;
+ }
+ }
+ }
+
/**
* Adds completed versions to an entry.
*
@@ -575,13 +588,13 @@ public class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
cached.innerRemove(this, eventNodeId(), nodeId, false, false, true, true,
topVer, null, replicate ? DR_BACKUP : DR_NONE,
near() ? null : explicitVer, CU.subjectId(this, cctx),
- resolveTaskName());
+ resolveTaskName(), txEntry.partIdx());
else {
cached.innerSet(this, eventNodeId(), nodeId, val, false, false,
txEntry.ttl(), true, true, topVer, null,
replicate ? DR_BACKUP : DR_NONE, txEntry.conflictExpireTime(),
near() ? null : explicitVer, CU.subjectId(this, cctx),
- resolveTaskName());
+ resolveTaskName(), txEntry.partIdx());
// Keep near entry up to date.
if (nearCached != null) {
@@ -599,7 +612,8 @@ public class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
else if (op == DELETE) {
cached.innerRemove(this, eventNodeId(), nodeId, false, false, true, true,
topVer, null, replicate ? DR_BACKUP : DR_NONE,
- near() ? null : explicitVer, CU.subjectId(this, cctx), resolveTaskName());
+ near() ? null : explicitVer, CU.subjectId(this, cctx), resolveTaskName(),
+ txEntry.partIdx());
// Keep near entry up to date.
if (nearCached != null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 098a60d..4616b17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -302,6 +302,8 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
long updateSeq = this.updateSeq.incrementAndGet();
+ cntrMap.clear();
+
// If this is the oldest node.
if (oldest.id().equals(loc.id()) || exchFut.isCacheAdded(cctx.cacheId(), exchId.topologyVersion())) {
if (node2part == null) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index 79bccc2..9f2b482 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -31,6 +31,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheFuture;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
@@ -375,6 +376,14 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
add(fut); // Append new future.
+ Collection<Long> updateIdxs = F.transform(dhtMapping.entries(), new C1<IgniteTxEntry, Long>() {
+ @Override public Long apply(IgniteTxEntry entry) {
+ assert entry != null;
+
+ return entry.partIdx();
+ }
+ });
+
GridDhtTxFinishRequest req = new GridDhtTxFinishRequest(
tx.nearNodeId(),
futId,
@@ -397,7 +406,8 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCompoundIdentityFutur
tx.pendingVersions(),
tx.size(),
tx.subjectId(),
- tx.taskNameHash());
+ tx.taskNameHash(),
+ updateIdxs);
req.writeVersion(tx.writeVersion() != null ? tx.writeVersion() : tx.xidVersion());
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
index be59a95..805a557 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
@@ -26,6 +26,7 @@ import org.apache.ignite.internal.GridDirectCollection;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxFinishRequest;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.GridLongList;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteUuid;
@@ -66,6 +67,11 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
/** Check comitted flag. */
private boolean checkCommitted;
+ /** Partition update counter. */
+ @GridToStringInclude
+ @GridDirectCollection(Long.class)
+ private GridLongList partUpdateCnt;
+
/** One phase commit write version. */
private GridCacheVersion writeVer;
@@ -160,6 +166,74 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
}
/**
+ * @param nearNodeId Near node ID.
+ * @param futId Future ID.
+ * @param miniId Mini future ID.
+ * @param topVer Topology version.
+ * @param xidVer Transaction ID.
+ * @param threadId Thread ID.
+ * @param commitVer Commit version.
+ * @param isolation Transaction isolation.
+ * @param commit Commit flag.
+ * @param invalidate Invalidate flag.
+ * @param sys System flag.
+ * @param sysInvalidate System invalidation flag.
+ * @param syncCommit Synchronous commit flag.
+ * @param syncRollback Synchronous rollback flag.
+ * @param baseVer Base version.
+ * @param committedVers Committed versions.
+ * @param rolledbackVers Rolled back versions.
+ * @param pendingVers Pending versions.
+ * @param txSize Expected transaction size.
+ * @param subjId Subject ID.
+ * @param taskNameHash Task name hash.
+ * @param updateIdxs Partition update idxs.
+ */
+ public GridDhtTxFinishRequest(
+ UUID nearNodeId,
+ IgniteUuid futId,
+ IgniteUuid miniId,
+ @NotNull AffinityTopologyVersion topVer,
+ GridCacheVersion xidVer,
+ GridCacheVersion commitVer,
+ long threadId,
+ TransactionIsolation isolation,
+ boolean commit,
+ boolean invalidate,
+ boolean sys,
+ byte plc,
+ boolean sysInvalidate,
+ boolean syncCommit,
+ boolean syncRollback,
+ GridCacheVersion baseVer,
+ Collection<GridCacheVersion> committedVers,
+ Collection<GridCacheVersion> rolledbackVers,
+ Collection<GridCacheVersion> pendingVers,
+ int txSize,
+ @Nullable UUID subjId,
+ int taskNameHash,
+ Collection<Long> updateIdxs
+ ) {
+ this(nearNodeId, futId, miniId, topVer, xidVer, commitVer, threadId, isolation, commit, invalidate, sys, plc,
+ sysInvalidate, syncCommit, syncRollback, baseVer, committedVers, rolledbackVers, pendingVers, txSize,
+ subjId, taskNameHash);
+
+ if (updateIdxs != null && !updateIdxs.isEmpty()) {
+ partUpdateCnt = new GridLongList(updateIdxs.size());
+
+ for (Long idx : updateIdxs)
+ partUpdateCnt.add(idx);
+ }
+ }
+
+ /**
+ * @return Partition update counters.
+ */
+ public GridLongList partUpdateCounters(){
+ return partUpdateCnt;
+ }
+
+ /**
* @return Mini ID.
*/
public IgniteUuid miniId() {
@@ -326,6 +400,11 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
writer.incrementState();
+ case 28:
+ if (!writer.writeMessage("partUpdateCnt", partUpdateCnt))
+ return false;
+
+ writer.incrementState();
}
return true;
@@ -426,6 +505,14 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
reader.incrementState();
+ case 28:
+ partUpdateCnt = reader.readMessage("partUpdateCnt");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
}
return reader.afterMessageRead(GridDhtTxFinishRequest.class);
@@ -438,6 +525,6 @@ public class GridDhtTxFinishRequest extends GridDistributedTxFinishRequest {
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 28;
+ return 29;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 4734998..520dd46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -316,10 +316,8 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
}
}
else {
- locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
-
- if (!skipPrimaryCheck)
- sendBackupAcknowledge(ackBuf.onAcknowledged(entry), routineId, ctx);
+ if (!entry.filtered())
+ locLsnr.onUpdated(F.<CacheEntryEvent<? extends K, ? extends V>>asList(evt));
}
}
else {
@@ -560,13 +558,7 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
rec = oldRec;
}
- Collection<CacheContinuousQueryEntry> entries = rec.collectEntries(e);
-
- if (CacheContinuousQueryManager.SUPER_DEBUG)
- ctx.log(getClass()).error("Fire the following event for partition : " + e.partition() +
- " Entries: " + Arrays.toString(entries.toArray()));
-
- return entries;
+ return rec.collectEntries(e);
}
/**
@@ -608,9 +600,6 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
synchronized (pendingEnts) {
// Received first event.
if (lastFiredEvt == INIT_VALUE) {
- if (CacheContinuousQueryManager.SUPER_DEBUG)
- log.error("First event. " + entry);
-
lastFiredEvt = entry.updateIndex();
firedEvents.add(new T2<>(lastFiredEvt, entry));
@@ -624,29 +613,18 @@ class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler {
lastFiredEvt = 1;
- if (CacheContinuousQueryManager.SUPER_DEBUG)
- log.error("Lost partition. Start from 1. Entry: " + entry);
-
firedEvents.add(new T2<>(lastFiredEvt, entry));
return F.asList(entry);
}
// Check duplicate.
- if (entry.updateIndex() > lastFiredEvt) {
- if (CacheContinuousQueryManager.SUPER_DEBUG)
- log.error("Put message to pending queue. Counter value: " + lastFiredEvt + " Entry: " + entry);
-
+ if (entry.updateIndex() > lastFiredEvt)
pendingEnts.put(entry.updateIndex(), entry);
- }
else {
if (log.isDebugEnabled())
log.debug("Skip duplicate continuous query message: " + entry);
- if (CacheContinuousQueryManager.SUPER_DEBUG)
- log.error("Received duplicate. Counter value: " + lastFiredEvt + " Entry: " + entry
- + ", Proceed message " + Arrays.toString(firedEvents.toArray()));
-
return Collections.emptyList();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 7c04053..680a96c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -65,7 +65,6 @@ import static javax.cache.event.EventType.EXPIRED;
import static javax.cache.event.EventType.REMOVED;
import static javax.cache.event.EventType.UPDATED;
import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_OBJECT_READ;
-import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE;
/**
@@ -87,8 +86,6 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
/** */
private static final long BACKUP_ACK_FREQ = 5000;
- public static final boolean SUPER_DEBUG = false;
-
/** Listeners. */
private final ConcurrentMap<UUID, CacheContinuousQueryListener> lsnrs = new ConcurrentHashMap8<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java
index 3c33d19..541ffae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java
@@ -175,6 +175,9 @@ public class IgniteTxEntry implements GridPeerDeployAware, Message {
*/
private byte flags;
+ /** Partition update index. */
+ private long partIdx;
+
/**
* Required by {@link Externalizable}
*/
@@ -363,6 +366,22 @@ public class IgniteTxEntry implements GridPeerDeployAware, Message {
}
/**
+ * Sets partition index.
+ *
+ * @param partIdx Partition index.
+ */
+ public void partIdx(long partIdx) {
+ this.partIdx = partIdx;
+ }
+
+ /**
+ * @return Partition index.
+ */
+ public long partIdx() {
+ return partIdx;
+ }
+
+ /**
* @param val Value to set.
*/
void setAndMarkValid(CacheObject val) {
@@ -901,6 +920,11 @@ public class IgniteTxEntry implements GridPeerDeployAware, Message {
writer.incrementState();
+ case 11:
+ if (!writer.writeLong("partIdx", partIdx))
+ return false;
+
+ writer.incrementState();
}
return true;
@@ -1002,6 +1026,14 @@ public class IgniteTxEntry implements GridPeerDeployAware, Message {
reader.incrementState();
+ case 11:
+ partIdx = reader.readLong("partIdx");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
}
return reader.afterMessageRead(IgniteTxEntry.class);
@@ -1014,7 +1046,7 @@ public class IgniteTxEntry implements GridPeerDeployAware, Message {
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 11;
+ return 12;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index c2cc629..59209f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -979,6 +979,9 @@ public class IgniteTxHandler {
// Complete remote candidates.
tx.doneRemote(req.baseVersion(), null, null, null);
+ tx.setPartitionUpdateIdx(
+ req.partUpdateCounters() != null ? req.partUpdateCounters().array() : null);
+
tx.commit();
}
else {
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index aa0ffe8..2ea1b61 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -930,7 +930,11 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter
txEntry.conflictExpireTime(),
cached.isNear() ? null : explicitVer,
CU.subjectId(this, cctx),
- resolveTaskName());
+ resolveTaskName(),
+ null);
+
+ if (updRes.success())
+ txEntry.partIdx(updRes.partIdx());
if (nearCached != null && updRes.success())
nearCached.innerSet(
@@ -949,7 +953,8 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter
txEntry.conflictExpireTime(),
null,
CU.subjectId(this, cctx),
- resolveTaskName());
+ resolveTaskName(),
+ null);
}
else if (op == DELETE) {
GridCacheUpdateTxResult updRes = cached.innerRemove(
@@ -965,7 +970,11 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter
cached.detached() ? DR_NONE : drType,
cached.isNear() ? null : explicitVer,
CU.subjectId(this, cctx),
- resolveTaskName());
+ resolveTaskName(),
+ null);
+
+ if (updRes.success())
+ txEntry.partIdx(updRes.partIdx());
if (nearCached != null && updRes.success())
nearCached.innerRemove(
@@ -981,7 +990,8 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter
DR_NONE,
null,
CU.subjectId(this, cctx),
- resolveTaskName());
+ resolveTaskName(),
+ null);
}
else if (op == RELOAD) {
cached.innerReload();
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteEx.java
index 9660e4e..845f4f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxRemoteEx.java
@@ -43,4 +43,9 @@ public interface IgniteTxRemoteEx extends IgniteInternalTx {
* @return {@code True} if entry was found.
*/
public boolean setWriteValue(IgniteTxEntry e);
-}
\ No newline at end of file
+
+ /**
+ * @param idxs Partition update indexes.
+ */
+ public void setPartitionUpdateIdx(long[] idxs);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index c7676d2..3ed186e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -37,7 +37,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.events.CacheRebalancingEvent;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.GridKernalContext;
@@ -74,9 +73,7 @@ import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.thread.IgniteThread;
import org.jetbrains.annotations.Nullable;
import org.jsr166.ConcurrentHashMap8;
-import org.jsr166.ConcurrentLinkedDeque8;
-import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 26d1f91..44984d5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -457,7 +457,8 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
boolean metrics,
AffinityTopologyVersion topVer,
CacheEntryPredicate[] filter, GridDrType drType,
- long drExpireTime, @Nullable GridCacheVersion drVer, UUID subjId, String taskName) throws IgniteCheckedException,
+ long drExpireTime, @Nullable GridCacheVersion drVer, UUID subjId, String taskName, @Nullable Long updateIdx)
+ throws IgniteCheckedException,
GridCacheEntryRemovedException {
return new GridCacheUpdateTxResult(true, rawPut(val, ttl));
}
@@ -530,7 +531,8 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
GridDrType drType,
@Nullable GridCacheVersion drVer,
UUID subjId,
- String taskName
+ String taskName,
+ Long updatePartIdx
) throws IgniteCheckedException, GridCacheEntryRemovedException {
obsoleteVer = ver;
@@ -859,4 +861,4 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
@Override public void onUnlock() {
// No-op.
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
index ca754af..6029761 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractTest.java
@@ -36,6 +36,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import javax.cache.Cache;
+import javax.cache.CacheException;
import javax.cache.event.CacheEntryEvent;
import javax.cache.event.CacheEntryListenerException;
import javax.cache.event.CacheEntryUpdatedListener;
@@ -55,6 +56,7 @@ import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cache.query.ContinuousQuery;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.cluster.ClusterTopologyException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteInternalFuture;
@@ -84,6 +86,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
@@ -297,7 +300,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
IgniteCache<Object, Object> srvCache = igniteSrv.cache(null);
- List<Integer> keys = testKeys(srvCache, 1);
+ List<Integer> keys = testKeys(srvCache, 3);
int keyCnt = keys.size();
@@ -371,6 +374,9 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
* @throws Exception If failed.
*/
public void testLeftPrimaryAndBackupNodes() throws Exception {
+ if (cacheMode() == REPLICATED)
+ return;
+
this.backups = 1;
final int SRV_NODES = 3;
@@ -485,7 +491,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
return qryClient.cluster().nodes().size() == (SRV_NODES + 1 /** client node */)
- 1 /** Primary node */ - backups;
}
- }, 10000L);
+ }, 5000L);
for (; keyIter < keys.size(); keyIter++) {
int key = keys.get(keyIter);
@@ -560,7 +566,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
final List<T3<Object, Object, Object>> expEvts = new ArrayList<>();
- for (int i = 0; i < SRV_NODES - 1; i++) {
+ for (int i = 0; i < (atomicityMode() == CacheAtomicityMode.ATOMIC ? SRV_NODES - 1 : SRV_NODES - 2); i++) {
log.info("Stop iteration: " + i);
TestCommunicationSpi spi = (TestCommunicationSpi)ignite(i).configuration().getCommunicationSpi();
@@ -654,7 +660,8 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
* @throws Exception If failed.
*/
private void checkBackupQueue(int backups, boolean updateFromClient) throws Exception {
- this.backups = backups;
+ this.backups = atomicityMode() == CacheAtomicityMode.ATOMIC ? backups :
+ backups < 2 ? 2 : backups;
final int SRV_NODES = 4;
@@ -668,9 +675,6 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
IgniteCache<Object, Object> qryClientCache = qryClient.cache(null);
- if (cacheMode() != REPLICATED)
- assertEquals(backups, qryClientCache.getConfiguration(CacheConfiguration.class).getBackups());
-
Affinity<Object> aff = qryClient.affinity(null);
CacheEventListener1 lsnr = new CacheEventListener1(false);
@@ -687,7 +691,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
List<T3<Object, Object, Object>> expEvts = new ArrayList<>();
- for (int i = 0; i < SRV_NODES - 1; i++) {
+ for (int i = 0; i < (atomicityMode() == CacheAtomicityMode.ATOMIC ? SRV_NODES - 1 : SRV_NODES - 2); i++) {
log.info("Stop iteration: " + i);
TestCommunicationSpi spi = (TestCommunicationSpi)ignite(i).configuration().getCommunicationSpi();
@@ -709,6 +713,39 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
T2<Object, Object> t = updates.get(key);
+ if (updateFromClient) {
+ if (atomicityMode() == CacheAtomicityMode.TRANSACTIONAL) {
+ try (Transaction tx = qryClient.transactions().txStart()) {
+ qryClientCache.put(key, key);
+
+ tx.commit();
+ }
+ catch (CacheException | ClusterTopologyException e) {
+ log.warning("Failed put. [Key=" + key + ", val=" + key + "]");
+
+ continue;
+ }
+ }
+ else
+ qryClientCache.put(key, key);
+ }
+ else {
+ if (atomicityMode() == CacheAtomicityMode.TRANSACTIONAL) {
+ try (Transaction tx = ignite.transactions().txStart()) {
+ cache.put(key, key);
+
+ tx.commit();
+ }
+ catch (CacheException | ClusterTopologyException e) {
+ log.warning("Failed put. [Key=" + key + ", val=" + key + "]");
+
+ continue;
+ }
+ }
+ else
+ cache.put(key, key);
+ }
+
if (t == null) {
updates.put(key, new T2<>((Object)key, null));
@@ -720,11 +757,6 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
expEvts.add(new T3<>((Object)key, (Object)key, (Object)key));
}
- if (updateFromClient)
- qryClientCache.put(key, key);
- else
- cache.put(key, key);
-
if (first) {
spi.skipMsg = true;
@@ -747,7 +779,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
checkEvents(expEvts, lsnr);
}
- for (int i = 0; i < SRV_NODES - 1; i++) {
+ for (int i = 0; i < (atomicityMode() == CacheAtomicityMode.ATOMIC ? SRV_NODES - 1 : SRV_NODES - 2); i++) {
log.info("Start iteration: " + i);
Ignite ignite = startGrid(i);
@@ -782,7 +814,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
cache.put(key, key);
}
- if (!latch.await(5, SECONDS)) {
+ if (!latch.await(10, SECONDS)) {
Set<Integer> keys0 = new HashSet<>(keys);
keys0.removeAll(lsnr.keys);
@@ -824,7 +856,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
*/
private void checkEvents(final List<T3<Object, Object, Object>> expEvts, final CacheEventListener2 lsnr,
boolean lostAllow) throws Exception {
- boolean b = GridTestUtils.waitForCondition(new PA() {
+ GridTestUtils.waitForCondition(new PA() {
@Override public boolean apply() {
return expEvts.size() == lsnr.size();
}
@@ -910,7 +942,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
for (T3<Object, Object, Object> e : lostEvents)
log.error("Lost event: " + e);
- assertTrue("Lose events, see log for details.", false);
+ fail("Lose events, see log for details.");
}
log.error("Lost event cnt: " + lostEvents.size());
@@ -1155,17 +1187,19 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
* @throws Exception If failed.
*/
public void testFailover() throws Exception {
+ this.backups = 2;
+
final int SRV_NODES = 4;
startGridsMultiThreaded(SRV_NODES);
client = true;
- Ignite qryClient = startGrid(SRV_NODES);
+ final Ignite qryCln = startGrid(SRV_NODES);
client = false;
- IgniteCache<Object, Object> qryClientCache = qryClient.cache(null);
+ final IgniteCache<Object, Object> qryClnCache = qryCln.cache(null);
final CacheEventListener2 lsnr = new CacheEventListener2();
@@ -1173,7 +1207,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
qry.setLocalListener(lsnr);
- QueryCursor<?> cur = qryClientCache.query(qry);
+ QueryCursor<?> cur = qryClnCache.query(qry);
final AtomicBoolean stop = new AtomicBoolean();
@@ -1194,7 +1228,12 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
log.info("Stop node: " + idx);
- stopGrid(idx);
+ try {
+ stopGrid(idx);
+ }
+ catch (Exception e) {
+ log.warning("Failed to stop nodes.", e);
+ }
CountDownLatch latch = new CountDownLatch(1);
@@ -1216,9 +1255,9 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
final Map<Integer, List<T2<Integer, Integer>>> expEvts = new HashMap<>();
try {
- long stopTime = System.currentTimeMillis() + 1 * 60_000;
+ long stopTime = System.currentTimeMillis() + 60_000;
- final int PARTS = qryClient.affinity(null).partitions();
+ final int PARTS = qryCln.affinity(null).partitions();
ThreadLocalRandom rnd = ThreadLocalRandom.current();
@@ -1234,17 +1273,51 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
val = val + 1;
if (processorPut && prevVal != null) {
- qryClientCache.invoke(key, new CacheEntryProcessor<Object, Object, Void>() {
- @Override public Void process(MutableEntry<Object, Object> entry,
- Object... arguments) throws EntryProcessorException {
- entry.setValue(arguments[0]);
+ if (atomicityMode() == CacheAtomicityMode.TRANSACTIONAL) {
+ try (Transaction tx = qryCln.transactions().txStart()) {
+ qryClnCache.invoke(key, new CacheEntryProcessor<Object, Object, Void>() {
+ @Override public Void process(MutableEntry<Object, Object> e,
+ Object... arg) throws EntryProcessorException {
+ e.setValue(arg[0]);
+
+ return null;
+ }
+ }, val);
+
+ tx.commit();
+ }
+ catch (CacheException | ClusterTopologyException e) {
+ log.warning("Failed put. [Key=" + key + ", val=" + val + "]");
- return null;
+ continue;
}
- }, val);
+ }
+ else
+ qryClnCache.invoke(key, new CacheEntryProcessor<Object, Object, Void>() {
+ @Override public Void process(MutableEntry<Object, Object> e,
+ Object... arg) throws EntryProcessorException {
+ e.setValue(arg[0]);
+
+ return null;
+ }
+ }, val);
+ }
+ else {
+ if (atomicityMode() == CacheAtomicityMode.TRANSACTIONAL) {
+ try (Transaction tx = qryCln.transactions().txStart()) {
+ qryClnCache.put(key, val);
+
+ tx.commit();
+ }
+ catch (CacheException | ClusterTopologyException e) {
+ log.warning("Failed put. [Key=" + key + ", val=" + val + "]");
+
+ continue;
+ }
+ }
+ else
+ qryClnCache.put(key, val);
}
- else
- qryClientCache.put(key, val);
processorPut = !processorPut;
@@ -1306,11 +1379,14 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
restartFut.get();
- boolean check = GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- return checkEvents(false, expEvts, lsnr);
- }
- }, 10_000);
+ boolean check = true;
+
+ if (!expEvts.isEmpty())
+ check = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return checkEvents(false, expEvts, lsnr);
+ }
+ }, 10_000);
if (!check)
assertTrue(checkEvents(true, expEvts, lsnr));
@@ -1324,6 +1400,8 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
* @throws Exception If failed.
*/
public void testFailoverFilter() throws Exception {
+ this.backups = 2;
+
final int SRV_NODES = 4;
startGridsMultiThreaded(SRV_NODES);
@@ -1385,7 +1463,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
final Map<Integer, List<T2<Integer, Integer>>> expEvts = new HashMap<>();
try {
- long stopTime = System.currentTimeMillis() + 1 * 60_000;
+ long stopTime = System.currentTimeMillis() + 60_000;
final int PARTS = qryClient.affinity(null).partitions();
@@ -1510,15 +1588,15 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
/**
* @throws Exception If failed.
*/
- public void testFailoverStartStopOneBackup() throws Exception {
- failoverStartStopFilter(1);
+ public void testFailoverStartStopBackup() throws Exception {
+ failoverStartStopFilter(atomicityMode() == CacheAtomicityMode.ATOMIC ? 1 : 2);
}
/**
* @throws Exception If failed.
*/
- public void _testStartStop() throws Exception {
- this.backups = 0;
+ public void testStartStop() throws Exception {
+ this.backups = 2;
final int SRV_NODES = 4;
@@ -1532,6 +1610,8 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
IgniteCache<Object, Object> qryClnCache = qryClient.cache(null);
+ Affinity<Object> aff = qryClient.affinity(null);
+
final CacheEventListener2 lsnr = new CacheEventListener2();
ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
@@ -1542,18 +1622,18 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
QueryCursor<?> cur = qryClnCache.query(qry);
- for (int i = 0; i < 100; i++) {
+ for (int i = 0; i < 20; i++) {
final int idx = i % (SRV_NODES - 1);
log.info("Stop node: " + idx);
stopGrid(idx);
- Thread.sleep(200);
+ awaitPartitionMapExchange();
List<T3<Object, Object, Object>> afterRestEvents = new ArrayList<>();
- for (int j = 0; j < 10; j++) {
+ for (int j = 0; j < aff.partitions(); j++) {
Integer oldVal = (Integer)qryClnCache.get(j);
qryClnCache.put(j, i);
@@ -1646,7 +1726,7 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
final List<T3<Object, Object, Object>> expEvtsLsnr = new ArrayList<>();
try {
- long stopTime = System.currentTimeMillis() + 60_000;
+ long stopTime = System.currentTimeMillis() + 10_000;
// Start new filter each 5 sec.
long startFilterTime = System.currentTimeMillis() + 5_000;
@@ -1785,13 +1865,11 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
dinLsnr.evts.clear();
dinLsnr.vals.clear();
-
- dinQry.close();
}
List<T3<Object, Object, Object>> afterRestEvents = new ArrayList<>();
- for (int i = 0; i < 1024; i++) {
+ for (int i = 0; i < qryClient.affinity(null).partitions(); i++) {
Integer oldVal = (Integer)qryClnCache.get(i);
qryClnCache.put(i, i);
@@ -1801,12 +1879,13 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
checkEvents(new ArrayList<>(afterRestEvents), lsnr, false);
- //checkEvents(new ArrayList<>(afterRestEvents), dinLsnr, false);
-
cur.close();
- if (dinQry != null)
+ if (dinQry != null) {
+ checkEvents(new ArrayList<>(afterRestEvents), dinLsnr, false);
+
dinQry.close();
+ }
assertFalse("Unexpected error during test, see log for details.", err);
}
@@ -1815,6 +1894,8 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
* @throws Exception If failed.
*/
public void testMultiThreaded() throws Exception {
+ this.backups = 2;
+
final int SRV_NODES = 3;
startGridsMultiThreaded(SRV_NODES);
@@ -1957,8 +2038,24 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
T2<Integer, Integer> expEvt = exp.get(i);
CacheEntryEvent<?, ?> rcvdEvt = rcvdEvts.get(i);
- assertEquals(key, rcvdEvt.getKey());
- assertEquals(expEvt.get1(), rcvdEvt.getValue());
+ if (pass) {
+ assertEquals(key, rcvdEvt.getKey());
+ assertEquals(expEvt.get1(), rcvdEvt.getValue());
+ }
+ else {
+ if (!key.equals(rcvdEvt.getKey()) || !expEvt.get1().equals(rcvdEvt.getValue()))
+ log.warning("Missed events. [key=" + key + ", actKey=" + rcvdEvt.getKey()
+ + ", expVal=" + expEvt.get1() + ", actVal=" + rcvdEvt.getValue() + "]");
+ }
+ }
+
+ if (!pass) {
+ for (int i = cnt; i < exp.size(); i++) {
+ T2<Integer, Integer> val = exp.get(i);
+
+ log.warning("Missed events. [key=" + key + ", expVal=" + val.get1()
+ + ", prevVal=" + val.get2() + "]");
+ }
}
}
}
@@ -2168,7 +2265,8 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
if (msg0 instanceof GridContinuousMessage) {
if (skipMsg) {
- log.info("Skip continuous message: " + msg0);
+ if (log.isDebugEnabled())
+ log.debug("Skip continuous message: " + msg0);
return;
}
@@ -2176,7 +2274,8 @@ public abstract class CacheContinuousQueryFailoverAbstractTest extends GridCommo
AtomicBoolean sndFirstOnly = this.sndFirstOnly;
if (sndFirstOnly != null && !sndFirstOnly.compareAndSet(false, true)) {
- log.info("Skip continuous message: " + msg0);
+ if (log.isDebugEnabled())
+ log.debug("Skip continuous message: " + msg0);
return;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest.java
index 4ddcf0d..8bd7ea7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest.java
@@ -18,15 +18,27 @@
package org.apache.ignite.internal.processors.cache.query.continuous;
import org.apache.ignite.cache.CacheAtomicWriteOrderMode;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
import static org.apache.ignite.cache.CacheAtomicWriteOrderMode.PRIMARY;
/**
*
*/
-public class CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest extends CacheContinuousQueryFailoverAtomicTest {
+public class CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest extends CacheContinuousQueryFailoverAbstractTest {
/** {@inheritDoc} */
@Override protected CacheAtomicWriteOrderMode writeOrderMode() {
return PRIMARY;
}
+
+ /** {@inheritDoc} */
+ @Override protected CacheMode cacheMode() {
+ return CacheMode.PARTITIONED;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicityMode atomicityMode() {
+ return CacheAtomicityMode.ATOMIC;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicReplicatedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicReplicatedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicReplicatedTest.java
index 8fc58d3..db5b8cb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicReplicatedTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicReplicatedTest.java
@@ -26,7 +26,8 @@ import static org.apache.ignite.cache.CacheMode.REPLICATED;
/**
*
*/
-public class CacheContinuousQueryFailoverAtomicReplicatedTest extends CacheContinuousQueryFailoverAtomicTest {
+public class CacheContinuousQueryFailoverAtomicReplicatedTest
+ extends CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest {
/** {@inheritDoc} */
@Override protected CacheMode cacheMode() {
return REPLICATED;
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicTest.java
deleted file mode 100644
index fb50387..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAtomicTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.query.continuous;
-
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
-
-/**
- *
- */
-public class CacheContinuousQueryFailoverAtomicTest extends CacheContinuousQueryFailoverAbstractTest {
- /** {@inheritDoc} */
- @Override protected CacheMode cacheMode() {
- return PARTITIONED;
- }
-
- /** {@inheritDoc} */
- @Override protected CacheAtomicityMode atomicityMode() {
- return ATOMIC;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java
new file mode 100644
index 0000000..560f2e0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import java.util.concurrent.CountDownLatch;
+import javax.cache.event.CacheEntryEvent;
+import javax.cache.event.CacheEntryUpdatedListener;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.query.ContinuousQuery;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteClientReconnectAbstractTest;
+import org.apache.ignite.resources.LoggerResource;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class IgniteCacheContinuousQueryClientReconnectTest extends IgniteClientReconnectAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected int serverCount() {
+ return 4;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int clientCount() {
+ return 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ ccfg.setCacheMode(PARTITIONED);
+ ccfg.setAtomicityMode(atomicMode());
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ return cfg;
+ }
+
+ /**
+ * @return Atomic mode.
+ */
+ protected CacheAtomicityMode atomicMode() {
+ return ATOMIC;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReconnectClient() throws Exception {
+ Ignite client = grid(serverCount());
+
+ Ignite srv = clientRouter(client);
+
+ assertTrue(client.cluster().localNode().isClient());
+
+ final CacheEventListener lsnr = new CacheEventListener();
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ IgniteCache<Object, Object> clnCache = client.cache(null);
+
+ QueryCursor<?> cur = clnCache.query(qry);
+
+ int keyCnt = 100;
+
+ for (int i = 0; i < 30; i++) {
+ lsnr.latch = new CountDownLatch(keyCnt);
+
+ for (int key = 0; key < keyCnt; key++)
+ clnCache.put(key, key);
+
+ assertTrue("Failed to wait for event.", lsnr.latch.await(5, SECONDS));
+
+ reconnectClientNode(client, srv, null);
+
+ lsnr.latch = new CountDownLatch(keyCnt);
+
+ for (int key = 0; key < keyCnt; key++)
+ clnCache.put(key, key);
+
+ assertTrue("Failed to wait for event.", lsnr.latch.await(5, SECONDS));
+ }
+
+ cur.close();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReconnectClientAndLeftRouter() throws Exception {
+ Ignite client = grid(serverCount());
+
+ final Ignite srv = clientRouter(client);
+
+ final String clnRouterName = srv.name();
+
+ assertTrue(client.cluster().localNode().isClient());
+
+ final CacheEventListener lsnr = new CacheEventListener();
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ IgniteCache<Object, Object> clnCache = client.cache(null);
+
+ QueryCursor<?> cur = clnCache.query(qry);
+
+ int keyCnt = 100;
+
+ lsnr.latch = new CountDownLatch(keyCnt);
+
+ for (int key = 0; key < keyCnt; key++)
+ clnCache.put(key, key);
+
+ assertTrue("Failed to wait for event.", lsnr.latch.await(5, SECONDS));
+
+ reconnectClientNode(client, srv, new Runnable() {
+ @Override public void run() {
+ stopGrid(clnRouterName);
+ }
+ });
+
+ assertFalse("Client connected to the same server node.", clnRouterName.equals(clientRouter(client).name()));
+
+ lsnr.latch = new CountDownLatch(keyCnt);
+
+ for (int key = 0; key < keyCnt; key++)
+ clnCache.put(key, key);
+
+ assertTrue("Failed to wait for event.", lsnr.latch.await(5, SECONDS));
+
+ cur.close();
+ }
+
+ /**
+ *
+ */
+ private static class CacheEventListener implements CacheEntryUpdatedListener<Object, Object> {
+ /** */
+ private volatile CountDownLatch latch = new CountDownLatch(1);
+
+ /** */
+ @LoggerResource
+ private IgniteLogger log;
+
+ /** {@inheritDoc} */
+ @Override public void onUpdated(Iterable<CacheEntryEvent<?, ?>> evts) {
+ for (CacheEntryEvent<?, ?> evt : evts) {
+ log.info("Received cache event: " + evt);
+
+ latch.countDown();
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientTest.java
index 1afeb05..534f298 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientTest.java
@@ -27,11 +27,13 @@ import org.apache.ignite.cache.query.ContinuousQuery;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.resources.LoggerResource;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import static java.util.concurrent.TimeUnit.SECONDS;
@@ -83,11 +85,13 @@ public class IgniteCacheContinuousQueryClientTest extends GridCommonAbstractTest
client = true;
- Ignite clientNode = startGrid(3);
+ final int CLIENT_ID = 3;
+
+ Ignite clientNode = startGrid(CLIENT_ID);
client = false;
- CacheEventListener lsnr = new CacheEventListener();
+ final CacheEventListener lsnr = new CacheEventListener();
ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
@@ -95,27 +99,154 @@ public class IgniteCacheContinuousQueryClientTest extends GridCommonAbstractTest
QueryCursor<?> cur = clientNode.cache(null).query(qry);
- Ignite joined1 = startGrid(4);
+ for (int i = 0; i < 10; i++) {
+ log.info("Start iteration: " + i);
+
+ lsnr.latch = new CountDownLatch(1);
+
+ Ignite joined1 = startGrid(4);
+
+ IgniteCache<Object, Object> joinedCache1 = joined1.cache(null);
+
+ joinedCache1.put(primaryKey(joinedCache1), 1);
+
+ assertTrue("Failed to wait for event.", lsnr.latch.await(5, SECONDS));
+
+ lsnr.latch = new CountDownLatch(1);
+
+ Ignite joined2 = startGrid(5);
+
+ IgniteCache<Object, Object> joinedCache2 = joined2.cache(null);
- IgniteCache<Object, Object> joinedCache1 = joined1.cache(null);
+ joinedCache2.put(primaryKey(joinedCache2), 2);
- joinedCache1.put(primaryKey(joinedCache1), 1);
+ assertTrue("Failed to wait for event.", lsnr.latch.await(5, SECONDS));
- assertTrue("Failed to wait for event.", lsnr.latch.await(5, SECONDS));
+ stopGrid(4);
+
+ stopGrid(5);
+ }
cur.close();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNodeJoinsRestartQuery() throws Exception {
+ startGrids(2);
+
+ client = true;
+
+ final int CLIENT_ID = 3;
+
+ Ignite clientNode = startGrid(CLIENT_ID);
+
+ client = false;
+
+ for (int i = 0; i < 10; i++) {
+ log.info("Start iteration: " + i);
+
+ final CacheEventListener lsnr = new CacheEventListener();
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ QueryCursor<?> cur = clientNode.cache(null).query(qry);
+
+ lsnr.latch = new CountDownLatch(1);
+
+ Ignite joined1 = startGrid(4);
+
+ IgniteCache<Object, Object> joinedCache1 = joined1.cache(null);
+
+ joinedCache1.put(primaryKey(joinedCache1), 1);
+
+ assertTrue("Failed to wait for event.", lsnr.latch.await(5, SECONDS));
+
+ cur.close();
- lsnr.latch = new CountDownLatch(1);
+ lsnr.latch = new CountDownLatch(1);
- Ignite joined2 = startGrid(5);
+ Ignite joined2 = startGrid(5);
- IgniteCache<Object, Object> joinedCache2 = joined2.cache(null);
+ IgniteCache<Object, Object> joinedCache2 = joined2.cache(null);
- joinedCache2.put(primaryKey(joinedCache2), 2);
+ joinedCache2.put(primaryKey(joinedCache2), 2);
- U.sleep(1000);
+ assertFalse("Unexpected event received.", GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return 1 != lsnr.latch.getCount();
+ }
+ }, 1000));
- assertEquals("Unexpected event received.", 1, lsnr.latch.getCount());
+ stopGrid(4);
+
+ stopGrid(5);
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testServerNodeLeft() throws Exception {
+ startGrids(3);
+
+ client = true;
+
+ final int CLIENT_ID = 3;
+
+ Ignite clnNode = startGrid(CLIENT_ID);
+
+ client = false;
+
+ IgniteOutClosure<IgniteCache<Integer, Integer>> rndCache =
+ new IgniteOutClosure<IgniteCache<Integer, Integer>>() {
+ int cnt = 0;
+
+ @Override public IgniteCache<Integer, Integer> apply() {
+ ++cnt;
+
+ return grid(CLIENT_ID).cache(null);
+ }
+ };
+
+ final CacheEventListener lsnr = new CacheEventListener();
+
+ ContinuousQuery<Object, Object> qry = new ContinuousQuery<>();
+
+ qry.setLocalListener(lsnr);
+
+ QueryCursor<?> cur = clnNode.cache(null).query(qry);
+
+ boolean first = true;
+
+ int keyCnt = 1;
+
+ for (int i = 0; i < 10; i++) {
+ log.info("Start iteration: " + i);
+
+ if (first)
+ first = false;
+ else {
+ for (int srv = 0; srv < CLIENT_ID - 1; srv++)
+ startGrid(srv);
+ }
+
+ lsnr.latch = new CountDownLatch(keyCnt);
+
+ for (int key = 0; key < keyCnt; key++)
+ rndCache.apply().put(key, key);
+
+ assertTrue("Failed to wait for event. Left events: " + lsnr.latch.getCount(),
+ lsnr.latch.await(10, SECONDS));
+
+ for (int srv = 0; srv < CLIENT_ID - 1; srv++)
+ stopGrid(srv);
+ }
+
+ cur.close();
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientTxReconnectTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientTxReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientTxReconnectTest.java
new file mode 100644
index 0000000..a10ebc9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientTxReconnectTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query.continuous;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+
+/**
+ *
+ */
+public class IgniteCacheContinuousQueryClientTxReconnectTest extends IgniteCacheContinuousQueryClientReconnectTest {
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicityMode atomicMode() {
+ return TRANSACTIONAL;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/318607ba/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 6cb1a52..91dc388 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -65,6 +65,10 @@ import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalAtomicQ
import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalFieldsQuerySelfTest;
import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalQuerySelfTest;
import org.apache.ignite.internal.processors.cache.query.GridCacheSwapScanQuerySelfTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverAtomicReplicatedTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverTxReplicatedTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryFailoverTxTest;
import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicNearEnabledSelfTest;
import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicP2PDisabledSelfTest;
import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryAtomicSelfTest;
@@ -77,7 +81,9 @@ import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheCon
import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedOneNodeSelfTest;
import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedP2PDisabledSelfTest;
import org.apache.ignite.internal.processors.cache.query.continuous.GridCacheContinuousQueryReplicatedSelfTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryClientReconnectTest;
import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryClientTest;
+import org.apache.ignite.internal.processors.cache.query.continuous.IgniteCacheContinuousQueryClientTxReconnectTest;
import org.apache.ignite.internal.processors.cache.reducefields.GridCacheReduceFieldsQueryAtomicSelfTest;
import org.apache.ignite.internal.processors.cache.reducefields.GridCacheReduceFieldsQueryLocalSelfTest;
import org.apache.ignite.internal.processors.cache.reducefields.GridCacheReduceFieldsQueryPartitionedSelfTest;
@@ -160,8 +166,14 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
suite.addTestSuite(GridCacheContinuousQueryAtomicSelfTest.class);
suite.addTestSuite(GridCacheContinuousQueryAtomicNearEnabledSelfTest.class);
suite.addTestSuite(GridCacheContinuousQueryAtomicP2PDisabledSelfTest.class);
- suite.addTestSuite(IgniteCacheContinuousQueryClientTest.class);
suite.addTestSuite(GridCacheContinuousQueryReplicatedOneNodeSelfTest.class);
+ suite.addTestSuite(IgniteCacheContinuousQueryClientTest.class);
+ suite.addTestSuite(IgniteCacheContinuousQueryClientReconnectTest.class);
+ suite.addTestSuite(IgniteCacheContinuousQueryClientTxReconnectTest.class);
+ suite.addTestSuite(CacheContinuousQueryFailoverAtomicPrimaryWriteOrderTest.class);
+ suite.addTestSuite(CacheContinuousQueryFailoverAtomicReplicatedTest.class);
+ suite.addTestSuite(CacheContinuousQueryFailoverTxTest.class);
+ suite.addTestSuite(CacheContinuousQueryFailoverTxReplicatedTest.class);
// Reduce fields queries.
suite.addTestSuite(GridCacheReduceFieldsQueryLocalSelfTest.class);
[9/9] ignite git commit: IGNITE-426 Cleanup code.
Posted by nt...@apache.org.
IGNITE-426 Cleanup code.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1725d51f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1725d51f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1725d51f
Branch: refs/heads/ignite-426-2-reb
Commit: 1725d51f4d0ff7f1fe2e554974bc41324974915a
Parents: 318607b
Author: nikolay_tikhonov <nt...@gridgain.com>
Authored: Thu Oct 22 13:30:26 2015 +0300
Committer: nikolay_tikhonov <nt...@gridgain.com>
Committed: Thu Oct 22 13:30:26 2015 +0300
----------------------------------------------------------------------
.../processors/cache/GridCacheMapEntry.java | 10 ++---
.../cache/GridCacheUpdateTxResult.java | 8 +---
.../dht/GridClientPartitionTopology.java | 2 -
.../distributed/dht/GridDhtLocalPartition.java | 40 +++++++++-----------
.../dht/GridDhtPartitionTopologyImpl.java | 6 +--
.../dht/atomic/GridDhtAtomicCache.java | 12 +++++-
.../dht/atomic/GridDhtAtomicUpdateFuture.java | 6 +--
.../dht/atomic/GridDhtAtomicUpdateRequest.java | 19 +++++++++-
.../distributed/near/GridNearAtomicCache.java | 2 +-
.../continuous/CacheContinuousQueryEntry.java | 7 ++++
.../continuous/CacheContinuousQueryHandler.java | 1 -
.../CacheContinuousQueryListener.java | 4 +-
.../continuous/CacheContinuousQueryManager.java | 3 +-
13 files changed, 65 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index c550e7c..061d20a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -1707,7 +1707,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
CacheObject oldVal;
CacheObject updated;
- GridCacheVersion rmvVer = null;
+ GridCacheVersion enqueueVer = null;
GridCacheVersionConflictContext<?, ?> conflictCtx = null;
@@ -2264,7 +2264,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
}
- rmvVer = newVer;
+ enqueueVer = newVer;
boolean hasValPtr = hasOffHeapPointer();
@@ -2343,7 +2343,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
invokeRes,
newSysTtl,
newSysExpireTime,
- rmvVer,
+ enqueueVer,
conflictCtx,
true,
updateIdx0);
@@ -4045,9 +4045,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
*/
protected void deletedUnlocked(boolean deleted) {
assert Thread.holdsLock(this);
-
- if (!cctx.deferredDelete())
- return;
+ assert cctx.deferredDelete();
if (deleted) {
assert !deletedUnlocked() : this;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
index 0f63777..bea1000 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
@@ -59,14 +59,8 @@ public class GridCacheUpdateTxResult {
}
/**
- * Sets partition idx.
- *
- * @param partIdx Partition idx.
+ * @return Partition idx.
*/
- public void partIdx(long partIdx) {
- this.partIdx = partIdx;
- }
-
public long partIdx() {
return partIdx;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 516b7bd..217073a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -882,8 +882,6 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
/** {@inheritDoc} */
@Override public boolean rebalanceFinished(AffinityTopologyVersion topVer) {
- assert false;
-
return false;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index baa8520..410b2f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -17,18 +17,6 @@
package org.apache.ignite.internal.processors.cache.distributed.dht;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.atomic.AtomicStampedReference;
-import java.util.concurrent.locks.ReentrantLock;
-import javax.cache.CacheException;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.IgniteInternalFuture;
@@ -58,6 +46,19 @@ import org.jetbrains.annotations.NotNull;
import org.jsr166.ConcurrentHashMap8;
import org.jsr166.LongAdder8;
+import javax.cache.CacheException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicStampedReference;
+import java.util.concurrent.locks.ReentrantLock;
+
import static org.apache.ignite.IgniteSystemProperties.IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE;
import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_OBJECT_UNLOADED;
import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
@@ -111,7 +112,7 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
private final LongAdder8 mapPubSize = new LongAdder8();
/** Remove queue. */
- private GridCircularBuffer<T2<KeyCacheObject, GridCacheVersion>> rmvQueue;
+ private final GridCircularBuffer<T2<KeyCacheObject, GridCacheVersion>> rmvQueue;
/** Group reservations. */
private final CopyOnWriteArrayList<GridDhtPartitionsReservation> reservations = new CopyOnWriteArrayList<>();
@@ -144,8 +145,7 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
int delQueueSize = CU.isSystemCache(cctx.name()) ? 100 :
Math.max(MAX_DELETE_QUEUE_SIZE / cctx.affinity().partitions(), 20);
- if (cctx.deferredDelete())
- rmvQueue = new GridCircularBuffer<>(U.ceilPow2(delQueueSize));
+ rmvQueue = new GridCircularBuffer<>(U.ceilPow2(delQueueSize));
}
/**
@@ -299,8 +299,6 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
* @throws IgniteCheckedException If failed.
*/
public void onDeferredDelete(KeyCacheObject key, GridCacheVersion ver) throws IgniteCheckedException {
- assert cctx.deferredDelete();
-
try {
T2<KeyCacheObject, GridCacheVersion> evicted = rmvQueue.add(new T2<>(key, ver));
@@ -502,8 +500,7 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
((GridDhtPreloader)cctx.preloader()).onPartitionEvicted(this, updateSeq);
- if (cctx.deferredDelete())
- clearDeferredDeletes();
+ clearDeferredDeletes();
return new GridFinishedFuture<>(true);
}
@@ -556,8 +553,7 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
((GridDhtPreloader)cctx.preloader()).onPartitionEvicted(this, updateSeq);
- if (cctx.deferredDelete())
- clearDeferredDeletes();
+ clearDeferredDeletes();
return true;
}
@@ -792,8 +788,6 @@ public class GridDhtLocalPartition implements Comparable<GridDhtLocalPartition>,
*
*/
private void clearDeferredDeletes() {
- assert cctx.deferredDelete();
-
rmvQueue.forEach(new CI1<T2<KeyCacheObject, GridCacheVersion>>() {
@Override public void apply(T2<KeyCacheObject, GridCacheVersion> t) {
cctx.dht().removeVersionedEntry(t.get1(), t.get2());
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 4616b17..1195ddd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -229,7 +229,7 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
topReadyFut = exchFut;
- rebalancedTopVer = AffinityTopologyVersion.NONE;;
+ rebalancedTopVer = AffinityTopologyVersion.NONE;
}
finally {
lock.writeLock().unlock();
@@ -1339,13 +1339,13 @@ class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
/** {@inheritDoc} */
@Override public void printMemoryStats(int threshold) {
- X.println(">>> Cache partition topology stats [grid=" + cctx.gridName() + ", cache=" + cctx.name() + ']');
+ X.println(">>> Cache partition topology stats [grid=" + cctx.gridName() + ", cache=" + cctx.name() + ']');
for (GridDhtLocalPartition part : locParts.values()) {
int size = part.size();
if (size >= threshold)
- X.println(">>> Local partition [part=" + part.id() + ", size=" + size + ']');
+ X.println(">>> Local partition [part=" + part.id() + ", size=" + size + ']');
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index a487593..a9ee79a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1815,10 +1815,12 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
else if (conflictCtx.isMerge())
newConflictVer = null; // Conflict version is discarded in case of merge.
+ EntryProcessor<Object, Object, Object> entryProcessor = null;
+
if (!readersOnly) {
dhtFut.addWriteEntry(entry,
updRes.newValue(),
- op == TRANSFORM ? req.entryProcessor(i) : null,
+ entryProcessor,
updRes.newTtl(),
updRes.conflictExpireTime(),
newConflictVer,
@@ -1831,6 +1833,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
dhtFut.addNearWriteEntries(filteredReaders,
entry,
updRes.newValue(),
+ entryProcessor,
updRes.newTtl(),
updRes.conflictExpireTime());
}
@@ -2101,10 +2104,13 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
}
if (dhtFut != null) {
+ EntryProcessor<Object, Object, Object> entryProcessor =
+ entryProcessorMap == null ? null : entryProcessorMap.get(entry.key());
+
if (!batchRes.readersOnly())
dhtFut.addWriteEntry(entry,
writeVal,
- entryProcessorMap == null ? null : entryProcessorMap.get(entry.key()),
+ entryProcessor,
updRes.newTtl(),
CU.EXPIRE_TIME_CALCULATE,
null,
@@ -2116,6 +2122,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
dhtFut.addNearWriteEntries(filteredReaders,
entry,
writeVal,
+ entryProcessor,
updRes.newTtl(),
CU.EXPIRE_TIME_CALCULATE);
}
@@ -2519,6 +2526,7 @@ public class GridDhtAtomicCache<K, V> extends GridDhtCacheAdapter<K, V> {
CacheObject val = req.value(i);
CacheObject prevVal = req.previousValue(i);
+
EntryProcessor<Object, Object, Object> entryProcessor = req.entryProcessor(i);
Long updateIdx = req.updateIdx(i);
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
index 4019579..f48b0e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateFuture.java
@@ -133,9 +133,6 @@ public class GridDhtAtomicUpdateFuture extends GridFutureAdapter<Void>
boolean topLocked = updateReq.topologyLocked() || (updateReq.fastMap() && !updateReq.clientRequest());
waitForExchange = !topLocked;
-
- // We can send entry processor instead of value to backup if updates are ordered.
- forceTransformBackups = updateReq.operation() == GridCacheOperation.TRANSFORM;
}
/** {@inheritDoc} */
@@ -265,12 +262,14 @@ public class GridDhtAtomicUpdateFuture extends GridFutureAdapter<Void>
* @param readers Entry readers.
* @param entry Entry.
* @param val Value.
+ * @param entryProcessor Entry processor..
* @param ttl TTL for near cache update (optional).
* @param expireTime Expire time for near cache update (optional).
*/
public void addNearWriteEntries(Iterable<UUID> readers,
GridDhtCacheEntry entry,
@Nullable CacheObject val,
+ EntryProcessor<Object, Object, Object> entryProcessor,
long ttl,
long expireTime) {
CacheWriteSynchronizationMode syncMode = updateReq.writeSynchronizationMode();
@@ -311,6 +310,7 @@ public class GridDhtAtomicUpdateFuture extends GridFutureAdapter<Void>
updateReq.addNearWriteValue(entry.key(),
val,
+ entryProcessor,
ttl,
expireTime);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
index 380b194..00513a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
@@ -295,21 +295,36 @@ public class GridDhtAtomicUpdateRequest extends GridCacheMessage implements Grid
/**
* @param key Key to add.
* @param val Value, {@code null} if should be removed.
+ * @param entryProcessor Entry processor.
* @param ttl TTL.
* @param expireTime Expire time.
*/
public void addNearWriteValue(KeyCacheObject key,
@Nullable CacheObject val,
+ EntryProcessor<Object, Object, Object> entryProcessor,
long ttl,
long expireTime)
{
if (nearKeys == null) {
nearKeys = new ArrayList<>();
- nearVals = new ArrayList<>();
+
+ if (forceTransformBackups) {
+ nearEntryProcessors = new ArrayList<>();
+ nearEntryProcessorsBytes = new ArrayList<>();
+ }
+ else
+ nearVals = new ArrayList<>();
}
nearKeys.add(key);
- nearVals.add(val);
+
+ if (forceTransformBackups) {
+ assert entryProcessor != null;
+
+ nearEntryProcessors.add(entryProcessor);
+ }
+ else
+ nearVals.add(val);
if (ttl >= 0) {
if (nearTtls == null) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index eaeb5f7..5373b2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -361,7 +361,7 @@ public class GridNearAtomicCache<K, V> extends GridNearCacheAdapter<K, V> {
expireTime,
null,
false,
- /*intercept*/false,
+ intercept,
req.subjectId(),
taskName,
null,
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index eefbbae..d96c824 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -143,6 +143,13 @@ public class CacheContinuousQueryEntry implements GridCacheDeployable, Message {
}
/**
+ * @return Cache ID.
+ */
+ int cacheId() {
+ return cacheId;
+ }
+
+ /**
* @return Event type.
*/
EventType eventType() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 520dd46..5298a2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
index 2f9e111..4937ee7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryListener.java
@@ -38,9 +38,7 @@ interface CacheContinuousQueryListener<K, V> {
* @param primary Primary flag.
* @param recordIgniteEvt Whether to record event.
*/
- public void onEntryUpdated(CacheContinuousQueryEvent<K, V> evt,
- boolean primary,
- boolean recordIgniteEvt);
+ public void onEntryUpdated(CacheContinuousQueryEvent<K, V> evt, boolean primary, boolean recordIgniteEvt);
/**
* Listener unregistered callback.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1725d51f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 680a96c..b8922a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -183,7 +183,6 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
{
assert e != null;
assert key != null;
- assert Thread.holdsLock(e) : e;
boolean internal = e.isInternal() || !e.context().userCache();
@@ -663,7 +662,7 @@ public class CacheContinuousQueryManager extends GridCacheManagerAdapter {
CacheEntryEventFilter fltr = null;
if (cfg.getCacheEntryEventFilterFactory() != null) {
- fltr = (CacheEntryEventFilter) cfg.getCacheEntryEventFilterFactory().create();
+ fltr = (CacheEntryEventFilter)cfg.getCacheEntryEventFilterFactory().create();
if (!(fltr instanceof Serializable))
throw new IgniteCheckedException("Cache entry event filter must implement java.io.Serializable: "