[jira] [Created] (IGNITE-6507) Commit can be lost in network split scenario

["Alexei Scherbakov (JIRA)" <ji...@apache.org>]

Alexei Scherbakov created IGNITE-6507:
-----------------------------------------

             Summary: Commit can be lost in network split scenario
                 Key: IGNITE-6507
                 URL: https://issues.apache.org/jira/browse/IGNITE-6507
             Project: Ignite
          Issue Type: Bug
          Components: general
    Affects Versions: 2.1
            Reporter: Alexei Scherbakov
            Priority: Critical
             Fix For: 2.3


Commit can be lost in network split scenario

{noformat}
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.ignite.internal.processors.cache.distributed.dht;

import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.BinaryConfiguration;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.TestRecordingCommunicationSpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;

import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;

/**
 * Tests commit consitency in split-brain scenario.
 */
public class GridCacheGridSplitTxConsistencyTest extends GridCommonAbstractTest {
    /** */
    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);

    /**
     * {@inheritDoc}
     */
    @Override protected void afterTest() throws Exception {
        super.afterTest();

        stopAllGrids();

        GridTestUtils.deleteDbFiles();
    }

    /**
     * {@inheritDoc}
     */
    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
        IgniteConfiguration cfg = super.getConfiguration(gridName);

        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());

        cfg.setConsistentId(gridName);

        MemoryConfiguration memCfg = new MemoryConfiguration();
        memCfg.setPageSize(1024);
        memCfg.setDefaultMemoryPolicySize(100 * 1024 * 1024);

        cfg.setMemoryConfiguration(memCfg);

        ((TcpDiscoverySpi) cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);

        CacheConfiguration ccfg = new CacheConfiguration();
        ccfg.setName(DEFAULT_CACHE_NAME);
        ccfg.setAtomicityMode(TRANSACTIONAL);
        ccfg.setWriteSynchronizationMode(FULL_SYNC);
        ccfg.setAffinity(new RendezvousAffinityFunction(false, 3));
        ccfg.setBackups(2);

        cfg.setCacheConfiguration(ccfg);

        return cfg;
    }

    /**
     * Tests if commits are working as expected.
     * @throws Exception
     */
    public void testSplitTxConsistency() throws Exception {
        IgniteEx grid0 = startGrid(0);
        grid0.active(true);

        IgniteEx grid1 = startGrid(1);
        IgniteEx grid2 = startGrid(2);

        int key = 0;

        Affinity<Object> aff = grid0.affinity(DEFAULT_CACHE_NAME);
        assertTrue(aff.isPrimary(grid0.localNode(), key));
        assertTrue(aff.isBackup(grid1.localNode(), key));
        assertTrue(aff.isBackup(grid2.localNode(), key));

        final TestRecordingCommunicationSpi spi0 = (TestRecordingCommunicationSpi) grid0.configuration().getCommunicationSpi();

        spi0.blockMessages(GridDhtTxFinishRequest.class, grid1.name());
        spi0.blockMessages(GridDhtTxFinishRequest.class, grid2.name());

        IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
            @Override public void run() {
                try {
                    spi0.waitForBlocked();

                } catch (InterruptedException e) {
                    fail();
                }

                stopGrid(1);
                stopGrid(2);

                spi0.stopBlock(true);
            }
        }, 1, "stop-thread");

        IgniteCache cache = grid0.cache(DEFAULT_CACHE_NAME);

        int val0 = 1;

        cache.put(key, val0);

        fut.get();

        assertTrue("Expecting committed key", cache.containsKey(0));

        // Some actions can be done after successful commit at this point.

        // Now split-brain happens.
        // In real life grid0 and grid1 are separated and do not see each other.
        // Pretend second DC is active.
        stopGrid(0);

        grid1 = startGrid(1);
        grid1.active(true);

        grid2 = startGrid(2);

        // Test if previous commit is not lost.
        assertTrue("Expecting committed key", grid1.cache(DEFAULT_CACHE_NAME).containsKey(0));
        assertTrue("Expecting committed key", grid2.cache(DEFAULT_CACHE_NAME).containsKey(0));
    }
}
{noformat}

If routing will be switched to second data center, new transactions will no see commited state.



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)