You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2018/01/12 09:53:39 UTC

[09/15] ignite git commit: zk

http://git-wip-us.apache.org/repos/asf/ignite/blob/4805be06/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
deleted file mode 100644
index 20dba12..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ /dev/null
@@ -1,4483 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.zk.internal;
-
-import java.io.ByteArrayInputStream;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.zip.DataFormatException;
-import java.util.zip.Deflater;
-import java.util.zip.Inflater;
-import java.util.zip.InflaterInputStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteClientDisconnectedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteInterruptedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.CommunicationFailureResolver;
-import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.ClusterMetricsSnapshot;
-import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
-import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.IgnitionEx;
-import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
-import org.apache.ignite.internal.events.DiscoveryCustomEvent;
-import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.security.SecurityContext;
-import org.apache.ignite.internal.util.GridLongList;
-import org.apache.ignite.internal.util.GridSpinBusyLock;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.io.GridByteArrayOutputStream;
-import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.LT;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteInClosure;
-import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.lang.IgniteUuid;
-import org.apache.ignite.marshaller.MarshallerUtils;
-import org.apache.ignite.marshaller.jdk.JdkMarshaller;
-import org.apache.ignite.plugin.security.SecurityCredentials;
-import org.apache.ignite.spi.IgniteNodeValidationResult;
-import org.apache.ignite.spi.IgniteSpiException;
-import org.apache.ignite.spi.IgniteSpiTimeoutObject;
-import org.apache.ignite.spi.discovery.DiscoveryDataBag;
-import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
-import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
-import org.apache.ignite.spi.discovery.DiscoverySpiListener;
-import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
-import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
-import org.apache.ignite.thread.IgniteThreadPoolExecutor;
-import org.apache.zookeeper.AsyncCallback;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.data.Stat;
-import org.jboss.netty.util.internal.ConcurrentHashMap;
-import org.jetbrains.annotations.Nullable;
-
-import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
-import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
-import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
-import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
-import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2;
-import static org.apache.zookeeper.CreateMode.EPHEMERAL_SEQUENTIAL;
-import static org.apache.zookeeper.CreateMode.PERSISTENT;
-
-/**
- *
- */
-public class ZookeeperDiscoveryImpl {
-    /** */
-    static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD";
-
-    /** */
-    static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT";
-
-    /** */
-    static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS";
-
-    /** */
-    private static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_EVTS_THROTTLE = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_EVTS_THROTTLE";
-
-    /** */
-    final ZookeeperDiscoverySpi spi;
-
-    /** */
-    private final String igniteInstanceName;
-
-    /** */
-    private final String connectString;
-
-    /** */
-    private final int sesTimeout;
-
-    /** */
-    private final JdkMarshaller marsh = new JdkMarshaller();
-
-    /** */
-    private final ZkIgnitePaths zkPaths;
-
-    /** */
-    private final IgniteLogger log;
-
-    /** */
-    final GridSpinBusyLock busyLock = new GridSpinBusyLock();
-
-    /** */
-    private final ZookeeperClusterNode locNode;
-
-    /** */
-    private final DiscoverySpiListener lsnr;
-
-    /** */
-    private final DiscoverySpiDataExchange exchange;
-
-    /** */
-    private final boolean clientReconnectEnabled;
-
-    /** */
-    private final GridFutureAdapter<Void> joinFut = new GridFutureAdapter<>();
-
-    /** */
-    private final int evtsAckThreshold;
-
-    /** */
-    private IgniteThreadPoolExecutor utilityPool;
-
-    /** */
-    private ZkRuntimeState rtState;
-
-    /** */
-    private volatile ConnectionState connState = ConnectionState.STARTED;
-
-    /** */
-    private final AtomicBoolean stop = new AtomicBoolean();
-
-    /** */
-    private final Object stateMux = new Object();
-
-    /** */
-    public volatile IgniteDiscoverySpiInternalListener internalLsnr;
-
-    /** */
-    private final ConcurrentHashMap<Long, PingFuture> pingFuts = new ConcurrentHashMap<>();
-
-    /** */
-    private final AtomicReference<ZkCommunicationErrorProcessFuture> commErrProcFut = new AtomicReference<>();
-
-    /**
-     * @param spi Discovery SPI.
-     * @param igniteInstanceName Instance name.
-     * @param log Logger.
-     * @param zkRootPath Zookeeper base path node all nodes.
-     * @param locNode Local node instance.
-     * @param lsnr Discovery events listener.
-     * @param exchange Discovery data exchange.
-     * @param internalLsnr Internal listener (used for testing only).
-     */
-    public ZookeeperDiscoveryImpl(
-        ZookeeperDiscoverySpi spi,
-        String igniteInstanceName,
-        IgniteLogger log,
-        String zkRootPath,
-        ZookeeperClusterNode locNode,
-        DiscoverySpiListener lsnr,
-        DiscoverySpiDataExchange exchange,
-        IgniteDiscoverySpiInternalListener internalLsnr) {
-        assert locNode.id() != null && locNode.isLocal() : locNode;
-
-        zkRootPath = zkRootPath.trim();
-
-        if (zkRootPath.endsWith(ZkIgnitePaths.PATH_SEPARATOR))
-            zkRootPath = zkRootPath.substring(0, zkRootPath.length() - 1);
-
-        ZkIgnitePaths.validatePath(zkRootPath);
-
-        MarshallerUtils.setNodeName(marsh, igniteInstanceName);
-
-        zkPaths = new ZkIgnitePaths(zkRootPath);
-
-        this.spi = spi;
-        this.igniteInstanceName = igniteInstanceName;
-        this.connectString = spi.getZkConnectionString();
-        this.sesTimeout = (int)spi.getSessionTimeout();
-        this.log = log.getLogger(getClass());
-        this.locNode = locNode;
-        this.lsnr = lsnr;
-        this.exchange = exchange;
-        this.clientReconnectEnabled = locNode.isClient() && !spi.isClientReconnectDisabled();
-
-        int evtsAckThreshold = IgniteSystemProperties.getInteger(IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD, 5);
-
-        if (evtsAckThreshold <= 0)
-            evtsAckThreshold = 1;
-
-        this.evtsAckThreshold = evtsAckThreshold;
-
-        if (internalLsnr != null)
-            this.internalLsnr = internalLsnr;
-    }
-
-    /**
-     * @return Exception.
-     */
-    private static IgniteClientDisconnectedCheckedException disconnectError() {
-        return new IgniteClientDisconnectedCheckedException(null, "Client node disconnected.");
-    }
-
-    /**
-     * @return Logger.
-     */
-    IgniteLogger log() {
-        return log;
-    }
-
-    /**
-     * @return Local node instance.
-     */
-    public ClusterNode localNode() {
-        return locNode;
-    }
-
-    /**
-     * @param nodeId Node ID.
-     * @return Node instance.
-     */
-    @Nullable public ZookeeperClusterNode node(UUID nodeId) {
-        assert nodeId != null;
-
-        return rtState.top.nodesById.get(nodeId);
-    }
-
-    /**
-     * @param nodeOrder Node order.
-     * @return Node instance.
-     */
-    @Nullable public ZookeeperClusterNode node(long nodeOrder) {
-        assert nodeOrder > 0 : nodeOrder;
-
-        return rtState.top.nodesByOrder.get(nodeOrder);
-    }
-
-    /**
-     * @param fut Future to remove.
-     */
-    void clearCommunicationErrorProcessFuture(ZkCommunicationErrorProcessFuture fut) {
-        assert fut.isDone() : fut;
-
-        commErrProcFut.compareAndSet(fut, null);
-    }
-
-    /**
-     * @param node0 Problem node ID
-     * @param err Connect error.
-     */
-    public void resolveCommunicationError(ClusterNode node0, Exception err) {
-        ZookeeperClusterNode node = node(node0.id());
-
-        if (node == null)
-            throw new IgniteSpiException(new ClusterTopologyCheckedException("Node failed: " + node0.id()));
-
-        IgniteInternalFuture<Boolean> nodeStatusFut;
-
-        for (;;) {
-            checkState();
-
-            ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
-
-            if (fut == null || fut.isDone()) {
-                ZkCommunicationErrorProcessFuture newFut = ZkCommunicationErrorProcessFuture.createOnCommunicationError(
-                    this,
-                    node.sessionTimeout() + 1000);
-
-                if (commErrProcFut.compareAndSet(fut, newFut)) {
-                    fut = newFut;
-
-                    if (log.isInfoEnabled()) {
-                        log.info("Created new communication error process future [errNode=" + node0.id() +
-                            ", err=" + err + ']');
-                    }
-
-                    try {
-                        checkState();
-                    }
-                    catch (Exception e) {
-                        fut.onError(e);
-
-                        throw e;
-                    }
-
-                    fut.scheduleCheckOnTimeout();
-                }
-                else {
-                    fut = commErrProcFut.get();
-
-                    if (fut == null)
-                        continue;
-                }
-            }
-
-            nodeStatusFut = fut.nodeStatusFuture(node);
-
-            if (nodeStatusFut != null)
-                break;
-            else {
-                try {
-                    fut.get();
-                }
-                catch (IgniteCheckedException e) {
-                    U.warn(log, "Previous communication error process future failed: " + e);
-                }
-            }
-        }
-
-        try {
-            if (!nodeStatusFut.get())
-                throw new IgniteSpiException(new ClusterTopologyCheckedException("Node failed: " + node0.id()));
-        }
-        catch (IgniteCheckedException e) {
-            throw new IgniteSpiException(e);
-        }
-    }
-
-    /**
-     * @param nodeId Node ID.
-     * @return Ping result.
-     */
-    public boolean pingNode(UUID nodeId) {
-        checkState();
-
-        ZkRuntimeState rtState = this.rtState;
-
-        ZookeeperClusterNode node = rtState.top.nodesById.get(nodeId);
-
-        if (node == null)
-            return false;
-
-        if (node.isLocal())
-            return true;
-
-        PingFuture fut = pingFuts.get(node.order());
-
-        if (fut == null) {
-            fut = new PingFuture(rtState, node);
-
-            PingFuture old = pingFuts.putIfAbsent(node.order(), fut);
-
-            if (old == null) {
-                if (fut.checkNodeAndState())
-                    spi.getSpiContext().addTimeoutObject(fut);
-                else
-                    assert fut.isDone();
-            }
-            else
-                fut = old;
-        }
-
-        try {
-            return fut.get();
-        }
-        catch (IgniteCheckedException e) {
-            throw new IgniteSpiException(e);
-        }
-    }
-
-    /**
-     * @param nodeId Node ID.
-     * @param warning Warning.
-     */
-    public void failNode(UUID nodeId, @Nullable String warning) {
-        ZookeeperClusterNode node = rtState.top.nodesById.get(nodeId);
-
-        if (node == null) {
-            if (log.isDebugEnabled())
-                log.debug("Ignore forcible node fail request, node does not exist: " + nodeId);
-
-            return;
-        }
-
-        if (!node.isClient()) {
-            U.warn(log, "Ignore forcible node fail request for non-client node: " + node);
-
-            return;
-        }
-
-        sendCustomMessage(new ZkForceNodeFailMessage(node.internalId(), warning));
-    }
-
-    /**
-     *
-     */
-    public void reconnect() {
-        assert clientReconnectEnabled;
-
-        synchronized (stateMux) {
-            if (connState == ConnectionState.STARTED) {
-                connState = ConnectionState.DISCONNECTED;
-
-                rtState.onCloseStart(disconnectError());
-            }
-            else
-                return;
-        }
-
-        busyLock.block();
-
-        busyLock.unblock();
-
-        rtState.zkClient.close();
-
-        UUID newId = UUID.randomUUID();
-
-        U.quietAndWarn(log, "Local node will try to reconnect to cluster with new id due to network problems [" +
-            "newId=" + newId +
-            ", prevId=" + locNode.id() +
-            ", locNode=" + locNode + ']');
-
-        runInWorkerThread(new ReconnectClosure(newId));
-    }
-
-    /**
-     * @param newId New ID.
-     */
-    private void doReconnect(UUID newId) {
-        if (rtState.joined) {
-            assert rtState.evtsData != null;
-
-            lsnr.onDiscovery(EVT_CLIENT_NODE_DISCONNECTED,
-                rtState.evtsData.topVer,
-                locNode,
-                rtState.top.topologySnapshot(),
-                Collections.<Long, Collection<ClusterNode>>emptyMap(),
-                null);
-        }
-
-        try {
-            locNode.onClientDisconnected(newId);
-
-            joinTopology(rtState);
-        }
-        catch (Exception e) {
-            if (stopping()) {
-                if (log.isDebugEnabled())
-                    log.debug("Reconnect failed, node is stopping [err=" + e + ']');
-
-                return;
-            }
-
-            U.error(log, "Failed to reconnect: " + e, e);
-
-            onSegmented(e);
-        }
-    }
-
-    /**
-     * @return {@code True} if started to stop.
-     */
-    private boolean stopping() {
-        if (stop.get())
-            return true;
-
-        synchronized (stateMux) {
-            if (connState == ConnectionState.STOPPED)
-                return true;
-        }
-
-        return false;
-    }
-
-    /**
-     * @param e Error.
-     */
-    private void onSegmented(Exception e) {
-        rtState.errForClose = e;
-
-        if (rtState.joined || joinFut.isDone()) {
-            synchronized (stateMux) {
-                connState = ConnectionState.STOPPED;
-            }
-
-            notifySegmented();
-        }
-        else
-            joinFut.onDone(e);
-    }
-
-    /**
-     *
-     */
-    private void notifySegmented() {
-        List<ClusterNode> nodes = rtState.top.topologySnapshot();
-
-        if (nodes.isEmpty())
-            nodes = Collections.singletonList((ClusterNode)locNode);
-
-        lsnr.onDiscovery(EVT_NODE_SEGMENTED,
-            rtState.evtsData != null ? rtState.evtsData.topVer : 1L,
-            locNode,
-            nodes,
-            Collections.<Long, Collection<ClusterNode>>emptyMap(),
-            null);
-    }
-
-    /**
-     * @return Remote nodes.
-     */
-    public Collection<ClusterNode> remoteNodes() {
-        checkState();
-
-        return rtState.top.remoteNodes();
-    }
-
-    /**
-     *
-     */
-    private void checkState() {
-        switch (connState) {
-            case STARTED:
-                break;
-
-            case STOPPED:
-                throw new IgniteSpiException("Node stopped.");
-
-            case DISCONNECTED:
-                throw new IgniteClientDisconnectedException(null, "Client is disconnected.");
-        }
-    }
-
-    /**
-     * @param nodeId Node ID.
-     * @return {@code True} if node joined or joining topology.
-     */
-    public boolean knownNode(UUID nodeId) {
-        while (!busyLock.enterBusy())
-            checkState();
-
-        try {
-            List<String> children = rtState.zkClient.getChildren(zkPaths.aliveNodesDir);
-
-            for (int i = 0; i < children.size(); i++) {
-                UUID id = ZkIgnitePaths.aliveNodeId(children.get(i));
-
-                if (nodeId.equals(id))
-                    return true;
-            }
-
-            return false;
-        }
-        catch (ZookeeperClientFailedException e) {
-            if (clientReconnectEnabled)
-                throw new IgniteClientDisconnectedException(null, "Client is disconnected.");
-
-            throw new IgniteException(e);
-        }
-        catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-
-            throw new IgniteInterruptedException(e);
-        }
-        finally {
-            busyLock.leaveBusy();
-        }
-    }
-
-    /**
-     * @param msg Message.
-     */
-    public void sendCustomMessage(DiscoverySpiCustomMessage msg) {
-        assert msg != null;
-
-        byte[] msgBytes;
-
-        try {
-            msgBytes = marshalZip(msg);
-        }
-        catch (IgniteCheckedException e) {
-            throw new IgniteSpiException("Failed to marshal custom message: " + msg, e);
-        }
-
-        while (!busyLock.enterBusy())
-            checkState();
-
-        try {
-            ZookeeperClient zkClient = rtState.zkClient;
-
-            saveCustomMessage(zkClient, msgBytes);
-        }
-        catch (ZookeeperClientFailedException e) {
-            if (clientReconnectEnabled)
-                throw new IgniteClientDisconnectedException(null, "Client is disconnected.");
-
-            throw new IgniteException(e);
-        }
-        catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-
-            throw new IgniteInterruptedException(e);
-        }
-        finally {
-            busyLock.leaveBusy();
-        }
-    }
-
-    /**
-     * @param zkClient Client.
-     * @param msgBytes Marshalled message.
-     * @throws ZookeeperClientFailedException If connection to zk was lost.
-     * @throws InterruptedException If interrupted.
-     */
-    private void saveCustomMessage(ZookeeperClient zkClient, byte[] msgBytes)
-        throws ZookeeperClientFailedException, InterruptedException
-    {
-        String prefix = UUID.randomUUID().toString();
-
-        int partCnt = 1;
-
-        int overhead = 10;
-
-        UUID locId = locNode.id();
-
-        String path = zkPaths.createCustomEventPath(prefix, locId, partCnt);
-
-        if (zkClient.needSplitNodeData(path, msgBytes, overhead)) {
-            List<byte[]> parts = zkClient.splitNodeData(path, msgBytes, overhead);
-
-            String partsBasePath = zkPaths.customEventPartsBasePath(prefix, locId);
-
-            saveMultipleParts(zkClient, partsBasePath, parts);
-
-            msgBytes = null;
-
-            partCnt = parts.size();
-        }
-
-        zkClient.createSequential(prefix,
-            zkPaths.customEvtsDir,
-            zkPaths.createCustomEventPath(prefix, locId, partCnt),
-            msgBytes,
-            CreateMode.PERSISTENT_SEQUENTIAL);
-    }
-
-    /**
-     * @return Cluster start time.
-     */
-    public long gridStartTime() {
-        return rtState.gridStartTime;
-    }
-
-    /**
-     * Starts join procedure and waits for {@link EventType#EVT_NODE_JOINED} event for local node.
-     *
-     * @throws InterruptedException If interrupted.
-     */
-    public void startJoinAndWait() throws InterruptedException {
-        joinTopology(null);
-
-        for (;;) {
-            try {
-                joinFut.get(10_000);
-
-                break;
-            }
-            catch (IgniteFutureTimeoutCheckedException e) {
-                U.warn(log, "Waiting for local join event [nodeId=" + locNode.id() + ", name=" + igniteInstanceName + ']');
-            }
-            catch (Exception e) {
-                IgniteSpiException spiErr = X.cause(e, IgniteSpiException.class);
-
-                if (spiErr != null)
-                    throw spiErr;
-
-                throw new IgniteSpiException("Failed to join cluster", e);
-            }
-        }
-    }
-
-    /**
-     * @param prevState Previous state in case of connect retry.
-     * @throws InterruptedException If interrupted.
-     */
-    private void joinTopology(@Nullable ZkRuntimeState prevState) throws InterruptedException {
-        if (!busyLock.enterBusy())
-            return;
-
-        try {
-            boolean reconnect = prevState != null;
-
-            // Need fire EVT_CLIENT_NODE_RECONNECTED event if reconnect after already joined.
-            boolean prevJoined = prevState != null && prevState.joined;
-
-            IgniteDiscoverySpiInternalListener internalLsnr = this.internalLsnr;
-
-            if (internalLsnr != null)
-                internalLsnr.beforeJoin(locNode, log);
-
-            if (locNode.isClient() && reconnect)
-                locNode.setAttributes(spi.getSpiContext().nodeAttributes());
-
-            marshalCredentialsOnJoin(locNode);
-
-            synchronized (stateMux) {
-                if (connState == ConnectionState.STOPPED)
-                    return;
-
-                connState = ConnectionState.STARTED;
-            }
-
-            ZkRuntimeState rtState = this.rtState = new ZkRuntimeState(prevJoined);
-
-            DiscoveryDataBag discoDataBag = new DiscoveryDataBag(locNode.id());
-
-            exchange.collect(discoDataBag);
-
-            ZkJoiningNodeData joinData = new ZkJoiningNodeData(locNode, discoDataBag.joiningNodeData());
-
-            byte[] joinDataBytes;
-
-            try {
-                joinDataBytes = marshalZip(joinData);
-            }
-            catch (Exception e) {
-                throw new IgniteSpiException("Failed to marshal joining node data", e);
-            }
-
-            try {
-                rtState.zkClient = new ZookeeperClient(
-                    igniteInstanceName,
-                    log,
-                    connectString,
-                    sesTimeout,
-                    new ConnectionLossListener());
-            }
-            catch (Exception e) {
-                throw new IgniteSpiException("Failed to create Zookeeper client", e);
-            }
-
-            startJoin(rtState, prevState, joinDataBytes);
-        }
-        finally {
-            busyLock.leaveBusy();
-        }
-    }
-
-    /**
-     * @throws InterruptedException If interrupted.
-     */
-    private void initZkNodes() throws InterruptedException {
-        try {
-            ZookeeperClient client = rtState.zkClient;
-
-            if (client.exists(zkPaths.aliveNodesDir))
-                return; // This path is created last, assume all others dirs are created.
-
-            if (!client.exists(zkPaths.aliveNodesDir))
-                createRootPathParents(zkPaths.clusterDir, client);
-
-            List<String> dirs = new ArrayList<>();
-
-            dirs.add(zkPaths.clusterDir);
-            dirs.add(zkPaths.evtsPath);
-            dirs.add(zkPaths.joinDataDir);
-            dirs.add(zkPaths.customEvtsDir);
-            dirs.add(zkPaths.customEvtsPartsDir);
-            dirs.add(zkPaths.customEvtsAcksDir);
-            dirs.add(zkPaths.aliveNodesDir);
-
-            try {
-                client.createAll(dirs, PERSISTENT);
-            }
-            catch (KeeperException.NodeExistsException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Failed to create nodes using bulk operation: " + e);
-
-                for (String dir : dirs)
-                    client.createIfNeeded(dir, null, PERSISTENT);
-            }
-        }
-        catch (ZookeeperClientFailedException e) {
-            throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e);
-        }
-    }
-
-    /**
-     * @param rootDir Root directory.
-     * @param client Client.
-     * @throws ZookeeperClientFailedException If connection to zk was lost.
-     * @throws InterruptedException If interrupted.
-     */
-    private void createRootPathParents(String rootDir, ZookeeperClient client)
-        throws ZookeeperClientFailedException, InterruptedException {
-        int startIdx = 0;
-
-        for (;;) {
-            int separatorIdx = rootDir.indexOf(ZkIgnitePaths.PATH_SEPARATOR, startIdx);
-
-            if (separatorIdx == -1)
-                break;
-
-            if (separatorIdx > 0) {
-                String path = rootDir.substring(0, separatorIdx);
-
-                client.createIfNeeded(path, null, CreateMode.PERSISTENT);
-            }
-
-            startIdx = separatorIdx + 1;
-        }
-    }
-
-    /**
-     * @param zkClient Client.
-     * @param basePath Base path.
-     * @param partCnt Parts count.
-     */
-    private void deleteMultiplePartsAsync(ZookeeperClient zkClient, String basePath, int partCnt) {
-        for (int i = 0; i < partCnt; i++) {
-            String path = multipartPathName(basePath, i);
-
-            zkClient.deleteIfExistsAsync(path);
-        }
-    }
-
-    /**
-     * @param zkClient Client.
-     * @param basePath Base path.
-     * @param partCnt Parts count.
-     * @return Read parts.
-     * @throws Exception If failed.
-     */
-    private byte[] readMultipleParts(ZookeeperClient zkClient, String basePath, int partCnt)
-        throws Exception {
-        assert partCnt >= 1;
-
-        if (partCnt > 1) {
-            List<byte[]> parts = new ArrayList<>(partCnt);
-
-            int totSize = 0;
-
-            for (int i = 0; i < partCnt; i++) {
-                byte[] part = zkClient.getData(multipartPathName(basePath, i));
-
-                parts.add(part);
-
-                totSize += part.length;
-            }
-
-            byte[] res = new byte[totSize];
-
-            int pos = 0;
-
-            for (int i = 0; i < partCnt; i++) {
-                byte[] part = parts.get(i);
-
-                System.arraycopy(part, 0, res, pos, part.length);
-
-                pos += part.length;
-            }
-
-            return res;
-        }
-        else
-            return zkClient.getData(multipartPathName(basePath, 0));
-    }
-
-    /**
-     * @param zkClient Client.
-     * @param basePath Base path.
-     * @param parts Data parts.
-     * @return Number of parts.
-     * @throws ZookeeperClientFailedException If client failed.
-     * @throws InterruptedException If interrupted.
-     */
-    private int saveMultipleParts(ZookeeperClient zkClient, String basePath, List<byte[]> parts)
-        throws ZookeeperClientFailedException, InterruptedException
-    {
-        assert parts.size() > 1;
-
-        for (int i = 0; i < parts.size(); i++) {
-            byte[] part = parts.get(i);
-
-            String path = multipartPathName(basePath, i);
-
-            zkClient.createIfNeeded(path, part, PERSISTENT);
-        }
-
-        return parts.size();
-    }
-
-    /**
-     * @param basePath Base path.
-     * @param part Part number.
-     * @return Path.
-     */
-    private static String multipartPathName(String basePath, int part) {
-        return basePath + String.format("%04d", part);
-    }
-
-    /**
-     * @param rtState Runtime state.
-     * @param joinDataBytes Joining node data.
-     * @param prevState Previous state in case of connect retry.
-     * @throws InterruptedException If interrupted.
-     */
-    private void startJoin(ZkRuntimeState rtState, @Nullable ZkRuntimeState prevState, final byte[] joinDataBytes)
-        throws InterruptedException
-    {
-        try {
-            long startTime = System.currentTimeMillis();
-
-            initZkNodes();
-
-            String prefix = UUID.randomUUID().toString();
-
-            rtState.init(new ZkWatcher(rtState), new AliveNodeDataWatcher(rtState));
-
-            ZookeeperClient zkClient = rtState.zkClient;
-
-            final int OVERHEAD = 5;
-
-            // TODO ZK: need clean up join data if failed before was able to create alive node.
-            String joinDataPath = zkPaths.joinDataDir + "/" + prefix + ":" + locNode.id();
-
-            if (zkClient.needSplitNodeData(joinDataPath, joinDataBytes, OVERHEAD)) {
-                List<byte[]> parts = zkClient.splitNodeData(joinDataPath, joinDataBytes, OVERHEAD);
-
-                rtState.joinDataPartCnt = parts.size();
-
-                saveMultipleParts(zkClient, joinDataPath + ":", parts);
-
-                joinDataPath = zkClient.createIfNeeded(
-                    joinDataPath,
-                    marshalZip(new ZkJoiningNodeData(parts.size())),
-                    PERSISTENT);
-            }
-            else {
-                joinDataPath = zkClient.createIfNeeded(
-                    joinDataPath,
-                    joinDataBytes,
-                    PERSISTENT);
-            }
-
-            rtState.locNodeZkPath = zkClient.createSequential(
-                prefix,
-                zkPaths.aliveNodesDir,
-                zkPaths.aliveNodePathForCreate(prefix, locNode),
-                null,
-                EPHEMERAL_SEQUENTIAL);
-
-            rtState.internalOrder = ZkIgnitePaths.aliveInternalId(rtState.locNodeZkPath);
-
-            if (log.isInfoEnabled()) {
-                log.info("Node started join [nodeId=" + locNode.id() +
-                    ", instanceName=" + locNode.attribute(ATTR_IGNITE_INSTANCE_NAME) +
-                    ", joinDataSize=" + joinDataBytes.length +
-                    (rtState.joinDataPartCnt > 1 ? (", joinDataPartCnt=" + rtState.joinDataPartCnt) : "") +
-                    ", consistentId=" + locNode.consistentId() +
-                    ", initTime=" + (System.currentTimeMillis() - startTime) +
-                    ", nodePath=" + rtState.locNodeZkPath + ']');
-            }
-
-            /*
-            If node can not join due to validation error this error is reported in join data,
-            As a minor optimization do not start watch join data immediately, but only if do not receive
-            join event after some timeout.
-             */
-            CheckJoinErrorWatcher joinErrorWatcher = new CheckJoinErrorWatcher(5000, joinDataPath, rtState);
-
-            rtState.joinErrTo = joinErrorWatcher.timeoutObj;
-
-            if (locNode.isClient() && spi.getJoinTimeout() > 0) {
-                ZkTimeoutObject joinTimeoutObj = prevState != null ? prevState.joinTo : null;
-
-                if (joinTimeoutObj == null) {
-                    joinTimeoutObj = new JoinTimeoutObject(spi.getJoinTimeout());
-
-                    spi.getSpiContext().addTimeoutObject(joinTimeoutObj);
-                }
-
-                rtState.joinTo = joinTimeoutObj;
-            }
-
-            if (!locNode.isClient())
-                zkClient.getChildrenAsync(zkPaths.aliveNodesDir, null, new CheckCoordinatorCallback(rtState));
-
-            zkClient.getDataAsync(zkPaths.evtsPath, rtState.watcher, rtState.watcher);
-
-            spi.getSpiContext().addTimeoutObject(rtState.joinErrTo);
-        }
-        catch (IgniteCheckedException | ZookeeperClientFailedException e) {
-            throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e);
-        }
-    }
-
-    /**
-     * Authenticate local node.
-     *
-     * @param nodeAuth Authenticator.
-     * @param locCred Local security credentials for authentication.
-     * @throws IgniteSpiException If any error occurs.
-     */
-    private void localAuthentication(DiscoverySpiNodeAuthenticator nodeAuth, SecurityCredentials locCred){
-        assert nodeAuth != null;
-        assert locCred != null;
-
-        try {
-            SecurityContext subj = nodeAuth.authenticateNode(locNode, locCred);
-
-            // Note: exception message is checked in tests.
-            if (subj == null)
-                throw new IgniteSpiException("Authentication failed for local node.");
-
-            if (!(subj instanceof Serializable))
-                throw new IgniteSpiException("Authentication subject is not Serializable.");
-
-            Map<String, Object> attrs = new HashMap<>(locNode.attributes());
-
-            attrs.put(ATTR_SECURITY_SUBJECT_V2, U.marshal(marsh, subj));
-
-            locNode.setAttributes(attrs);
-        }
-        catch (Exception e) {
-            throw new IgniteSpiException("Failed to authenticate local node (will shutdown local node).", e);
-        }
-    }
-
-    /**
-     * @param node Node.
-     * @param zipBytes Zip-compressed marshalled security subject.
-     * @throws Exception If failed.
-     */
-    private void setNodeSecuritySubject(ZookeeperClusterNode node, byte[] zipBytes) throws Exception {
-        assert zipBytes != null;
-
-        Map<String, Object> attrs = new HashMap<>(node.getAttributes());
-
-        attrs.put(ATTR_SECURITY_SUBJECT_V2, unzip(zipBytes));
-
-        node.setAttributes(attrs);
-    }
-
-    /**
-     * @param node Node.
-     * @return Credentials.
-     * @throws IgniteCheckedException If failed to unmarshal.
-     */
-    private SecurityCredentials unmarshalCredentials(ZookeeperClusterNode node) throws Exception {
-        byte[] credBytes = (byte[])node.getAttributes().get(ATTR_SECURITY_CREDENTIALS);
-
-        if (credBytes == null)
-            return null;
-
-        return unmarshalZip(credBytes);
-    }
-
-    /**
-     * Marshalls credentials with discovery SPI marshaller (will replace attribute value).
-     *
-     * @param node Node to marshall credentials for.
-     * @throws IgniteSpiException If marshalling failed.
-     */
-    private void marshalCredentialsOnJoin(ZookeeperClusterNode node) throws IgniteSpiException {
-        try {
-            // Use security-unsafe getter.
-            Map<String, Object> attrs0 = node.getAttributes();
-
-            Object creds = attrs0.get(ATTR_SECURITY_CREDENTIALS);
-
-            if (creds != null) {
-                Map<String, Object> attrs = new HashMap<>(attrs0);
-
-                assert !(creds instanceof byte[]);
-
-                attrs.put(ATTR_SECURITY_CREDENTIALS, marshalZip(creds));
-
-                node.setAttributes(attrs);
-            }
-        }
-        catch (IgniteCheckedException e) {
-            throw new IgniteSpiException("Failed to marshal node security credentials: " + node.id(), e);
-        }
-    }
-
-    /**
-     *
-     */
-    private class UpdateProcessedEventsTimeoutObject extends ZkTimeoutObject {
-        /** */
-        private final ZkRuntimeState rtState;
-
-        /**
-         * @param rtState Runtime state.
-         * @param timeout Timeout.
-         */
-        UpdateProcessedEventsTimeoutObject(ZkRuntimeState rtState, long timeout) {
-            super(timeout);
-
-            this.rtState = rtState;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void onTimeout() {
-            runInWorkerThread(new ZkRunnable(rtState, ZookeeperDiscoveryImpl.this) {
-                @Override protected void run0() throws Exception {
-                    updateProcessedEventsOnTimeout(rtState, UpdateProcessedEventsTimeoutObject.this);
-                }
-            });
-        }
-    }
-
-    /**
-     *
-     */
-    private class JoinTimeoutObject extends ZkTimeoutObject {
-        /**
-         * @param timeout Timeout.
-         */
-        JoinTimeoutObject(long timeout) {
-            super(timeout);
-        }
-
-        /** {@inheritDoc} */
-        @Override public void onTimeout() {
-            if (cancelled || rtState.joined)
-                return;
-
-            runInWorkerThread(new Runnable() {
-                @Override public void run() {
-                    synchronized (stateMux) {
-                        if (cancelled || rtState.joined)
-                            return;
-
-                        if (connState == ConnectionState.STOPPED)
-                            return;
-
-                        connState = ConnectionState.STOPPED;
-                    }
-
-                    U.warn(log, "Failed to connect to cluster, either connection to ZooKeeper can not be established or there " +
-                        "are no alive server nodes (consider increasing 'joinTimeout' configuration  property) [" +
-                        "joinTimeout=" + spi.getJoinTimeout() + ']');
-
-                    // Note: exception message is checked in tests.
-                    onSegmented(new IgniteSpiException("Failed to connect to cluster within configured timeout"));
-                }
-            });
-        }
-    }
-
-    /**
-     *
-     */
-    private class CheckJoinErrorWatcher extends ZkAbstractWatcher implements AsyncCallback.DataCallback {
-        /** */
-        private final String joinDataPath;
-
-        /** */
-        private ZkTimeoutObject timeoutObj;
-
-        /**
-         * @param timeout Timeout.
-         * @param joinDataPath0 Node joined data path.
-         * @param rtState0 State.
-         */
-        CheckJoinErrorWatcher(long timeout, String joinDataPath0, ZkRuntimeState rtState0) {
-            super(rtState0, ZookeeperDiscoveryImpl.this);
-
-            this.joinDataPath = joinDataPath0;
-
-            timeoutObj = new ZkTimeoutObject(timeout) {
-                @Override public void onTimeout() {
-                    if (rtState.errForClose != null || rtState.joined)
-                        return;
-
-                    synchronized (stateMux) {
-                        if (connState != ConnectionState.STARTED)
-                            return;
-                    }
-
-                    rtState.zkClient.getDataAsync(joinDataPath,
-                        CheckJoinErrorWatcher.this,
-                        CheckJoinErrorWatcher.this);
-                }
-            };
-        }
-
-        /** {@inheritDoc} */
-        @Override public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) {
-            if (rc != 0)
-                return;
-
-            if (!onProcessStart())
-                return;
-
-            try {
-                Object obj = unmarshalZip(data);
-
-                if (obj instanceof ZkInternalJoinErrorMessage) {
-                    ZkInternalJoinErrorMessage joinErr = (ZkInternalJoinErrorMessage)obj;
-
-                    onSegmented(new IgniteSpiException(joinErr.err));
-                }
-
-                onProcessEnd();
-            }
-            catch (Throwable e) {
-                onProcessError(e);
-            }
-        }
-
-        /** {@inheritDoc} */
-        @Override public void process0(WatchedEvent evt) {
-            if (rtState.errForClose != null || rtState.joined)
-                return;
-
-            if (evt.getType() == Event.EventType.NodeDataChanged)
-                rtState.zkClient.getDataAsync(evt.getPath(), this, this);
-        }
-    }
-
-    /**
-     * @param aliveNodes Alive nodes.
-     * @throws Exception If failed.
-     */
-    private void checkIsCoordinator(final List<String> aliveNodes) throws Exception {
-        assert !locNode.isClient();
-
-        TreeMap<Long, String> aliveSrvs = new TreeMap<>();
-
-        long locInternalOrder = rtState.internalOrder;
-
-        for (String aliveNodePath : aliveNodes) {
-            if (ZkIgnitePaths.aliveNodeClientFlag(aliveNodePath))
-                continue;
-
-            Long internalId = ZkIgnitePaths.aliveInternalId(aliveNodePath);
-
-            aliveSrvs.put(internalId, aliveNodePath);
-        }
-
-        assert !aliveSrvs.isEmpty();
-
-        Map.Entry<Long, String> crdE = aliveSrvs.firstEntry();
-
-        if (locInternalOrder == crdE.getKey())
-            onBecomeCoordinator(aliveNodes);
-        else {
-            assert aliveSrvs.size() > 1 : aliveSrvs;
-
-            Map.Entry<Long, String> prevE = aliveSrvs.floorEntry(locInternalOrder - 1);
-
-            assert prevE != null;
-
-            if (log.isInfoEnabled()) {
-                log.info("Discovery coordinator already exists, watch for previous server node [" +
-                    "locId=" + locNode.id() +
-                    ", watchPath=" + prevE.getValue() + ']');
-             }
-
-            PreviousNodeWatcher watcher = new ServerPreviousNodeWatcher(rtState);
-
-            rtState.zkClient.existsAsync(zkPaths.aliveNodesDir + "/" + prevE.getValue(), watcher, watcher);
-        }
-    }
-
-    /**
-     * @param aliveNodes Alive nodes.
-     * @throws Exception If failed.
-     */
-    private void checkClientsStatus(final List<String> aliveNodes) throws Exception {
-        assert locNode.isClient() : locNode;
-        assert rtState.joined;
-        assert rtState.evtsData != null;
-
-        TreeMap<Long, String> aliveClients = new TreeMap<>();
-
-        String srvPath = null;
-        Long srvInternalOrder = null;
-
-        long locInternalOrder = rtState.internalOrder;
-
-        for (String aliveNodePath : aliveNodes) {
-            Long internalId = ZkIgnitePaths.aliveInternalId(aliveNodePath);
-
-            if (ZkIgnitePaths.aliveNodeClientFlag(aliveNodePath))
-                aliveClients.put(internalId, aliveNodePath);
-            else {
-                if (srvInternalOrder == null || internalId < srvInternalOrder) {
-                    srvPath = aliveNodePath;
-                    srvInternalOrder = internalId;
-                }
-            }
-        }
-
-        assert !aliveClients.isEmpty();
-
-        Map.Entry<Long, String> oldest = aliveClients.firstEntry();
-
-        boolean oldestClient = locInternalOrder == oldest.getKey();
-
-        if (srvPath == null) {
-            if (oldestClient) {
-                Stat stat = new Stat();
-
-                ZkDiscoveryEventsData prevEvts = rtState.evtsData;
-
-                byte[] evtsBytes = rtState.zkClient.getData(zkPaths.evtsPath, stat);
-
-                assert evtsBytes.length > 0;
-
-                ZkDiscoveryEventsData newEvts = unmarshalZip(evtsBytes);
-
-                if (prevEvts.clusterId.equals(newEvts.clusterId)) {
-                    U.warn(log, "All server nodes failed, notify all clients [locId=" + locNode.id() + ']');
-
-                    generateNoServersEvent(newEvts, stat);
-                }
-                else
-                    U.warn(log, "All server nodes failed (received events from new cluster).");
-            }
-        }
-        else {
-            String watchPath;
-
-            if (oldestClient) {
-                watchPath = srvPath;
-
-                if (log.isInfoEnabled()) {
-                    log.info("Servers exists, watch for server node [locId=" + locNode.id() +
-                        ", watchPath=" + watchPath + ']');
-                }
-            }
-            else {
-                assert aliveClients.size() > 1 : aliveClients;
-
-                Map.Entry<Long, String> prevE = aliveClients.floorEntry(locInternalOrder - 1);
-
-                assert prevE != null;
-
-                watchPath = prevE.getValue();
-
-                if (log.isInfoEnabled()) {
-                    log.info("Servers exists, watch for previous node [locId=" + locNode.id() +
-                        ", watchPath=" + watchPath + ']');
-                }
-            }
-
-            PreviousNodeWatcher watcher = new ClientPreviousNodeWatcher(rtState);
-
-            rtState.zkClient.existsAsync(zkPaths.aliveNodesDir + "/" + watchPath, watcher, watcher);
-        }
-    }
-
-    /**
-     * @param evtsData Events data.
-     * @param evtsStat Events zookeeper state.
-     * @throws Exception If failed.
-     */
-    private void generateNoServersEvent(ZkDiscoveryEventsData evtsData, Stat evtsStat) throws Exception {
-        evtsData.evtIdGen++;
-
-        ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
-            evtsData.evtIdGen,
-            0L,
-            evtsData.topVer,
-            locNode.id(),
-            new ZkNoServersMessage(),
-            null);
-
-        Collection<ZookeeperClusterNode> nodesToAck = Collections.emptyList();
-
-        evtsData.addEvent(nodesToAck, evtData);
-
-        byte[] newEvtsBytes = marshalZip(evtsData);
-
-        try {
-            rtState.zkClient.setData(zkPaths.evtsPath, newEvtsBytes, evtsStat.getVersion());
-        }
-        catch (KeeperException.BadVersionException e) {
-            // Version can change if new cluster started and saved new events.
-            if (log.isDebugEnabled())
-                log.debug("Failed to save no servers message");
-        }
-    }
-
-    /**
-     * @param lastEvts Last events from previous coordinator.
-     * @throws Exception If failed.
-     */
-    private void previousCoordinatorCleanup(ZkDiscoveryEventsData lastEvts) throws Exception {
-        for (ZkDiscoveryEventData evtData : lastEvts.evts.values()) {
-            if (evtData instanceof ZkDiscoveryCustomEventData) {
-                ZkDiscoveryCustomEventData evtData0 = (ZkDiscoveryCustomEventData)evtData;
-
-                // It is possible previous coordinator failed before finished cleanup.
-                if (evtData0.msg instanceof ZkCommunicationErrorResolveFinishMessage) {
-                    try {
-                        ZkCommunicationErrorResolveFinishMessage msg =
-                            (ZkCommunicationErrorResolveFinishMessage)evtData0.msg;
-
-                        ZkCommunicationErrorResolveResult res = unmarshalZip(
-                            ZkDistributedCollectDataFuture.readResult(rtState.zkClient, zkPaths, msg.futId));
-
-                        deleteAliveNodes(res.killedNodes);
-                    }
-                    catch (KeeperException.NoNodeException ignore) {
-                        // No-op.
-                    }
-                }
-                else if (evtData0.resolvedMsg instanceof ZkForceNodeFailMessage)
-                    deleteAliveNode(((ZkForceNodeFailMessage)evtData0.resolvedMsg).nodeInternalId);
-            }
-        }
-    }
-
-    /**
-     * @param aliveNodes Alive nodes paths.
-     * @throws Exception If failed.
-     */
-    private void onBecomeCoordinator(List<String> aliveNodes) throws Exception {
-        ZkDiscoveryEventsData prevEvts = processNewEvents(rtState.zkClient.getData(zkPaths.evtsPath));
-
-        rtState.crd = true;
-
-        if (rtState.joined) {
-            if (log.isInfoEnabled())
-                log.info("Node is new discovery coordinator [locId=" + locNode.id() + ']');
-
-            assert locNode.order() > 0 : locNode;
-            assert rtState.evtsData != null;
-
-            previousCoordinatorCleanup(rtState.evtsData);
-
-            UUID futId = rtState.evtsData.communicationErrorResolveFutureId();
-
-            if (futId != null) {
-                if (log.isInfoEnabled()) {
-                    log.info("New discovery coordinator will handle already started cluster-wide communication " +
-                        "error resolve [reqId=" + futId + ']');
-                }
-
-                ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
-
-                ZkDistributedCollectDataFuture collectResFut = collectCommunicationStatusFuture(futId);
-
-                if (fut != null)
-                    fut.nodeResultCollectFuture(collectResFut);
-            }
-
-            for (ZkDiscoveryEventData evtData : rtState.evtsData.evts.values())
-                evtData.initRemainingAcks(rtState.top.nodesByOrder.values());
-
-            handleProcessedEvents("crd");
-        }
-        else {
-            String locAlivePath = rtState.locNodeZkPath.substring(rtState.locNodeZkPath.lastIndexOf('/') + 1);
-
-            deleteJoiningNodeData(locNode.id(),
-                ZkIgnitePaths.aliveNodePrefixId(locAlivePath),
-                rtState.joinDataPartCnt);
-
-            DiscoverySpiNodeAuthenticator nodeAuth = spi.getAuthenticator();
-
-            if (nodeAuth != null) {
-                try {
-                    if (log.isInfoEnabled()) {
-                        log.info("Node is first server node in cluster, try authenticate local node " +
-                            "[locId=" + locNode.id() + ']');
-                    }
-
-                    localAuthentication(nodeAuth, unmarshalCredentials(locNode));
-                }
-                catch (Exception e) {
-                    U.warn(log, "Local node authentication failed: " + e, e);
-
-                    onSegmented(e);
-
-                    // Stop any further processing.
-                    throw new ZookeeperClientFailedException("Local node authentication failed: " + e);
-                }
-            }
-
-            newClusterStarted(prevEvts);
-        }
-
-        rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
-        rtState.zkClient.getChildrenAsync(zkPaths.customEvtsDir, rtState.watcher, rtState.watcher);
-
-        for (String alivePath : aliveNodes)
-            watchAliveNodeData(alivePath);
-    }
-
-    /**
-     * @param alivePath Node path.
-     */
-    private void watchAliveNodeData(String alivePath) {
-        assert rtState.locNodeZkPath != null;
-
-        String path = zkPaths.aliveNodesDir + "/" + alivePath;
-
-        if (!path.equals(rtState.locNodeZkPath))
-            rtState.zkClient.getDataAsync(path, rtState.aliveNodeDataWatcher, rtState.aliveNodeDataWatcher);
-    }
-
-    /**
-     * @param aliveNodes ZK nodes representing alive cluster nodes.
-     * @throws Exception If failed.
-     */
-    private void generateTopologyEvents(List<String> aliveNodes) throws Exception {
-        assert rtState.crd;
-
-        if (log.isInfoEnabled())
-            log.info("Process alive nodes change [alives=" + aliveNodes.size() + "]");
-
-        TreeMap<Long, String> alives = new TreeMap<>();
-
-        for (String child : aliveNodes) {
-            Long internalId = ZkIgnitePaths.aliveInternalId(child);
-
-            Object old = alives.put(internalId, child);
-
-            assert old == null;
-        }
-
-        TreeMap<Long, ZookeeperClusterNode> curTop = new TreeMap<>(rtState.top.nodesByOrder);
-
-        int newEvts = 0;
-
-        final int MAX_NEW_EVTS = IgniteSystemProperties.getInteger(IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS, 100);
-
-        List<ZookeeperClusterNode> failedNodes = null;
-
-        for (Map.Entry<Long, ZookeeperClusterNode> e : rtState.top.nodesByInternalId.entrySet()) {
-            if (!alives.containsKey(e.getKey())) {
-                ZookeeperClusterNode failedNode = e.getValue();
-
-                if (failedNodes == null)
-                    failedNodes = new ArrayList<>();
-
-                failedNodes.add(failedNode);
-
-                generateNodeFail(curTop, failedNode);
-
-                newEvts++;
-
-                if (newEvts == MAX_NEW_EVTS) {
-                    saveAndProcessNewEvents();
-
-                    if (log.isInfoEnabled()) {
-                        log.info("Delay alive nodes change process, max event threshold reached [newEvts=" + newEvts +
-                            ", totalEvts=" + rtState.evtsData.evts.size() + ']');
-                    }
-
-                    handleProcessedEventsOnNodesFail(failedNodes);
-
-                    throttleNewEventsGeneration();
-
-                    rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
-
-                    return;
-                }
-            }
-        }
-
-        // Process failures before processing join, otherwise conflicts are possible in case of fast node stop/re-start.
-        if (newEvts > 0) {
-            saveAndProcessNewEvents();
-
-            handleProcessedEventsOnNodesFail(failedNodes);
-
-            rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
-
-            return;
-        }
-
-        generateJoinEvents(curTop, alives, MAX_NEW_EVTS);
-
-        if (failedNodes != null)
-            handleProcessedEventsOnNodesFail(failedNodes);
-    }
-
-   private void generateJoinEvents(TreeMap<Long, ZookeeperClusterNode> curTop,
-        TreeMap<Long, String> alives,
-        final int MAX_NEW_EVTS) throws Exception
-   {
-       ZkBulkJoinContext joinCtx = new ZkBulkJoinContext();
-
-       for (Map.Entry<Long, String> e : alives.entrySet()) {
-           Long internalId = e.getKey();
-
-           if (!rtState.top.nodesByInternalId.containsKey(internalId)) {
-               UUID rslvFutId = rtState.evtsData.communicationErrorResolveFutureId();
-
-               if (rslvFutId != null) {
-                   if (log.isInfoEnabled()) {
-                       log.info("Delay alive nodes change process while communication error resolve " +
-                           "is in progress [reqId=" + rslvFutId + ']');
-                   }
-
-                   break;
-               }
-
-               processJoinOnCoordinator(joinCtx, curTop, internalId, e.getValue());
-
-               if (joinCtx.nodes() == MAX_NEW_EVTS) {
-                   generateBulkJoinEvent(curTop, joinCtx);
-
-                   if (log.isInfoEnabled()) {
-                       log.info("Delay alive nodes change process, max event threshold reached [" +
-                           "newEvts=" + joinCtx.nodes() +
-                           ", totalEvts=" + rtState.evtsData.evts.size() + ']');
-                   }
-
-                   throttleNewEventsGeneration();
-
-                   rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
-
-                   return;
-               }
-           }
-       }
-
-       if (joinCtx.nodes() > 0)
-           generateBulkJoinEvent(curTop, joinCtx);
-   }
-
-    private void generateBulkJoinEvent(TreeMap<Long, ZookeeperClusterNode> curTop, ZkBulkJoinContext joinCtx)
-        throws Exception
-    {
-        rtState.evtsData.evtIdGen++;
-
-        long evtId = rtState.evtsData.evtIdGen;
-
-        List<T2<ZkJoinedNodeEvtData, Map<Integer, Serializable>>> nodes = joinCtx.nodes;
-
-        assert nodes != null && nodes.size() > 0;
-
-        int nodeCnt = nodes.size();
-
-        List<ZkJoinedNodeEvtData> joinedNodes = new ArrayList<>(nodeCnt);
-
-        Map<Long, byte[]> discoDataMap = U.newHashMap(nodeCnt);
-        Map<Long, Long> dupDiscoData = null;
-
-        for (int i = 0; i < nodeCnt; i++) {
-            T2<ZkJoinedNodeEvtData, Map<Integer, Serializable>> nodeEvtData = nodes.get(i);
-
-            Map<Integer, Serializable> discoData = nodeEvtData.get2();
-
-            byte[] discoDataBytes = U.marshal(marsh, discoData);
-
-            Long dupDataNode = null;
-
-            for (Map.Entry<Long, byte[]> e : discoDataMap.entrySet()) {
-                if (Arrays.equals(discoDataBytes, e.getValue())) {
-                    dupDataNode = e.getKey();
-
-                    break;
-                }
-            }
-
-            long nodeTopVer = nodeEvtData.get1().topVer;
-
-            if (dupDataNode != null) {
-                if (dupDiscoData == null)
-                    dupDiscoData = new HashMap<>();
-
-                Long old = dupDiscoData.put(nodeTopVer, dupDataNode);
-
-                assert old == null : old;
-            }
-            else
-                discoDataMap.put(nodeTopVer, discoDataBytes);
-
-            joinedNodes.add(nodeEvtData.get1());
-        }
-
-        int overhead = 5;
-
-        ZkJoinEventDataForJoined dataForJoined = new ZkJoinEventDataForJoined(
-            new ArrayList<>(curTop.values()),
-            discoDataMap,
-            dupDiscoData);
-
-        byte[] dataForJoinedBytes = marshalZip(dataForJoined);
-
-        long addDataStart = System.currentTimeMillis();
-
-        int dataForJoinedPartCnt = saveData(zkPaths.joinEventDataPathForJoined(evtId),
-            dataForJoinedBytes,
-            overhead);
-
-        long addDataTime = System.currentTimeMillis() - addDataStart;
-
-        ZkDiscoveryNodeJoinEventData evtData = new ZkDiscoveryNodeJoinEventData(
-            evtId,
-            rtState.evtsData.topVer,
-            joinedNodes,
-            dataForJoinedPartCnt);
-
-        rtState.evtsData.addEvent(curTop.values(), evtData);
-
-        if (log.isInfoEnabled()) {
-            log.info("Generated NODE_JOINED event [" +
-                "nodeCnt=" + nodeCnt +
-                ", dataForJoinedSize=" + dataForJoinedBytes.length +
-                ", dataForJoinedPartCnt=" + dataForJoinedPartCnt +
-                ", addDataTime=" + addDataTime +
-                ", evt=" + evtData + ']');
-        }
-
-        saveAndProcessNewEvents();
-    }
-
-    /**
-     *
-     */
-    private void throttleNewEventsGeneration() {
-        long delay = IgniteSystemProperties.getLong(IGNITE_ZOOKEEPER_DISCOVERY_SPI_EVTS_THROTTLE, 0);
-
-        if (delay > 0) {
-            if (log.isInfoEnabled())
-                log.info("Sleep delay before generate new events [delay=" + delay + ']');
-
-            try {
-                Thread.sleep(delay);
-            }
-            catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-            }
-        }
-    }
-
-    /**
-     * @param nodeId Node ID.
-     * @param prefixId Path prefix.
-     * @return Join data.
-     * @throws Exception If failed.
-     */
-    private ZkJoiningNodeData unmarshalJoinData(UUID nodeId, UUID prefixId) throws Exception {
-        String joinDataPath = zkPaths.joiningNodeDataPath(nodeId, prefixId);
-
-        byte[] joinData = rtState.zkClient.getData(joinDataPath);
-
-        Object dataObj = unmarshalZip(joinData);
-
-        if (!(dataObj instanceof ZkJoiningNodeData))
-            throw new Exception("Invalid joined node data: " + dataObj);
-
-        ZkJoiningNodeData joiningNodeData = (ZkJoiningNodeData)dataObj;
-
-        if (joiningNodeData.partCount() > 1) {
-            joinData = readMultipleParts(rtState.zkClient, joinDataPath + ":", joiningNodeData.partCount());
-
-            joiningNodeData = unmarshalZip(joinData);
-        }
-
-        return joiningNodeData;
-    }
-
-    /**
-     * @param nodeId Node ID.
-     * @param prefixId Path prefix.
-     * @param aliveNodePath Node path.
-     * @return Join data.
-     * @throws Exception If failed.
-     */
-    private Object unmarshalJoinDataOnCoordinator(UUID nodeId, UUID prefixId, String aliveNodePath) throws Exception {
-        String joinDataPath = zkPaths.joiningNodeDataPath(nodeId, prefixId);
-
-        byte[] joinData = rtState.zkClient.getData(joinDataPath);
-
-        Object dataObj;
-
-        try {
-            dataObj = unmarshalZip(joinData);
-
-            if (dataObj instanceof ZkInternalJoinErrorMessage)
-                return dataObj;
-        }
-        catch (Exception e) {
-            U.error(log, "Failed to unmarshal joining node data [nodePath=" + aliveNodePath + "']", e);
-
-            return new ZkInternalJoinErrorMessage(ZkIgnitePaths.aliveInternalId(aliveNodePath),
-                "Failed to unmarshal join data: " + e);
-        }
-
-        assert dataObj instanceof ZkJoiningNodeData : dataObj;
-
-        ZkJoiningNodeData joiningNodeData = (ZkJoiningNodeData)dataObj;
-
-        if (joiningNodeData.partCount() > 1) {
-            joinData = readMultipleParts(rtState.zkClient, joinDataPath + ":", joiningNodeData.partCount());
-
-            try {
-                joiningNodeData = unmarshalZip(joinData);
-            }
-            catch (Exception e) {
-                U.error(log, "Failed to unmarshal joining node data [nodePath=" + aliveNodePath + "']", e);
-
-                return new ZkInternalJoinErrorMessage(ZkIgnitePaths.aliveInternalId(aliveNodePath),
-                    "Failed to unmarshal join data: " + e);
-            }
-        }
-
-        assert joiningNodeData.node() != null : joiningNodeData;
-
-        return joiningNodeData;
-    }
-
-    /**
-     * @param curTop Current nodes.
-     * @param internalId Joined node internal ID.
-     * @param aliveNodePath Joined node path.
-     * @throws Exception If failed.
-     * @return {@code True} if new join event was added.
-     */
-    private boolean processJoinOnCoordinator(
-        ZkBulkJoinContext joinCtx,
-        TreeMap<Long, ZookeeperClusterNode> curTop,
-        long internalId,
-        String aliveNodePath)
-        throws Exception
-    {
-        UUID nodeId = ZkIgnitePaths.aliveNodeId(aliveNodePath);
-        UUID prefixId = ZkIgnitePaths.aliveNodePrefixId(aliveNodePath);
-
-        Object data = unmarshalJoinDataOnCoordinator(nodeId, prefixId, aliveNodePath);
-
-        if (data instanceof ZkJoiningNodeData) {
-            ZkJoiningNodeData joiningNodeData = (ZkJoiningNodeData)data;
-
-            ZkNodeValidateResult validateRes = validateJoiningNode(joiningNodeData.node());
-
-            if (validateRes.err == null) {
-                ZookeeperClusterNode joinedNode = joiningNodeData.node();
-
-                assert nodeId.equals(joinedNode.id()) : joiningNodeData.node();
-
-                addJoinedNode(
-                    joinCtx,
-                    curTop,
-                    joiningNodeData,
-                    internalId,
-                    prefixId,
-                    validateRes.secSubjZipBytes);
-
-                watchAliveNodeData(aliveNodePath);
-
-                return true;
-            }
-            else {
-                ZkInternalJoinErrorMessage joinErr = new ZkInternalJoinErrorMessage(
-                    ZkIgnitePaths.aliveInternalId(aliveNodePath),
-                    validateRes.err);
-
-                processJoinError(aliveNodePath, nodeId, prefixId, joinErr);
-
-                return false;
-            }
-        }
-        else {
-            assert data instanceof ZkInternalJoinErrorMessage : data;
-
-            processJoinError(aliveNodePath, nodeId, prefixId, (ZkInternalJoinErrorMessage)data);
-
-            return false;
-        }
-    }
-
-    /**
-     * @param aliveNodePath Joined node path.
-     * @param nodeId Node ID.
-     * @param prefixId Path prefix ID.
-     * @param joinErr Join error message.
-     * @throws Exception If failed.
-     */
-    private void processJoinError(String aliveNodePath,
-        UUID nodeId,
-        UUID prefixId,
-        ZkInternalJoinErrorMessage joinErr) throws Exception {
-        ZookeeperClient client = rtState.zkClient;
-
-        if (joinErr.notifyNode) {
-            String joinDataPath = zkPaths.joiningNodeDataPath(nodeId, prefixId);
-
-            client.setData(joinDataPath, marshalZip(joinErr), -1);
-
-            client.deleteIfExists(zkPaths.aliveNodesDir + "/" + aliveNodePath, -1);
-        }
-        else {
-            if (log.isInfoEnabled())
-                log.info("Ignore join data, node was failed by previous coordinator: " + aliveNodePath);
-
-            client.deleteIfExists(zkPaths.aliveNodesDir + "/" + aliveNodePath, -1);
-        }
-    }
-
-    /**
-     * @param node Joining node.
-     * @return Validation result.
-     */
-    private ZkNodeValidateResult validateJoiningNode(ZookeeperClusterNode node) {
-        ZookeeperClusterNode node0 = rtState.top.nodesById.get(node.id());
-
-        if (node0 != null) {
-            U.error(log, "Failed to include node in cluster, node with the same ID already exists [joiningNode=" + node +
-                ", existingNode=" + node0 + ']');
-
-            // Note: exception message is checked in tests.
-            return new ZkNodeValidateResult("Node with the same ID already exists: " + node0);
-        }
-
-        ZkNodeValidateResult res = authenticateNode(node);
-
-        if (res.err != null)
-            return res;
-
-        IgniteNodeValidationResult err = spi.getSpiContext().validateNode(node);
-
-        if (err != null) {
-            LT.warn(log, err.message());
-
-            res.err = err.sendMessage();
-        }
-
-        return res;
-    }
-
-    /**
-     * @param node Node.
-     * @return Validation result.
-     */
-    private ZkNodeValidateResult authenticateNode(ZookeeperClusterNode node) {
-        DiscoverySpiNodeAuthenticator nodeAuth = spi.getAuthenticator();
-
-        if (nodeAuth == null)
-            return new ZkNodeValidateResult((byte[])null);
-
-        SecurityCredentials cred;
-
-        try {
-            cred = unmarshalCredentials(node);
-        }
-        catch (Exception e) {
-            U.error(log, "Failed to unmarshal node credentials: " + e, e);
-
-            return new ZkNodeValidateResult("Failed to unmarshal node credentials");
-        }
-
-        SecurityContext subj = nodeAuth.authenticateNode(node, cred);
-
-        if (subj == null) {
-            U.warn(log, "Authentication failed [nodeId=" + node.id() +
-                    ", addrs=" + U.addressesAsString(node) + ']',
-                "Authentication failed [nodeId=" + U.id8(node.id()) + ", addrs=" +
-                    U.addressesAsString(node) + ']');
-
-            // Note: exception message test is checked in tests.
-            return new ZkNodeValidateResult("Authentication failed");
-        }
-
-        if (!(subj instanceof Serializable)) {
-            U.warn(log, "Authentication subject is not Serializable [nodeId=" + node.id() +
-                    ", addrs=" + U.addressesAsString(node) + ']',
-                "Authentication subject is not Serializable [nodeId=" + U.id8(node.id()) +
-                    ", addrs=" +
-                    U.addressesAsString(node) + ']');
-
-            return new ZkNodeValidateResult("Authentication subject is not serializable");
-        }
-
-        byte[] secSubjZipBytes;
-
-        try {
-            secSubjZipBytes = marshalZip(subj);
-        }
-        catch (Exception e) {
-            U.error(log, "Failed to marshal node security subject: " + e, e);
-
-            return new ZkNodeValidateResult("Failed to marshal node security subject");
-        }
-
-        return new ZkNodeValidateResult(secSubjZipBytes);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    private void saveAndProcessNewEvents() throws Exception {
-        long start = System.currentTimeMillis();
-
-        byte[] evtsBytes = marshalZip(rtState.evtsData);
-
-        rtState.zkClient.setData(zkPaths.evtsPath, evtsBytes, -1);
-
-        long time = System.currentTimeMillis() - start;
-
-        if (log.isInfoEnabled()) {
-            log.info("Discovery coordinator saved new topology events [topVer=" + rtState.evtsData.topVer +
-                ", size=" + evtsBytes.length +
-                ", evts=" + rtState.evtsData.evts.size() +
-                ", lastEvt=" + rtState.evtsData.evtIdGen +
-                ", saveTime=" + time + ']');
-        }
-
-        processNewEvents(rtState.evtsData);
-    }
-
-    /**
-     * @param curTop Current topology.
-     * @param failedNode Failed node.
-     */
-    private void generateNodeFail(TreeMap<Long, ZookeeperClusterNode> curTop, ZookeeperClusterNode failedNode) {
-        Object rmvd = curTop.remove(failedNode.order());
-
-        assert rmvd != null;
-
-        rtState.evtsData.topVer++;
-        rtState.evtsData.evtIdGen++;
-
-        ZkDiscoveryNodeFailEventData evtData = new ZkDiscoveryNodeFailEventData(
-            rtState.evtsData.evtIdGen,
-            rtState.evtsData.topVer,
-            failedNode.internalId());
-
-        rtState.evtsData.addEvent(curTop.values(), evtData);
-
-        if (log.isInfoEnabled())
-            log.info("Generated NODE_FAILED event [evt=" + evtData + ']');
-    }
-
-    /**
-     * @param curTop Current nodes.
-     * @param joiningNodeData Join data.
-     * @param internalId Joined node internal ID.
-     * @param prefixId Unique path prefix.
-     * @param secSubjZipBytes Marshalled security subject.
-     * @throws Exception If failed.
-     */
-    private void addJoinedNode(
-        ZkBulkJoinContext joinCtx,
-        TreeMap<Long, ZookeeperClusterNode> curTop,
-        ZkJoiningNodeData joiningNodeData,
-        long internalId,
-        UUID prefixId,
-        @Nullable byte[] secSubjZipBytes)
-        throws Exception
-    {
-        ZookeeperClusterNode joinedNode = joiningNodeData.node();
-
-        UUID nodeId = joinedNode.id();
-
-        rtState.evtsData.topVer++;
-
-        joinedNode.order(rtState.evtsData.topVer);
-        joinedNode.internalId(internalId);
-
-        DiscoveryDataBag joiningNodeBag = new DiscoveryDataBag(nodeId);
-
-        joiningNodeBag.joiningNodeData(joiningNodeData.discoveryData());
-
-        exchange.onExchange(joiningNodeBag);
-
-        DiscoveryDataBag collectBag = new DiscoveryDataBag(nodeId, new HashSet<Integer>());
-
-        exchange.collect(collectBag);
-
-        Map<Integer, Serializable> commonData = collectBag.commonData();
-
-        Object old = curTop.put(joinedNode.order(), joinedNode);
-
-        assert old == null;
-
-        int overhead = 5;
-
-        int secSubjPartCnt = 0;
-
-        if (secSubjZipBytes != null) {
-            secSubjPartCnt = saveData(zkPaths.joinEventSecuritySubjectPath(joinedNode.order()),
-                secSubjZipBytes,
-                overhead);
-
-            assert secSubjPartCnt > 0 : secSubjPartCnt;
-
-            setNodeSecuritySubject(joinedNode, secSubjZipBytes);
-        }
-
-        ZkJoinedNodeEvtData nodeEvtData = new ZkJoinedNodeEvtData(
-            rtState.evtsData.topVer,
-            joinedNode.id(),
-            joinedNode.internalId(),
-            prefixId,
-            joiningNodeData.partCount(),
-            secSubjPartCnt);
-
-        nodeEvtData.joiningNodeData = joiningNodeData;
-
-        joinCtx.addJoinedNode(nodeEvtData, commonData);
-
-        rtState.evtsData.onNodeJoin(joinedNode);
-    }
-
-    /**
-     * @param path Path to save.
-     * @param bytes Bytes to save.
-     * @param overhead Extra overhead.
-     * @return Parts count.
-     * @throws Exception If failed.
-     */
-    private int saveData(String path, byte[] bytes, int overhead) throws Exception {
-        int dataForJoinedPartCnt = 1;
-
-        if (rtState.zkClient.needSplitNodeData(path, bytes, overhead)) {
-            dataForJoinedPartCnt = saveMultipleParts(rtState.zkClient,
-                path,
-                rtState.zkClient.splitNodeData(path, bytes, overhead));
-        }
-        else {
-            rtState.zkClient.createIfNeeded(multipartPathName(path, 0),
-                bytes,
-                PERSISTENT);
-        }
-
-        return dataForJoinedPartCnt;
-    }
-
-    /**
-     * @param prevEvts Events from previous cluster.
-     * @throws Exception If failed.
-     */
-    @SuppressWarnings("unchecked")
-    private void newClusterStarted(@Nullable ZkDiscoveryEventsData prevEvts) throws Exception {
-        assert !locNode.isClient() : locNode;
-
-        long locInternalId = rtState.internalOrder;
-
-        assert prevEvts == null || prevEvts.maxInternalOrder < locInternalId;
-
-        spi.getSpiContext().removeTimeoutObject(rtState.joinErrTo);
-
-        cleanupPreviousClusterData(prevEvts != null ? prevEvts.maxInternalOrder + 1 : -1L);
-
-        rtState.joined = true;
-        rtState.gridStartTime = System.currentTimeMillis();
-
-        rtState.evtsData = ZkDiscoveryEventsData.createForNewCluster(rtState.gridStartTime);
-
-        if (log.isInfoEnabled()) {
-            log.info("New cluster started [locId=" + locNode.id() +
-                ", clusterId=" + rtState.evtsData.clusterId +
-                ", startTime=" + rtState.evtsData.clusterStartTime + ']');
-        }
-
-        locNode.internalId(locInternalId);
-        locNode.order(1);
-
-        rtState.evtsData.onNodeJoin(locNode);
-
-        rtState.top.addNode(locNode);
-
-        final List<ClusterNode> topSnapshot = Collections.singletonList((ClusterNode)locNode);
-
-        lsnr.onDiscovery(EVT_NODE_JOINED,
-            1L,
-            locNode,
-            topSnapshot,
-            Collections.<Long, Collection<ClusterNode>>emptyMap(),
-            null);
-
-        // Reset events (this is also notification for clients left from previous cluster).
-        rtState.zkClient.setData(zkPaths.evtsPath, marshalZip(rtState.evtsData), -1);
-
-        joinFut.onDone();
-    }
-
-    /**
-     * @param startInternalOrder Starting internal order for cluster (znodes having lower order belong
-     *      to clients from previous cluster and should be removed).
-
-     * @throws Exception If failed.
-     */
-    private void cleanupPreviousClusterData(long startInternalOrder) throws Exception {
-        long start = System.currentTimeMillis();
-
-        ZookeeperClient client = rtState.zkClient;
-
-        // TODO ZK: use multi, better batching + max-size safe + NoNodeException safe.
-        List<String> evtChildren = rtState.zkClient.getChildren(zkPaths.evtsPath);
-
-        for (String evtPath : evtChildren) {
-            String evtDir = zkPaths.evtsPath + "/" + evtPath;
-
-            removeChildren(evtDir);
-        }
-
-        client.deleteAll(zkPaths.evtsPath, evtChildren, -1);
-
-        client.deleteAll(zkPaths.customEvtsDir,
-            client.getChildren(zkPaths.customEvtsDir),
-            -1);
-
-        rtState.zkClient.deleteAll(zkPaths.customEvtsPartsDir,
-            rtState.zkClient.getChildren(zkPaths.customEvtsPartsDir),
-            -1);
-
-        rtState.zkClient.deleteAll(zkPaths.customEvtsAcksDir,
-            rtState.zkClient.getChildren(zkPaths.customEvtsAcksDir),
-            -1);
-
-        if (startInternalOrder > 0) {
-            for (String alive : rtState.zkClient.getChildren(zkPaths.aliveNodesDir)) {
-                if (ZkIgnitePaths.aliveInternalId(alive) < startInternalOrder)
-                    rtState.zkClient.deleteIfExists(zkPaths.aliveNodesDir + "/" + alive, -1);
-            }
-        }
-
-        long time = System.currentTimeMillis() - start;
-
-        if (time > 0) {
-            if (log.isInfoEnabled())
-                log.info("Previous cluster data cleanup time: " + time);
-        }
-    }
-
-    /**
-     * @param path Path.
-     * @throws Exception If failed.
-     */
-    private void removeChildren(String path) throws Exception {
-        rtState.zkClient.deleteAll(path, rtState.zkClient.getChildren(path), -1);
-    }
-
-    /**
-     * @param zkClient Client.
-     * @param evtPath Event path.
-     * @param sndNodeId Sender node ID.
-     * @return Event data.
-     * @throws Exception If failed.
-     */
-    private byte[] readCustomEventData(ZookeeperClient zkClient, String evtPath, UUID sndNodeId) throws Exception {
-        int partCnt = ZkIgnitePaths.customEventPartsCount(evtPath);
-
-        if (partCnt > 1) {
-            String partsBasePath = zkPaths.customEventPartsBasePath(
-                ZkIgnitePaths.customEventPrefix(evtPath), sndNodeId);
-
-            return readMultipleParts(zkClient, partsBasePath, partCnt);
-        }
-        else
-            return zkClient.getData(zkPaths.customEvtsDir + "/" + evtPath);
-    }
-
-    /**
-     * @param customEvtNodes ZK nodes representing custom events to process.
-     * @throws Exception If failed.
-     */
-    private void generateCustomEvents(List<String> customEvtNodes) throws Exception {
-        assert rtState.crd;
-
-        ZookeeperClient zkClient = rtState.zkClient;
-        ZkDiscoveryEventsData evtsData = rtState.evtsData;
-
-        TreeMap<Integer, String> unprocessedEvts = null;
-
-        for (int i = 0; i < customEvtNodes.size(); i++) {
-            String evtPath = customEvtNodes.get(i);
-
-            int evtSeq = ZkIgnitePaths.customEventSequence(evtPath);
-
-            if (evtSeq > evtsData.procCustEvt) {
-                if (unprocessedEvts == null)
-                    unprocessedEvts = new TreeMap<>();
-
-                unprocessedEvts.put(evtSeq, evtPath);
-            }
-        }
-
-        if (unprocessedEvts == null)
-            return;
-
-        for (Map.Entry<Integer, String> evtE : unprocessedEvts.entrySet()) {
-            evtsData.procCustEvt = evtE.getKey();
-
-            String evtPath = evtE.getValue();
-
-            UUID sndNodeId = ZkIgnitePaths.customEventSendNodeId(evtPath);
-
-            ZookeeperClusterNode sndNode = rtState.top.nodesById.get(sndNodeId);
-
-            if (sndNode != null) {
-                byte[] evtBytes = readCustomEventData(zkClient, evtPath, sndNodeId);
-
-                DiscoverySpiCustomMessage msg;
-
-                try {
-                    msg = unmarshalZip(evtBytes);
-                }
-                catch (Exception e) {
-                    U.error(log, "Failed to unmarshal custom discovery message: " + e, e);
-
-                    deleteCustomEventDataAsync(rtState.zkClient, evtPath);
-
-                    continue;
-                }
-
-                generateAndProcessCustomEventOnCoordinator(evtPath, sndNode, msg);
-            }
-            else {
-                U.warn(log, "Ignore custom event from unknown node: " + sndNodeId);
-
-                deleteCustomEventDataAsync(rtState.zkClient, evtPath);
-            }
-        }
-    }
-
-    /**
-     * @param evtPath Event data path.
-     * @param sndNode Sender node.
-     * @param msg Message instance.
-     * @throws Exception If failed.
-     */
-    private void generateAndProcessCustomEventOnCoordinator(String evtPath,
-        ZookeeperClusterNode sndNode,
-        DiscoverySpiCustomMessage msg) throws Exception
-    {
-        ZookeeperClient zkClient = rtState.zkClient;
-        ZkDiscoveryEventsData evtsData = rtState.evtsData;
-
-        ZookeeperClusterNode failedNode = null;
-
-        if (msg instanceof ZkForceNodeFailMessage) {
-            ZkForceNodeFailMessage msg0 = (ZkForceNodeFailMessage)msg;
-
-            failedNode = rtState.top.nodesByInternalId.get(msg0.nodeInternalId);
-
-            if (failedNode != null)
-                evtsData.topVer++;
-            else {
-                if (log.isDebugEnabled())
-                    log.debug("Ignore forcible node fail request for unknown node: " + msg0.nodeInternalId);
-
-                deleteCustomEventDataAsync(zkClient, evtPath);
-
-                return;
-            }
-        }
-        else if (msg instanceof ZkCommunicationErrorResolveStartMessage) {
-            ZkCommunicationErrorResolveStartMessage msg0 =
-                (ZkCommunicationErrorResolveStartMessage)msg;
-
-            if (evtsData.communicationErrorResolveFutureId() != null) {
-                if (log.isInfoEnabled()) {
-                    log.info("Ignore communication error resolve message, resolve process " +
-                        "already started [sndNode=" + sndNode + ']');
-                }
-
-                deleteCustomEventDataAsync(zkClient, evtPath);
-
-                return;
-            }
-            else {
-                if (log.isInfoEnabled()) {
-                    log.info("Start cluster-wide communication error resolve [sndNode=" + sndNode +
-                        ", reqId=" + msg0.id +
-                        ", topVer=" + evtsData.topVer + ']');
-                }
-
-                zkClient.createIfNeeded(zkPaths.distributedFutureBasePath(msg0.id),
-                    null,
-                    PERSISTENT);
-
-                evtsData.communicationErrorResolveFutureId(msg0.id);
-            }
-        }
-
-        evtsData.evtIdGen++;
-
-        ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
-            evtsData.evtIdGen,
-            0L,
-            evtsData.topVer,
-            sndNode.id(),
-            null,
-            evtPath);
-
-        evtData.resolvedMsg = msg;
-
-        if (log.isDebugEnabled())
-            log.debug("Generated CUSTOM event [evt=" + evtData + ", msg=" + msg + ']');
-
-        boolean fastStopProcess = false;
-
-        if (msg instanceof ZkInternalMessage)
-            processInternalMessage(evtData, (ZkInternalMessage)msg);
-        else {
-            notifyCustomEvent(evtData, msg);
-
-            if (msg.stopProcess()) {
-                if (log.isDebugEnabled())
-                    log.debug("Fast stop process custom event [evt=" + evtData + ", msg=" + msg + ']');
-
-                fastStopProcess = true;
-
-                // No need to process this event on others nodes, skip this event.
-                evtsData.evts.remove(evtData.eventId());
-
-                evtsData.evtIdGen--;
-
-                DiscoverySpiCustomMessage ack = msg.ackMessage();
-
-                if (ack != null) {
-                    evtData = createAckEvent(ack, evtData);
-
-                    if (log.isDebugEnabled())
-                        log.debug("Generated CUSTOM event (ack for fast stop process) [evt=" + evtData + ", msg=" + msg + ']');
-
-                    notifyCustomEvent(evtData, ack);
-                }
-                else
-                    evtData = null;
-            }
-        }
-
-        if (evtData != null) {
-            evtsData.addEvent(rtState.top.nodesByOrder.values(), evtData);
-
-            rtState.locNodeInfo.lastProcEvt = evtData.eventId();
-
-            saveAndProcessNewEvents();
-
-            if (fastStopProcess)
-                deleteCustomEventDataAsync(zkClient, evtPath);
-
-            if (failedNode != null) {
-                deleteAliveNode(failedNode.internalId());
-
-                handleProcessedEventsOnNodesFail(Collections.singletonList(failedNode));
-            }
-        }
-    }
-
-    /**
-     * @param internalId Node internal ID.
-     * @throws Exception If failed.
-     */
-    private void deleteAliveNode(long internalId) throws Exception {
-        for (String child : rtState.zkClient.getChildren(zkPaths.aliveNodesDir)) {
-            if (ZkIgnitePaths.aliveInternalId(child) == internalId) {
-                rtState.zkClient.deleteIfExistsAsync(zkPaths.aliveNodesDir + "/" + child);
-
-                break;
-            }
-        }
-    }
-
-    /**
-     * @param zkClient Client.
-     * @param evtPath Event path.
-     */
-    private void deleteCustomEventDataAsync(ZookeeperClient zkClient, String evtPath) {
-        if (log.isDebugEnabled())
-            log.debug("Delete custom event data: " + evtPath);
-
-        String prefix = ZkIgnitePaths.customEventPrefix(evtPath);
-        UUID sndNodeId = ZkIgnitePaths.customEventSendNodeId(evtPath);
-        int partCnt = ZkIgnitePaths.customEventPartsCount(evtPath);
-
-        assert partCnt >= 1 : partCnt;
-
-        if (partCnt > 1) {
-            for (int i = 0; i < partCnt; i++) {
-                String path0 = zkPaths.customEventPartPath(prefix, sndNodeId, i);
-
-                zkClient.deleteIfExistsAsync(path0);
-            }
-        }
-
-        zkClient.deleteIfExistsAsync(zkPaths.customEvtsDir + "/" + evtPath);
-    }
-
-    /**
-     * @param data Marshalled events.
-     * @throws Exception If failed.
-     * @return Events.
-     */
-    @Nullable private ZkDiscoveryEventsData processNewEvents(byte[] data) throws Exception {
-        ZkDiscoveryEventsData newEvts = data.length > 0 ? (ZkDiscoveryEventsData)unmarshalZip(data) : null;
-
-        if (rtState.joined && (newEvts == null || !rtState.evtsData.clusterId.equals(newEvts.clusterId))) {
-            assert locNode.isClient() : locNode;
-
-            throw localNodeFail("All server nodes failed, client node disconnected (received events from new custer) " +
-                "[locId=" + locNode.id() + ']', true);
-        }
-
-        if (newEvts == null)
-            return null;
-
-        assert !rtState.crd;
-
-        // Need keep processed custom events since they contain message object which is needed to create ack.
-        if (rtState.evtsData != null) {
-            for (Map.Entry<Long, ZkDiscoveryEventData> e : rtState.evtsData.evts.entrySet()) {
-                ZkDiscoveryEventData evtData = e.getValue();
-
-                if (evtData.eventType() == ZkDiscoveryEventData.ZK_EVT_CUSTOM_EVT) {
-                    ZkDiscoveryCustomEventData evtData0 =
-                        (ZkDiscoveryCustomEventData)newEvts.evts.get(evtData.eventId());
-
-                    if (evtData0 != null)
-                        evtData0.resolvedMsg = ((ZkDiscoveryCustomEventData)evtData).resolvedMsg;
-                }
-            }
-        }
-
-        processNewEvents(newEvts);
-
-        if (rtState.joined)
-            rtState.evtsData = newEvts;
-
-        return newEvts;
-    }
-
-    /**
-     * @param evtsData Events.
-     * @throws Exception If failed.
-     */
-    @SuppressWarnings("unchecked")
-    private void processNewEvents(final ZkDiscoveryEventsData evtsData) throws Exception {
-        TreeMap<Long, ZkDiscoveryEventData> evts = evtsData.evts;
-
-        ZookeeperClient zkClient = rtState.zkClient;
-
-        boolean evtProcessed = false;
-        boolean updateNodeInfo = false;
-
-        try {
-            for (ZkDiscoveryEventData evtData : evts.tailMap(rtState.locNodeInfo.lastProcEvt, false).values()) {
-                if (log.isDebugEnabled())
-                    log.debug("New discovery event data [evt=" + evtData + ", evtsHist=" + evts.size() + ']');
-
-                switch (evtData.eventType()) {
-                    case ZkDiscoveryEventData.ZK_EVT_NODE_JOIN: {
-                        evtProcessed = processBulkJoin(evtsData, (ZkDiscoveryNodeJoinEventData)evtData);
-
-                        break;
-                    }
-
-                    case ZkDiscoveryEventData.ZK_EVT_NODE_FAILED: {
-                        if (!rtState.joined)
-                            break;
-
-                        evtProcessed = true;
-
-                        notifyNodeFail((ZkDiscoveryNodeFailEventData)evtData);
-
-                        break;
-                    }
-
-                    case ZkDiscoveryEventData.ZK_EVT_CUSTOM_EVT: {
-                        if (!rtState.joined)
-                            break;
-
-                        evtProcessed = true;
-
-                        ZkDiscoveryCustomEventData evtData0 = (ZkDiscoveryCustomEventData)evtData;
-
-                        if (evtData0.ackEvent() && evtData0.topologyVersion() < locNode.order())
-                            break;
-
-                        DiscoverySpiCustomMessage msg;
-
-                        if (rtState.crd) {
-                            assert evtData0.resolvedMsg != null : evtData0;
-
-                            m

<TRUNCATED>