You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/11/09 21:19:22 UTC

[16/50] [abbrv] lucene-solr:jira/solr-11285-sim: SOLR-11003: Support bi-directional syncing of cdcr clusters. We still only support actively into one cluster cluster, but have the ability to switch indexing clusters and cdcr will replicate correctly

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/db18de7e/solr/core/src/test/org/apache/solr/cloud/cdcr/BaseCdcrDistributedZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/cdcr/BaseCdcrDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/cdcr/BaseCdcrDistributedZkTest.java
new file mode 100644
index 0000000..e3a00a8
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/cdcr/BaseCdcrDistributedZkTest.java
@@ -0,0 +1,907 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud.cdcr;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.AbstractZkTestCase;
+import org.apache.solr.cloud.ChaosMonkey;
+import org.apache.solr.cloud.OverseerCollectionMessageHandler;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkCoreNodeProps;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.CdcrParams;
+import org.apache.solr.util.TimeOut;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET;
+import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES;
+import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARDS_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.CLUSTER_PROPS;
+import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
+import static org.apache.solr.handler.admin.CoreAdminHandler.COMPLETED;
+import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_STATUS;
+
+/**
+ * <p>
+ * Abstract class for CDCR unit testing. This class emulates two clusters, a source and target, by using different
+ * collections in the same SolrCloud cluster. Therefore, the two clusters will share the same Zookeeper cluster. In
+ * real scenario, the two collections/clusters will likely have their own zookeeper cluster.
+ * </p>
+ * <p>
+ * This class will automatically create two collections, the source and the target. Each collection will have
+ * {@link #shardCount} shards, and {@link #replicationFactor} replicas per shard. One jetty instance will
+ * be created per core.
+ * </p>
+ * <p>
+ * The source and target collection can be reinitialised at will by calling {@link #clearSourceCollection()} and
+ * {@link #clearTargetCollection()}. After reinitialisation, a collection will have a new fresh index and update log.
+ * </p>
+ * <p>
+ * Servers can be restarted at will by calling
+ * {@link #restartServer(BaseCdcrDistributedZkTest.CloudJettyRunner)} or
+ * {@link #restartServers(java.util.List)}.
+ * </p>
+ * <p>
+ * The creation of the target collection can be disabled with the flag {@link #createTargetCollection};
+ * </p>
+ * <p>
+ * NB: We cannot use multiple cores per jetty instance, as jetty will load only one core when restarting. It seems
+ * that this is a limitation of the {@link org.apache.solr.client.solrj.embedded.JettySolrRunner}. This class
+ * tries to ensure that there always is one single core per jetty instance.
+ * </p>
+ */
+public class BaseCdcrDistributedZkTest extends AbstractDistribZkTestBase {
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  protected int shardCount = 2;
+  protected int replicationFactor = 2;
+  protected boolean createTargetCollection = true;
+
+  private static final String CDCR_PATH = "/cdcr";
+
+  protected static final String SOURCE_COLLECTION = "source_collection";
+  protected static final String TARGET_COLLECTION = "target_collection";
+
+  public static final String SHARD1 = "shard1";
+  public static final String SHARD2 = "shard2";
+
+  @Override
+  protected String getCloudSolrConfig() {
+    return "solrconfig-cdcr.xml";
+  }
+
+  @Override
+  public void distribSetUp() throws Exception {
+    super.distribSetUp();
+
+    if (shardCount > 0) {
+      System.setProperty("numShards", Integer.toString(shardCount));
+    } else {
+      System.clearProperty("numShards");
+    }
+
+    if (isSSLMode()) {
+      System.clearProperty("urlScheme");
+      ZkStateReader zkStateReader = new ZkStateReader(zkServer.getZkAddress(),
+          AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT);
+      try {
+        zkStateReader.getZkClient().create(ZkStateReader.CLUSTER_PROPS,
+            Utils.toJSON(Collections.singletonMap("urlScheme", "https")),
+            CreateMode.PERSISTENT, true);
+      } catch (KeeperException.NodeExistsException e) {
+        ZkNodeProps props = ZkNodeProps.load(zkStateReader.getZkClient().getData(ZkStateReader.CLUSTER_PROPS,
+            null, null, true));
+        props = props.plus("urlScheme", "https");
+        zkStateReader.getZkClient().setData(CLUSTER_PROPS, Utils.toJSON(props), true);
+      } finally {
+        zkStateReader.close();
+      }
+    }
+  }
+
+  @Override
+  protected void createServers(int numServers) throws Exception {
+  }
+
+  @BeforeClass
+  public static void beforeClass() {
+    System.setProperty("solrcloud.update.delay", "0");
+  }
+
+  @AfterClass
+  public static void afterClass() throws Exception {
+    System.clearProperty("solrcloud.update.delay");
+  }
+
+  @Before
+  public void baseBefore() throws Exception {
+    this.createSourceCollection();
+    if (this.createTargetCollection) this.createTargetCollection();
+    RandVal.uniqueValues = new HashSet(); //reset random values
+  }
+
+  @After
+  public void baseAfter() throws Exception {
+    for (List<CloudJettyRunner> runners : cloudJettys.values()) {
+      for (CloudJettyRunner runner : runners) {
+        runner.client.close();
+      }
+    }
+    destroyServers();
+  }
+
+  protected CloudSolrClient createCloudClient(String defaultCollection) {
+    CloudSolrClient server = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean());
+    server.setParallelUpdates(random().nextBoolean());
+    if (defaultCollection != null) server.setDefaultCollection(defaultCollection);
+    return server;
+  }
+
+  protected void printLayout() throws Exception {
+    SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT);
+    zkClient.printLayoutToStdOut();
+    zkClient.close();
+  }
+
+  protected SolrInputDocument getDoc(Object... fields) throws Exception {
+    SolrInputDocument doc = new SolrInputDocument();
+    addFields(doc, fields);
+    return doc;
+  }
+
+  protected void index(String collection, SolrInputDocument doc) throws IOException, SolrServerException {
+    CloudSolrClient client = createCloudClient(collection);
+    try {
+      client.add(doc);
+      client.commit(true, true);
+    } finally {
+      client.close();
+    }
+  }
+
+  protected void index(String collection, List<SolrInputDocument> docs) throws IOException, SolrServerException {
+    CloudSolrClient client = createCloudClient(collection);
+    try {
+      client.add(docs);
+      client.commit(true, true);
+    } finally {
+      client.close();
+    }
+  }
+
+  protected void deleteById(String collection, List<String> ids) throws IOException, SolrServerException {
+    CloudSolrClient client = createCloudClient(collection);
+    try {
+      client.deleteById(ids);
+      client.commit(true, true);
+    } finally {
+      client.close();
+    }
+  }
+
+  protected void deleteByQuery(String collection, String q) throws IOException, SolrServerException {
+    CloudSolrClient client = createCloudClient(collection);
+    try {
+      client.deleteByQuery(q);
+      client.commit(true, true);
+    } finally {
+      client.close();
+    }
+  }
+
+  /**
+   * Invokes a commit on the given collection.
+   */
+  protected void commit(String collection) throws IOException, SolrServerException {
+    CloudSolrClient client = createCloudClient(collection);
+    try {
+      client.commit(true, true);
+    } finally {
+      client.close();
+    }
+  }
+
+  /**
+   * Assert the number of documents in a given collection
+   */
+  protected void assertNumDocs(int expectedNumDocs, String collection)
+  throws SolrServerException, IOException, InterruptedException {
+    CloudSolrClient client = createCloudClient(collection);
+    try {
+      int cnt = 30; // timeout after 15 seconds
+      AssertionError lastAssertionError = null;
+      while (cnt > 0) {
+        try {
+          assertEquals(expectedNumDocs, client.query(new SolrQuery("*:*")).getResults().getNumFound());
+          return;
+        }
+        catch (AssertionError e) {
+          lastAssertionError = e;
+          cnt--;
+          Thread.sleep(500);
+        }
+      }
+      throw new AssertionError("Timeout while trying to assert number of documents @ " + collection, lastAssertionError);
+    } finally {
+      client.close();
+    }
+  }
+
+  /**
+   * Invokes a CDCR action on a given node.
+   */
+  protected NamedList invokeCdcrAction(CloudJettyRunner jetty, CdcrParams.CdcrAction action) throws Exception {
+    ModifiableSolrParams params = new ModifiableSolrParams();
+    params.set(CommonParams.ACTION, action.toString());
+
+    SolrRequest request = new QueryRequest(params);
+    request.setPath(CDCR_PATH);
+
+    return jetty.client.request(request);
+  }
+
+  protected void waitForCdcrStateReplication(String collection) throws Exception {
+    log.info("Wait for CDCR state to replicate - collection: " + collection);
+
+    int cnt = 30;
+    while (cnt > 0) {
+      NamedList status = null;
+      boolean allEquals = true;
+      for (CloudJettyRunner jetty : cloudJettys.get(collection)) { // check all replicas
+        NamedList rsp = invokeCdcrAction(jetty, CdcrParams.CdcrAction.STATUS);
+        if (status == null) {
+          status = (NamedList) rsp.get(CdcrParams.CdcrAction.STATUS.toLower());
+          continue;
+        }
+        allEquals &= status.equals(rsp.get(CdcrParams.CdcrAction.STATUS.toLower()));
+      }
+
+      if (allEquals) {
+        break;
+      }
+      else {
+        if (cnt == 0) {
+          throw new RuntimeException("Timeout waiting for CDCR state to replicate: collection="+collection);
+        }
+        cnt--;
+        Thread.sleep(500);
+      }
+    }
+
+    log.info("CDCR state is identical across nodes - collection: " + collection);
+  }
+
+  /**
+   * Assert the state of CDCR on each nodes of the given collection.
+   */
+  protected void assertState(String collection, CdcrParams.ProcessState processState, CdcrParams.BufferState bufferState)
+  throws Exception {
+    this.waitForCdcrStateReplication(collection); // ensure that cdcr state is replicated and stable
+    for (CloudJettyRunner jetty : cloudJettys.get(collection)) { // check all replicas
+      NamedList rsp = invokeCdcrAction(jetty, CdcrParams.CdcrAction.STATUS);
+      NamedList status = (NamedList) rsp.get(CdcrParams.CdcrAction.STATUS.toLower());
+      assertEquals(processState.toLower(), status.get(CdcrParams.ProcessState.getParam()));
+      assertEquals(bufferState.toLower(), status.get(CdcrParams.BufferState.getParam()));
+    }
+  }
+
+  /**
+   * A mapping between collection and node names. This is used when creating the collection in
+   * {@link #createCollection(String)}.
+   */
+  private Map<String, List<String>> collectionToNodeNames = new HashMap<>();
+
+  /**
+   * Starts the servers, saves and associates the node names to the source collection,
+   * and finally creates the source collection.
+   */
+  private void createSourceCollection() throws Exception {
+    List<String> nodeNames = this.startServers(shardCount * replicationFactor);
+    this.collectionToNodeNames.put(SOURCE_COLLECTION, nodeNames);
+    this.createCollection(SOURCE_COLLECTION);
+    this.waitForRecoveriesToFinish(SOURCE_COLLECTION, true);
+    this.updateMappingsFromZk(SOURCE_COLLECTION);
+  }
+
+  /**
+   * Clear the source collection. It will delete then create the collection through the collection API.
+   * The collection will have a new fresh index, i.e., including a new update log.
+   */
+  protected void clearSourceCollection() throws Exception {
+    this.deleteCollection(SOURCE_COLLECTION);
+    this.waitForCollectionToDisappear(SOURCE_COLLECTION);
+    this.createCollection(SOURCE_COLLECTION);
+    this.waitForRecoveriesToFinish(SOURCE_COLLECTION, true);
+    this.updateMappingsFromZk(SOURCE_COLLECTION);
+  }
+
+  /**
+   * Starts the servers, saves and associates the node names to the target collection,
+   * and finally creates the target collection.
+   */
+  private void createTargetCollection() throws Exception {
+    List<String> nodeNames = this.startServers(shardCount * replicationFactor);
+    this.collectionToNodeNames.put(TARGET_COLLECTION, nodeNames);
+    this.createCollection(TARGET_COLLECTION);
+    this.waitForRecoveriesToFinish(TARGET_COLLECTION, true);
+    this.updateMappingsFromZk(TARGET_COLLECTION);
+  }
+
+  /**
+   * Clear the source collection. It will delete then create the collection through the collection API.
+   * The collection will have a new fresh index, i.e., including a new update log.
+   */
+  protected void clearTargetCollection() throws Exception {
+    this.deleteCollection(TARGET_COLLECTION);
+    this.waitForCollectionToDisappear(TARGET_COLLECTION);
+    this.createCollection(TARGET_COLLECTION);
+    this.waitForRecoveriesToFinish(TARGET_COLLECTION, true);
+    this.updateMappingsFromZk(TARGET_COLLECTION);
+  }
+
+  /**
+   * Create a new collection through the Collection API. It enforces the use of one max shard per node.
+   * It will define the nodes to spread the new collection across by using the mapping {@link #collectionToNodeNames},
+   * to ensure that a node will not host more than one core (which will create problem when trying to restart servers).
+   */
+  private void createCollection(String name) throws Exception {
+    CloudSolrClient client = createCloudClient(null);
+    try {
+      // Create the target collection
+      Map<String, List<Integer>> collectionInfos = new HashMap<>();
+      int maxShardsPerNode = 1;
+
+      StringBuilder sb = new StringBuilder();
+      for (String nodeName : collectionToNodeNames.get(name)) {
+        sb.append(nodeName);
+        sb.append(',');
+      }
+      sb.deleteCharAt(sb.length() - 1);
+
+      createCollection(collectionInfos, name, shardCount, replicationFactor, maxShardsPerNode, client, sb.toString());
+    } finally {
+      client.close();
+    }
+  }
+
+  private CollectionAdminResponse createCollection(Map<String, List<Integer>> collectionInfos,
+                                                   String collectionName, int numShards, int replicationFactor,
+                                                   int maxShardsPerNode, SolrClient client, String createNodeSetStr)
+      throws SolrServerException, IOException {
+    return createCollection(collectionInfos, collectionName,
+        Utils.makeMap(
+            NUM_SLICES, numShards,
+            REPLICATION_FACTOR, replicationFactor,
+            CREATE_NODE_SET, createNodeSetStr,
+            MAX_SHARDS_PER_NODE, maxShardsPerNode),
+        client, "conf1");
+  }
+
+  private CollectionAdminResponse createCollection(Map<String, List<Integer>> collectionInfos, String collectionName,
+                                                   Map<String, Object> collectionProps, SolrClient client,
+                                                   String confSetName)
+      throws SolrServerException, IOException {
+    ModifiableSolrParams params = new ModifiableSolrParams();
+    params.set("action", CollectionParams.CollectionAction.CREATE.toString());
+    for (Map.Entry<String, Object> entry : collectionProps.entrySet()) {
+      if (entry.getValue() != null) params.set(entry.getKey(), String.valueOf(entry.getValue()));
+    }
+    Integer numShards = (Integer) collectionProps.get(NUM_SLICES);
+    if (numShards == null) {
+      String shardNames = (String) collectionProps.get(SHARDS_PROP);
+      numShards = StrUtils.splitSmart(shardNames, ',').size();
+    }
+    Integer replicationFactor = (Integer) collectionProps.get(REPLICATION_FACTOR);
+    if (replicationFactor == null) {
+      replicationFactor = (Integer) OverseerCollectionMessageHandler.COLL_PROPS.get(REPLICATION_FACTOR);
+    }
+
+    if (confSetName != null) {
+      params.set("collection.configName", confSetName);
+    }
+
+    List<Integer> list = new ArrayList<>();
+    list.add(numShards);
+    list.add(replicationFactor);
+    if (collectionInfos != null) {
+      collectionInfos.put(collectionName, list);
+    }
+    params.set("name", collectionName);
+    SolrRequest request = new QueryRequest(params);
+    request.setPath("/admin/collections");
+
+    CollectionAdminResponse res = new CollectionAdminResponse();
+    res.setResponse(client.request(request));
+    return res;
+  }
+
+  /**
+   * Delete a collection through the Collection API.
+   */
+  protected CollectionAdminResponse deleteCollection(String collectionName) throws Exception {
+    SolrClient client = createCloudClient(null);
+    CollectionAdminResponse res;
+
+    try {
+      ModifiableSolrParams params = new ModifiableSolrParams();
+      params.set("action", CollectionParams.CollectionAction.DELETE.toString());
+      params.set("name", collectionName);
+      QueryRequest request = new QueryRequest(params);
+      request.setPath("/admin/collections");
+
+      res = new CollectionAdminResponse();
+      res.setResponse(client.request(request));
+    } catch (Exception e) {
+      log.warn("Error while deleting the collection " + collectionName, e);
+      return new CollectionAdminResponse();
+    } finally {
+      client.close();
+    }
+
+    return res;
+  }
+
+  private void waitForCollectionToDisappear(String collection) throws Exception {
+    CloudSolrClient client = this.createCloudClient(null);
+    try {
+      client.connect();
+      ZkStateReader zkStateReader = client.getZkStateReader();
+      AbstractDistribZkTestBase.waitForCollectionToDisappear(collection, zkStateReader, false, true, 15);
+    } finally {
+      client.close();
+    }
+  }
+
+  private void waitForRecoveriesToFinish(String collection, boolean verbose) throws Exception {
+    CloudSolrClient client = this.createCloudClient(null);
+    try {
+      client.connect();
+      ZkStateReader zkStateReader = client.getZkStateReader();
+      super.waitForRecoveriesToFinish(collection, zkStateReader, verbose);
+    } finally {
+      client.close();
+    }
+  }
+
+  /**
+   * Asserts that the collection has the correct number of shards and replicas
+   */
+  protected void assertCollectionExpectations(String collectionName) throws Exception {
+    CloudSolrClient client = this.createCloudClient(null);
+    try {
+      client.connect();
+      ClusterState clusterState = client.getZkStateReader().getClusterState();
+
+      assertTrue("Could not find new collection " + collectionName, clusterState.hasCollection(collectionName));
+      Map<String, Slice> shards = clusterState.getCollection(collectionName).getSlicesMap();
+      // did we find expectedSlices shards/shards?
+      assertEquals("Found new collection " + collectionName + ", but mismatch on number of shards.", shardCount, shards.size());
+      int totalShards = 0;
+      for (String shardName : shards.keySet()) {
+        totalShards += shards.get(shardName).getReplicas().size();
+      }
+      int expectedTotalShards = shardCount * replicationFactor;
+      assertEquals("Found new collection " + collectionName + " with correct number of shards, but mismatch on number " +
+          "of shards.", expectedTotalShards, totalShards);
+    } finally {
+      client.close();
+    }
+  }
+
+  /**
+   * Restart a server.
+   */
+  protected void restartServer(CloudJettyRunner server) throws Exception {
+    // it seems we need to set the collection property to have the jetty properly restarted
+    System.setProperty("collection", server.collection);
+    JettySolrRunner jetty = server.jetty;
+    ChaosMonkey.stop(jetty);
+    ChaosMonkey.start(jetty);
+    System.clearProperty("collection");
+    waitForRecoveriesToFinish(server.collection, true);
+    updateMappingsFromZk(server.collection); // must update the mapping as the core node name might have changed
+  }
+
+  /**
+   * Restarts a list of servers.
+   */
+  protected void restartServers(List<CloudJettyRunner> servers) throws Exception {
+    for (CloudJettyRunner server : servers) {
+      this.restartServer(server);
+    }
+  }
+
+  private List<JettySolrRunner> jettys = new ArrayList<>();
+
+  /**
+   * Creates and starts a given number of servers.
+   */
+  protected List<String> startServers(int nServer) throws Exception {
+    String temporaryCollection = "tmp_collection";
+
+    for (int i = 1; i <= nServer; i++) {
+      // give everyone there own solrhome
+      File jettyDir = createTempDir("jetty").toFile();
+      jettyDir.mkdirs();
+      setupJettySolrHome(jettyDir);
+      JettySolrRunner jetty = createJetty(jettyDir, null, "shard" + i);
+      jettys.add(jetty);
+    }
+
+    try (SolrClient client = createCloudClient(temporaryCollection)) {
+      assertEquals(0, CollectionAdminRequest
+          .createCollection(temporaryCollection, "conf1", shardCount, 1)
+          .setCreateNodeSet("")
+          .process(client).getStatus());
+      for (int i = 0; i < jettys.size(); i++) {
+        assertTrue(CollectionAdminRequest
+            .addReplicaToShard(temporaryCollection, "shard"+((i % shardCount) + 1))
+            .setNode(jettys.get(i).getNodeName())
+            .process(client).isSuccess());
+      }
+    }
+
+    ZkStateReader zkStateReader = jettys.get(0).getCoreContainer().getZkController().getZkStateReader();
+
+    // now wait till we see the leader for each shard
+    for (int i = 1; i <= shardCount; i++) {
+      this.printLayout();
+      zkStateReader.getLeaderRetry(temporaryCollection, "shard" + i, 15000);
+    }
+
+    // store the node names
+    List<String> nodeNames = new ArrayList<>();
+    for (Slice shard : zkStateReader.getClusterState().getCollection(temporaryCollection).getSlices()) {
+      for (Replica replica : shard.getReplicas()) {
+        nodeNames.add(replica.getNodeName());
+      }
+    }
+
+    this.waitForRecoveriesToFinish(temporaryCollection,zkStateReader, true);
+    // delete the temporary collection - we will create our own collections later
+    this.deleteCollection(temporaryCollection);
+    this.waitForCollectionToDisappear(temporaryCollection);
+    System.clearProperty("collection");
+
+    return nodeNames;
+  }
+
+  @Override
+  protected void destroyServers() throws Exception {
+    for (JettySolrRunner runner : jettys) {
+      try {
+        ChaosMonkey.stop(runner);
+      } catch (Exception e) {
+        log.error("", e);
+      }
+    }
+
+    jettys.clear();
+  }
+
+  /**
+   * Mapping from collection to jettys
+   */
+  protected Map<String, List<CloudJettyRunner>> cloudJettys = new HashMap<>();
+
+  /**
+   * Mapping from collection/shard to jettys
+   */
+  protected Map<String, Map<String, List<CloudJettyRunner>>> shardToJetty = new HashMap<>();
+
+  /**
+   * Mapping from collection/shard leader to jettys
+   */
+  protected Map<String, Map<String, CloudJettyRunner>> shardToLeaderJetty = new HashMap<>();
+
+  /**
+   * Updates the mappings between the jetty's instances and the zookeeper cluster state.
+   */
+  protected void updateMappingsFromZk(String collection) throws Exception {
+    List<CloudJettyRunner> cloudJettys = new ArrayList<>();
+    Map<String, List<CloudJettyRunner>> shardToJetty = new HashMap<>();
+    Map<String, CloudJettyRunner> shardToLeaderJetty = new HashMap<>();
+
+    CloudSolrClient cloudClient = this.createCloudClient(null);
+    try {
+      cloudClient.connect();
+      ZkStateReader zkStateReader = cloudClient.getZkStateReader();
+      ClusterState clusterState = zkStateReader.getClusterState();
+      DocCollection coll = clusterState.getCollection(collection);
+
+      for (JettySolrRunner jetty : jettys) {
+        int port = jetty.getLocalPort();
+        if (port == -1) {
+          throw new RuntimeException("Cannot find the port for jetty");
+        }
+
+        nextJetty:
+        for (Slice shard : coll.getSlices()) {
+          Set<Map.Entry<String, Replica>> entries = shard.getReplicasMap().entrySet();
+          for (Map.Entry<String, Replica> entry : entries) {
+            Replica replica = entry.getValue();
+            if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
+              if (!shardToJetty.containsKey(shard.getName())) {
+                shardToJetty.put(shard.getName(), new ArrayList<CloudJettyRunner>());
+              }
+              boolean isLeader = shard.getLeader() == replica;
+              CloudJettyRunner cjr = new CloudJettyRunner(jetty, replica, collection, shard.getName(), entry.getKey());
+              shardToJetty.get(shard.getName()).add(cjr);
+              if (isLeader) {
+                shardToLeaderJetty.put(shard.getName(), cjr);
+              }
+              cloudJettys.add(cjr);
+              break nextJetty;
+            }
+          }
+        }
+      }
+
+      List<CloudJettyRunner> oldRunners = this.cloudJettys.putIfAbsent(collection, cloudJettys);
+      if (oldRunners != null)  {
+        // must close resources for the old entries
+        for (CloudJettyRunner oldRunner : oldRunners) {
+          IOUtils.closeQuietly(oldRunner.client);
+        }
+      }
+
+      this.cloudJettys.put(collection, cloudJettys);
+      this.shardToJetty.put(collection, shardToJetty);
+      this.shardToLeaderJetty.put(collection, shardToLeaderJetty);
+    } finally {
+      cloudClient.close();
+    }
+  }
+
+  /**
+   * Wrapper around a {@link org.apache.solr.client.solrj.embedded.JettySolrRunner} to map the jetty
+   * instance to various information of the cloud cluster, such as the collection and shard
+   * that is served by the jetty instance, the node name, core node name, url, etc.
+   */
+  public static class CloudJettyRunner {
+
+    public JettySolrRunner jetty;
+    public String nodeName;
+    public String coreNodeName;
+    public String url;
+    public SolrClient client;
+    public Replica info;
+    public String shard;
+    public String collection;
+
+    public CloudJettyRunner(JettySolrRunner jetty, Replica replica,
+                            String collection, String shard, String coreNodeName) {
+      this.jetty = jetty;
+      this.info = replica;
+      this.collection = collection;
+      
+      Properties nodeProperties = jetty.getNodeProperties();
+
+      // we need to update the jetty's shard so that it registers itself to the right shard when restarted
+      this.shard = shard;
+      nodeProperties.setProperty(CoreDescriptor.CORE_SHARD, this.shard);
+
+      // we need to update the jetty's shard so that it registers itself under the right core name when restarted
+      this.coreNodeName = coreNodeName;
+      nodeProperties.setProperty(CoreDescriptor.CORE_NODE_NAME, this.coreNodeName);
+
+      this.nodeName = replica.getNodeName();
+
+      ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(info);
+      this.url = coreNodeProps.getCoreUrl();
+
+      // strip the trailing slash as this can cause issues when executing requests
+      this.client = createNewSolrServer(this.url.substring(0, this.url.length() - 1));
+    }
+
+    @Override
+    public int hashCode() {
+      final int prime = 31;
+      int result = 1;
+      result = prime * result + ((url == null) ? 0 : url.hashCode());
+      return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) return true;
+      if (obj == null) return false;
+      if (getClass() != obj.getClass()) return false;
+      CloudJettyRunner other = (CloudJettyRunner) obj;
+      if (url == null) {
+        if (other.url != null) return false;
+      } else if (!url.equals(other.url)) return false;
+      return true;
+    }
+
+    @Override
+    public String toString() {
+      return "CloudJettyRunner [url=" + url + "]";
+    }
+
+  }
+
+  protected static SolrClient createNewSolrServer(String baseUrl) {
+    try {
+      // setup the server...
+      HttpSolrClient s = getHttpSolrClient(baseUrl, DEFAULT_CONNECTION_TIMEOUT);
+      return s;
+    } catch (Exception ex) {
+      throw new RuntimeException(ex);
+    }
+  }
+
+  protected void waitForBootstrapToComplete(String collectionName, String shardId) throws Exception {
+    NamedList rsp;// we need to wait until bootstrap is complete otherwise the replicator thread will never start
+    TimeOut timeOut = new TimeOut(60, TimeUnit.SECONDS);
+    while (!timeOut.hasTimedOut())  {
+      rsp = invokeCdcrAction(shardToLeaderJetty.get(collectionName).get(shardId), CdcrParams.CdcrAction.BOOTSTRAP_STATUS);
+      if (rsp.get(RESPONSE_STATUS).toString().equals(COMPLETED))  {
+        break;
+      }
+      Thread.sleep(1000);
+    }
+  }
+
+  protected void waitForReplicationToComplete(String collectionName, String shardId) throws Exception {
+    int cnt = 15;
+    while (cnt > 0) {
+      log.info("Checking queue size @ {}:{}", collectionName, shardId);
+      long size = this.getQueueSize(collectionName, shardId);
+      if (size == 0) { // if we received -1, it means that the log reader is not yet initialised, we should wait
+        return;
+      }
+      log.info("Waiting for replication to complete. Queue size: {} @ {}:{}", size, collectionName, shardId);
+      cnt--;
+      Thread.sleep(1000); // wait a bit for the replication to complete
+    }
+    throw new RuntimeException("Timeout waiting for CDCR replication to complete @" + collectionName + ":"  + shardId);
+  }
+
+  protected long getQueueSize(String collectionName, String shardId) throws Exception {
+    NamedList rsp = this.invokeCdcrAction(shardToLeaderJetty.get(collectionName).get(shardId), CdcrParams.CdcrAction.QUEUES);
+    NamedList host = (NamedList) ((NamedList) rsp.get(CdcrParams.QUEUES)).getVal(0);
+    NamedList status = (NamedList) host.get(TARGET_COLLECTION);
+    return (Long) status.get(CdcrParams.QUEUE_SIZE);
+  }
+
+  protected CollectionInfo collectInfo(String collection) throws Exception {
+    CollectionInfo info = new CollectionInfo(collection);
+    for (String shard : shardToJetty.get(collection).keySet()) {
+      List<CloudJettyRunner> jettyRunners = shardToJetty.get(collection).get(shard);
+      for (CloudJettyRunner jettyRunner : jettyRunners) {
+        for (SolrCore core : jettyRunner.jetty.getCoreContainer().getCores()) {
+          info.addCore(core, shard, shardToLeaderJetty.get(collection).containsValue(jettyRunner));
+        }
+      }
+    }
+
+    return info;
+  }
+
+  protected static class CollectionInfo {
+
+    List<CoreInfo> coreInfos = new ArrayList<>();
+
+    String collection;
+
+    CollectionInfo(String collection) {
+      this.collection = collection;
+    }
+
+    /**
+     * @return Returns a map shard -> list of cores
+     */
+    Map<String, List<CoreInfo>> getShardToCoresMap() {
+      Map<String, List<CoreInfo>> map = new HashMap<>();
+      for (CoreInfo info : coreInfos) {
+        List<CoreInfo> list = map.get(info.shard);
+        if (list == null) {
+          list = new ArrayList<>();
+          map.put(info.shard, list);
+        }
+        list.add(info);
+      }
+      return map;
+    }
+
+    CoreInfo getLeader(String shard) {
+      List<CoreInfo> coreInfos = getShardToCoresMap().get(shard);
+      for (CoreInfo info : coreInfos) {
+        if (info.isLeader) {
+          return info;
+        }
+      }
+      assertTrue(String.format(Locale.ENGLISH, "There is no leader for collection %s shard %s", collection, shard), false);
+      return null;
+    }
+
+    List<CoreInfo> getReplicas(String shard) {
+      List<CoreInfo> coreInfos = getShardToCoresMap().get(shard);
+      coreInfos.remove(getLeader(shard));
+      return coreInfos;
+    }
+
+    void addCore(SolrCore core, String shard, boolean isLeader) throws Exception {
+      CoreInfo info = new CoreInfo();
+      info.collectionName = core.getName();
+      info.shard = shard;
+      info.isLeader = isLeader;
+      info.ulogDir = core.getUpdateHandler().getUpdateLog().getLogDir();
+
+      this.coreInfos.add(info);
+    }
+
+    public static class CoreInfo {
+      String collectionName;
+      String shard;
+      boolean isLeader;
+      String ulogDir;
+    }
+
+  }
+
+}
+

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/db18de7e/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBidirectionalTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBidirectionalTest.java b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBidirectionalTest.java
new file mode 100644
index 0000000..70cb5eb
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBidirectionalTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.cdcr;
+
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.TimeUnit;
+
+import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.collect.ImmutableMap;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.handler.CdcrParams;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CdcrBidirectionalTest extends SolrTestCaseJ4 {
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  @Test
+  public void testBiDir() throws Exception {
+    MiniSolrCloudCluster cluster2 = new MiniSolrCloudCluster(1, createTempDir("cdcr-cluster2"), buildJettyConfig("/solr"));
+    cluster2.waitForAllNodes(30);
+    MiniSolrCloudCluster cluster1 = new MiniSolrCloudCluster(1, createTempDir("cdcr-cluster1"), buildJettyConfig("/solr"));
+    cluster1.waitForAllNodes(30);
+    try {
+      log.info("cluster2 zkHost = " + cluster2.getZkServer().getZkAddress());
+      System.setProperty("cdcr.cluster2.zkHost", cluster2.getZkServer().getZkAddress());
+
+      log.info("cluster1 zkHost = " + cluster1.getZkServer().getZkAddress());
+      System.setProperty("cdcr.cluster1.zkHost", cluster1.getZkServer().getZkAddress());
+
+
+      cluster1.uploadConfigSet(configset("cdcr-cluster1"), "cdcr-cluster1");
+      CollectionAdminRequest.createCollection("cdcr-cluster1", "cdcr-cluster1", 2, 1)
+          .withProperty("solr.directoryFactory", "solr.StandardDirectoryFactory")
+          .setMaxShardsPerNode(2)
+          .process(cluster1.getSolrClient());
+      CloudSolrClient cluster1SolrClient = cluster1.getSolrClient();
+      cluster1SolrClient.setDefaultCollection("cdcr-cluster1");
+
+      cluster2.uploadConfigSet(configset("cdcr-cluster2"), "cdcr-cluster2");
+      CollectionAdminRequest.createCollection("cdcr-cluster2", "cdcr-cluster2", 2, 1)
+          .withProperty("solr.directoryFactory", "solr.StandardDirectoryFactory")
+          .setMaxShardsPerNode(2)
+          .process(cluster2.getSolrClient());
+      CloudSolrClient cluster2SolrClient = cluster2.getSolrClient();
+      cluster2SolrClient.setDefaultCollection("cdcr-cluster2");
+
+      UpdateRequest req = null;
+
+      CdcrTestsUtil.cdcrStart(cluster1SolrClient);
+      Thread.sleep(2000);
+
+      // ADD operation on cluster 1
+      int docs = (TEST_NIGHTLY ? 100 : 10);
+      int numDocs_c1 = 0;
+      for (int k = 0; k < docs; k++) {
+        req = new UpdateRequest();
+        for (; numDocs_c1 < (k + 1) * 100; numDocs_c1++) {
+          SolrInputDocument doc = new SolrInputDocument();
+          doc.addField("id", "cluster1_" + numDocs_c1);
+          doc.addField("xyz", numDocs_c1);
+          req.add(doc);
+        }
+        req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
+        log.info("Adding " + docs + " docs with commit=true, numDocs=" + numDocs_c1);
+        req.process(cluster1SolrClient);
+      }
+
+      QueryResponse response = cluster1SolrClient.query(new SolrQuery("*:*"));
+      assertEquals("cluster 1 docs mismatch", numDocs_c1, response.getResults().getNumFound());
+
+      assertEquals("cluster 2 docs mismatch", numDocs_c1, CdcrTestsUtil.waitForClusterToSync(numDocs_c1, cluster2SolrClient));
+
+      CdcrTestsUtil.cdcrStart(cluster2SolrClient); // FULL BI-DIRECTIONAL CDCR FORWARDING ON
+      Thread.sleep(2000);
+
+      // ADD operation on cluster 2
+      int numDocs_c2 = 0;
+      for (int k = 0; k < docs; k++) {
+        req = new UpdateRequest();
+        for (; numDocs_c2 < (k + 1) * 100; numDocs_c2++) {
+          SolrInputDocument doc = new SolrInputDocument();
+          doc.addField("id", "cluster2_" + numDocs_c2);
+          doc.addField("xyz", numDocs_c2);
+          req.add(doc);
+        }
+        req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
+        log.info("Adding " + docs + " docs with commit=true, numDocs=" + numDocs_c2);
+        req.process(cluster2SolrClient);
+      }
+
+      int numDocs = numDocs_c1 + numDocs_c2;
+
+      response = cluster2SolrClient.query(new SolrQuery("*:*"));
+      assertEquals("cluster 2 docs mismatch", numDocs, response.getResults().getNumFound());
+
+      assertEquals("cluster 1 docs mismatch", numDocs, CdcrTestsUtil.waitForClusterToSync(numDocs, cluster1SolrClient));
+
+      // logging cdcr clusters queue response
+      response = CdcrTestsUtil.getCdcrQueue(cluster1SolrClient);
+      log.info("Cdcr cluster1 queue response: " + response.getResponse());
+      response = CdcrTestsUtil.getCdcrQueue(cluster2SolrClient);
+      log.info("Cdcr cluster2 queue response: " + response.getResponse());
+
+      // lets find and keep the maximum version assigned by cluster1 & cluster2 across all our updates
+
+      long maxVersion_c1 = Math.min((long)CdcrTestsUtil.getFingerPrintMaxVersion(cluster1SolrClient, "shard1", numDocs),
+          (long)CdcrTestsUtil.getFingerPrintMaxVersion(cluster1SolrClient, "shard2", numDocs));
+      long maxVersion_c2 = Math.min((long)CdcrTestsUtil.getFingerPrintMaxVersion(cluster2SolrClient, "shard1", numDocs),
+          (long)CdcrTestsUtil.getFingerPrintMaxVersion(cluster2SolrClient, "shard2", numDocs));
+
+      ModifiableSolrParams params = new ModifiableSolrParams();
+      params.set(CommonParams.ACTION, CdcrParams.CdcrAction.COLLECTIONCHECKPOINT.toString());
+      params.set(CommonParams.QT, "/cdcr");
+      response = cluster2SolrClient.query(params);
+      Long checkpoint_2 = (Long) response.getResponse().get(CdcrParams.CHECKPOINT);
+      assertNotNull(checkpoint_2);
+
+      params = new ModifiableSolrParams();
+      params.set(CommonParams.ACTION, CdcrParams.CdcrAction.COLLECTIONCHECKPOINT.toString());
+      params.set(CommonParams.QT, "/cdcr");
+      response = cluster1SolrClient.query(params);
+      Long checkpoint_1 = (Long) response.getResponse().get(CdcrParams.CHECKPOINT);
+      assertNotNull(checkpoint_1);
+
+      log.info("v1: " + maxVersion_c1 + "\t" + "v2: " + maxVersion_c2 + "\t" +
+          "checkpoint1: " + checkpoint_1 + "\t" + "checkpoint2: " + checkpoint_2);
+
+      assertEquals("COLLECTIONCHECKPOINT from cluster2 should have returned the maximum " +
+          "version across all updates made to cluster1", maxVersion_c1, checkpoint_2.longValue());
+      assertEquals("COLLECTIONCHECKPOINT from cluster1 should have returned the maximum " +
+          "version across all updates made to cluster2", maxVersion_c2, checkpoint_1.longValue());
+      assertEquals("max versions of updates in both clusters should be same", maxVersion_c1, maxVersion_c2);
+
+      // DELETE BY QUERY
+      String deleteByQuery = "id:cluster1_" +String.valueOf(random().nextInt(numDocs_c1));
+      response = cluster1SolrClient.query(new SolrQuery(deleteByQuery));
+      assertEquals("should match exactly one doc", 1, response.getResults().getNumFound());
+      cluster1SolrClient.deleteByQuery(deleteByQuery);
+      cluster1SolrClient.commit();
+      numDocs--;
+      numDocs_c1--;
+
+      response = cluster1SolrClient.query(new SolrQuery("*:*"));
+      assertEquals("cluster 1 docs mismatch", numDocs, response.getResults().getNumFound());
+      assertEquals("cluster 2 docs mismatch", numDocs, CdcrTestsUtil.waitForClusterToSync(numDocs, cluster2SolrClient));
+
+      // DELETE BY ID
+      SolrInputDocument doc;
+      String delete_id_query = "cluster2_" + random().nextInt(numDocs_c2);
+      cluster2SolrClient.deleteById(delete_id_query);
+      cluster2SolrClient.commit();
+      numDocs--;
+      numDocs_c2--;
+      response = cluster2SolrClient.query(new SolrQuery("*:*"));
+      assertEquals("cluster 2 docs mismatch", numDocs, response.getResults().getNumFound());
+      assertEquals("cluster 1 docs mismatch", numDocs, CdcrTestsUtil.waitForClusterToSync(numDocs, cluster1SolrClient));
+
+      // ATOMIC UPDATES
+      req = new UpdateRequest();
+      doc = new SolrInputDocument();
+      ImmutableMap.of("", "");
+      String atomicUpdateId = "cluster2_" + random().nextInt(numDocs_c2);
+      doc.addField("id", atomicUpdateId);
+      doc.addField("xyz", ImmutableMap.of("delete", ""));
+      doc.addField("abc", ImmutableMap.of("set", "ABC"));
+      req.add(doc);
+      req.process(cluster2SolrClient);
+      cluster2SolrClient.commit();
+
+      String atomicQuery = "id:" + atomicUpdateId;
+      response = cluster2SolrClient.query(new SolrQuery(atomicQuery));
+      assertEquals("cluster 2 wrong doc", "ABC", response.getResults().get(0).get("abc"));
+      assertEquals("cluster 1 wrong doc", "ABC", getDocFieldValue(cluster1SolrClient, atomicQuery, "ABC"));
+
+
+      // logging cdcr clusters queue response
+      response = CdcrTestsUtil.getCdcrQueue(cluster1SolrClient);
+      log.info("Cdcr cluster1 queue response at end of testcase: " + response.getResponse());
+      response = CdcrTestsUtil.getCdcrQueue(cluster2SolrClient);
+      log.info("Cdcr cluster2 queue response at end of testcase: " + response.getResponse());
+
+      CdcrTestsUtil.cdcrStop(cluster1SolrClient);
+      CdcrTestsUtil.cdcrStop(cluster2SolrClient);
+    } finally {
+      if (cluster1 != null) {
+        cluster1.shutdown();
+      }
+      if (cluster2 != null) {
+        cluster2.shutdown();
+      }
+    }
+  }
+
+  private String getDocFieldValue(CloudSolrClient clusterSolrClient, String query, String match) throws Exception {
+    long start = System.nanoTime();
+    QueryResponse response = null;
+    while (System.nanoTime() - start <= TimeUnit.NANOSECONDS.convert(120, TimeUnit.SECONDS)) {
+      clusterSolrClient.commit();
+      response = clusterSolrClient.query(new SolrQuery(query));
+      if (match.equals(response.getResults().get(0).get("abc"))) {
+        break;
+      }
+      Thread.sleep(1000);
+    }
+    return response != null ? (String) response.getResults().get(0).get("abc") : "";
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/db18de7e/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
new file mode 100644
index 0000000..2cabb50
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.cdcr;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.LinkedHashMap;
+
+import org.apache.lucene.store.FSDirectory;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.handler.CdcrParams;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CdcrBootstrapTest extends SolrTestCaseJ4 {
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  /**
+   * Starts a source cluster with no CDCR configuration, indexes enough documents such that
+   * the at least one old tlog is closed and thrown away so that the source cluster does not have
+   * all updates available in tlogs only.
+   * <p>
+   * Then we start a target cluster with CDCR configuration and we change the source cluster configuration
+   * to use CDCR (i.e. CdcrUpdateLog, CdcrRequestHandler and CdcrUpdateProcessor) and restart it.
+   * <p>
+   * We test that all updates eventually make it to the target cluster and that the collectioncheckpoint
+   * call returns the same version as the last update indexed on the source.
+   */
+  @Test
+  public void testConvertClusterToCdcrAndBootstrap() throws Exception {
+    // start the target first so that we know its zkhost
+    MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr"));
+    try {
+      target.waitForAllNodes(30);
+      log.info("Target zkHost = " + target.getZkServer().getZkAddress());
+      System.setProperty("cdcr.target.zkHost", target.getZkServer().getZkAddress());
+
+      // start a cluster with no cdcr
+      MiniSolrCloudCluster source = new MiniSolrCloudCluster(1, createTempDir("cdcr-source"), buildJettyConfig("/solr"));
+      try {
+        source.waitForAllNodes(30);
+        source.uploadConfigSet(configset("cdcr-source-disabled"), "cdcr-source");
+
+        // create a collection with the cdcr-source-disabled configset
+        CollectionAdminRequest.createCollection("cdcr-source", "cdcr-source", 1, 1)
+            // todo investigate why this is necessary??? because by default it selects a ram directory which deletes the tlogs on reloads?
+            .withProperty("solr.directoryFactory", "solr.StandardDirectoryFactory")
+            .process(source.getSolrClient());
+
+        CloudSolrClient sourceSolrClient = source.getSolrClient();
+        int docs = (TEST_NIGHTLY ? 100 : 10);
+        int numDocs = indexDocs(sourceSolrClient, "cdcr-source", docs);
+
+        QueryResponse response = sourceSolrClient.query(new SolrQuery("*:*"));
+        assertEquals("", numDocs, response.getResults().getNumFound());
+
+        // lets find and keep the maximum version assigned by source cluster across all our updates
+        long maxVersion = Long.MIN_VALUE;
+        ModifiableSolrParams params = new ModifiableSolrParams();
+        params.set(CommonParams.QT, "/get");
+        params.set("getVersions", numDocs);
+        params.set("fingerprint", true);
+        response = sourceSolrClient.query(params);
+        maxVersion = (long)(((LinkedHashMap)response.getResponse().get("fingerprint")).get("maxVersionEncountered"));
+
+//       upload the cdcr-enabled config and restart source cluster
+        source.uploadConfigSet(configset("cdcr-source"), "cdcr-source");
+        JettySolrRunner runner = source.stopJettySolrRunner(0);
+        source.startJettySolrRunner(runner);
+        assertTrue(runner.isRunning());
+        AbstractDistribZkTestBase.waitForRecoveriesToFinish("cdcr-source", source.getSolrClient().getZkStateReader(), true, true, 330);
+
+        response = sourceSolrClient.query(new SolrQuery("*:*"));
+        assertEquals("Document mismatch on source after restart", numDocs, response.getResults().getNumFound());
+
+        // setup the target cluster
+        target.uploadConfigSet(configset("cdcr-target"), "cdcr-target");
+        CollectionAdminRequest.createCollection("cdcr-target", "cdcr-target", 1, 1)
+            .process(target.getSolrClient());
+        CloudSolrClient targetSolrClient = target.getSolrClient();
+        targetSolrClient.setDefaultCollection("cdcr-target");
+        Thread.sleep(1000);
+
+        CdcrTestsUtil.cdcrStart(targetSolrClient);
+        CdcrTestsUtil.cdcrStart(sourceSolrClient);
+
+        response = CdcrTestsUtil.getCdcrQueue(sourceSolrClient);
+        log.info("Cdcr queue response: " + response.getResponse());
+        long foundDocs = CdcrTestsUtil.waitForClusterToSync(numDocs, targetSolrClient);
+        assertEquals("Document mismatch on target after sync", numDocs, foundDocs);
+
+        params = new ModifiableSolrParams();
+        params.set(CommonParams.ACTION, CdcrParams.CdcrAction.COLLECTIONCHECKPOINT.toString());
+        params.set(CommonParams.QT, "/cdcr");
+        response = targetSolrClient.query(params);
+        Long checkpoint = (Long) response.getResponse().get(CdcrParams.CHECKPOINT);
+        assertNotNull(checkpoint);
+        assertEquals("COLLECTIONCHECKPOINT from target cluster should have returned the maximum " +
+            "version across all updates made to source", maxVersion, checkpoint.longValue());
+      } finally {
+        source.shutdown();
+      }
+    } finally {
+      target.shutdown();
+    }
+  }
+
+  private int indexDocs(CloudSolrClient sourceSolrClient, String collection, int batches) throws IOException, SolrServerException {
+    sourceSolrClient.setDefaultCollection(collection);
+    int numDocs = 0;
+    for (int k = 0; k < batches; k++) {
+      UpdateRequest req = new UpdateRequest();
+      for (; numDocs < (k + 1) * 100; numDocs++) {
+        SolrInputDocument doc = new SolrInputDocument();
+        doc.addField("id", "source_" + numDocs);
+        doc.addField("xyz", numDocs);
+        req.add(doc);
+      }
+      req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
+      req.process(sourceSolrClient);
+    }
+    log.info("Adding numDocs=" + numDocs);
+    return numDocs;
+  }
+  /**
+   * This test start cdcr source, adds data,starts target cluster, verifies replication,
+   * stops cdcr replication and buffering, adds more data, re-enables cdcr and verify replication
+   */
+  public void testBootstrapWithSourceCluster() throws Exception {
+    // start the target first so that we know its zkhost
+    MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr"));
+    try {
+      target.waitForAllNodes(30);
+      System.out.println("Target zkHost = " + target.getZkServer().getZkAddress());
+      System.setProperty("cdcr.target.zkHost", target.getZkServer().getZkAddress());
+
+      MiniSolrCloudCluster source = new MiniSolrCloudCluster(1, createTempDir("cdcr-source"), buildJettyConfig("/solr"));
+      try {
+        source.waitForAllNodes(30);
+        source.uploadConfigSet(configset("cdcr-source"), "cdcr-source");
+
+        CollectionAdminRequest.createCollection("cdcr-source", "cdcr-source", 1, 1)
+            .withProperty("solr.directoryFactory", "solr.StandardDirectoryFactory")
+            .process(source.getSolrClient());
+
+        CloudSolrClient sourceSolrClient = source.getSolrClient();
+        int docs = (TEST_NIGHTLY ? 100 : 10);
+        int numDocs = indexDocs(sourceSolrClient, "cdcr-source", docs);
+
+        QueryResponse response = sourceSolrClient.query(new SolrQuery("*:*"));
+        assertEquals("", numDocs, response.getResults().getNumFound());
+
+        // setup the target cluster
+        target.uploadConfigSet(configset("cdcr-target"), "cdcr-target");
+        CollectionAdminRequest.createCollection("cdcr-target", "cdcr-target", 1, 1)
+            .process(target.getSolrClient());
+        CloudSolrClient targetSolrClient = target.getSolrClient();
+        targetSolrClient.setDefaultCollection("cdcr-target");
+
+        CdcrTestsUtil.cdcrStart(targetSolrClient);
+        CdcrTestsUtil.cdcrStart(sourceSolrClient);
+
+        response = CdcrTestsUtil.getCdcrQueue(sourceSolrClient);
+        log.info("Cdcr queue response: " + response.getResponse());
+        long foundDocs = CdcrTestsUtil.waitForClusterToSync(numDocs, targetSolrClient);
+        assertEquals("Document mismatch on target after sync", numDocs, foundDocs);
+
+        int total_tlogs_in_index = FSDirectory.open(target.getBaseDir().resolve("node1").
+            resolve("cdcr-target_shard1_replica_n1").resolve("data").
+            resolve("tlog")).listAll().length;
+
+        assertEquals("tlogs count should be ZERO",0, total_tlogs_in_index);
+
+        CdcrTestsUtil.cdcrStop(sourceSolrClient);
+        CdcrTestsUtil.cdcrDisableBuffer(sourceSolrClient);
+
+        int c = 0;
+        for (int k = 0; k < 10; k++) {
+          UpdateRequest req = new UpdateRequest();
+          for (; c < (k + 1) * 100; c++, numDocs++) {
+            SolrInputDocument doc = new SolrInputDocument();
+            doc.addField("id", "source_" + numDocs);
+            doc.addField("xyz", numDocs);
+            req.add(doc);
+          }
+          req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
+          log.info("Adding 100 docs with commit=true, numDocs=" + numDocs);
+          req.process(sourceSolrClient);
+        }
+
+        response = sourceSolrClient.query(new SolrQuery("*:*"));
+        assertEquals("", numDocs, response.getResults().getNumFound());
+
+        CdcrTestsUtil.cdcrStart(sourceSolrClient);
+        CdcrTestsUtil.cdcrEnableBuffer(sourceSolrClient);
+
+        foundDocs = CdcrTestsUtil.waitForClusterToSync(numDocs, targetSolrClient);
+        assertEquals("Document mismatch on target after sync", numDocs, foundDocs);
+
+      } finally {
+        source.shutdown();
+      }
+    } finally {
+      target.shutdown();
+    }
+  }
+
+  public void testBootstrapWithContinousIndexingOnSourceCluster() throws Exception {
+    // start the target first so that we know its zkhost
+    MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr"));
+    target.waitForAllNodes(30);
+    try {
+      log.info("Target zkHost = " + target.getZkServer().getZkAddress());
+      System.setProperty("cdcr.target.zkHost", target.getZkServer().getZkAddress());
+
+      MiniSolrCloudCluster source = new MiniSolrCloudCluster(1, createTempDir("cdcr-source"), buildJettyConfig("/solr"));
+      try {
+        source.waitForAllNodes(30);
+        source.uploadConfigSet(configset("cdcr-source"), "cdcr-source");
+
+        CollectionAdminRequest.createCollection("cdcr-source", "cdcr-source", 1, 1)
+            .withProperty("solr.directoryFactory", "solr.StandardDirectoryFactory")
+            .process(source.getSolrClient());
+
+        CloudSolrClient sourceSolrClient = source.getSolrClient();
+        int docs = (TEST_NIGHTLY ? 100 : 10);
+        int numDocs = indexDocs(sourceSolrClient, "cdcr-source", docs);
+
+        QueryResponse response = sourceSolrClient.query(new SolrQuery("*:*"));
+        assertEquals("", numDocs, response.getResults().getNumFound());
+
+        // setup the target cluster
+        target.uploadConfigSet(configset("cdcr-target"), "cdcr-target");
+        CollectionAdminRequest.createCollection("cdcr-target", "cdcr-target", 1, 1)
+            .process(target.getSolrClient());
+        CloudSolrClient targetSolrClient = target.getSolrClient();
+        targetSolrClient.setDefaultCollection("cdcr-target");
+        Thread.sleep(1000);
+
+        CdcrTestsUtil.cdcrStart(targetSolrClient);
+        CdcrTestsUtil.cdcrStart(sourceSolrClient);
+        int c = 0;
+        for (int k = 0; k < docs; k++) {
+          UpdateRequest req = new UpdateRequest();
+          for (; c < (k + 1) * 100; c++, numDocs++) {
+            SolrInputDocument doc = new SolrInputDocument();
+            doc.addField("id", "source_" + numDocs);
+            doc.addField("xyz", numDocs);
+            req.add(doc);
+          }
+          req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
+          log.info("Adding " + docs + " docs with commit=true, numDocs=" + numDocs);
+          req.process(sourceSolrClient);
+        }
+
+        response = sourceSolrClient.query(new SolrQuery("*:*"));
+        assertEquals("", numDocs, response.getResults().getNumFound());
+
+        response = CdcrTestsUtil.getCdcrQueue(sourceSolrClient);
+        log.info("Cdcr queue response: " + response.getResponse());
+        long foundDocs = CdcrTestsUtil.waitForClusterToSync(numDocs, targetSolrClient);
+        assertEquals("Document mismatch on target after sync", numDocs, foundDocs);
+
+      } finally {
+        source.shutdown();
+      }
+    } finally {
+      target.shutdown();
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/db18de7e/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrReplicationDistributedZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrReplicationDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrReplicationDistributedZkTest.java
new file mode 100644
index 0000000..206a08f
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrReplicationDistributedZkTest.java
@@ -0,0 +1,672 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud.cdcr;
+
+import java.io.File;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.util.LuceneTestCase.BadApple;
+import org.apache.lucene.util.LuceneTestCase.Nightly;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.CdcrParams;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Nightly
+@BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-10107")
+public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest {
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  @Override
+  public void distribSetUp() throws Exception {
+    schemaString = "schema15.xml";      // we need a string id
+    super.distribSetUp();
+  }
+
+  /**
+   * Checks that the test framework handles properly the creation and deletion of collections and the
+   * restart of servers.
+   */
+  @Test
+  @ShardsFixed(num = 4)
+  public void testDeleteCreateSourceCollection() throws Exception {
+    log.info("Indexing documents");
+
+    List<SolrInputDocument> docs = new ArrayList<>();
+    for (int i = 0; i < 10; i++) {
+      docs.add(getDoc(id, Integer.toString(i)));
+    }
+    index(SOURCE_COLLECTION, docs);
+    index(TARGET_COLLECTION, docs);
+
+    assertNumDocs(10, SOURCE_COLLECTION);
+    assertNumDocs(10, TARGET_COLLECTION);
+
+    log.info("Restarting leader @ source_collection:shard1");
+
+    this.restartServer(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1));
+
+    assertNumDocs(10, SOURCE_COLLECTION);
+    assertNumDocs(10, TARGET_COLLECTION);
+
+    log.info("Clearing source_collection");
+
+    this.clearSourceCollection();
+
+    assertNumDocs(0, SOURCE_COLLECTION);
+    assertNumDocs(10, TARGET_COLLECTION);
+
+    log.info("Restarting leader @ target_collection:shard1");
+
+    this.restartServer(shardToLeaderJetty.get(TARGET_COLLECTION).get(SHARD1));
+
+    assertNumDocs(0, SOURCE_COLLECTION);
+    assertNumDocs(10, TARGET_COLLECTION);
+
+    log.info("Clearing target_collection");
+
+    this.clearTargetCollection();
+
+    assertNumDocs(0, SOURCE_COLLECTION);
+    assertNumDocs(0, TARGET_COLLECTION);
+
+    assertCollectionExpectations(SOURCE_COLLECTION);
+    assertCollectionExpectations(TARGET_COLLECTION);
+  }
+
+  @Test
+  @ShardsFixed(num = 4)
+  public void testTargetCollectionNotAvailable() throws Exception {
+    // send start action to first shard
+    NamedList rsp = invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    NamedList status = (NamedList) rsp.get(CdcrParams.CdcrAction.STATUS.toLower());
+    assertEquals(CdcrParams.ProcessState.STARTED.toLower(), status.get(CdcrParams.ProcessState.getParam()));
+
+    // check status
+    this.assertState(SOURCE_COLLECTION, CdcrParams.ProcessState.STARTED, CdcrParams.BufferState.ENABLED);
+
+    this.waitForBootstrapToComplete(TARGET_COLLECTION, SHARD2);
+
+    // sleep for a bit to ensure that replicator threads are started
+    Thread.sleep(3000);
+
+    // Kill all the servers of the target
+    this.deleteCollection(TARGET_COLLECTION);
+
+    // Index a few documents to trigger the replication
+    index(SOURCE_COLLECTION, getDoc(id, "a"));
+    index(SOURCE_COLLECTION, getDoc(id, "b"));
+    index(SOURCE_COLLECTION, getDoc(id, "c"));
+    index(SOURCE_COLLECTION, getDoc(id, "d"));
+    index(SOURCE_COLLECTION, getDoc(id, "e"));
+    index(SOURCE_COLLECTION, getDoc(id, "f"));
+
+    assertNumDocs(6, SOURCE_COLLECTION);
+
+    // we need to wait until the replicator thread is triggered
+    int cnt = 15; // timeout after 15 seconds
+    AssertionError lastAssertionError = null;
+    while (cnt > 0) {
+      try {
+        rsp = invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD2), CdcrParams.CdcrAction.ERRORS);
+        NamedList collections = (NamedList) ((NamedList) rsp.get(CdcrParams.ERRORS)).getVal(0);
+        NamedList errors = (NamedList) collections.get(TARGET_COLLECTION);
+        assertTrue(0 < (Long) errors.get(CdcrParams.CONSECUTIVE_ERRORS));
+        NamedList lastErrors = (NamedList) errors.get(CdcrParams.LAST);
+        assertNotNull(lastErrors);
+        assertTrue(0 < lastErrors.size());
+        return;
+      }
+      catch (AssertionError e) {
+        lastAssertionError = e;
+        cnt--;
+        Thread.sleep(1000);
+      }
+    }
+
+    throw new AssertionError("Timeout while trying to assert replication errors", lastAssertionError);
+  }
+
+  @Test
+  @ShardsFixed(num = 4)
+  public void testReplicationStartStop() throws Exception {
+    int start = 0;
+    List<SolrInputDocument> docs = new ArrayList<>();
+    for (; start < 10; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    assertNumDocs(10, SOURCE_COLLECTION);
+    assertNumDocs(0, TARGET_COLLECTION);
+
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2);
+
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    commit(TARGET_COLLECTION);
+
+    assertNumDocs(10, SOURCE_COLLECTION);
+    assertNumDocs(10, TARGET_COLLECTION);
+
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.STOP);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    docs.clear();
+    for (; start < 110; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    assertNumDocs(110, SOURCE_COLLECTION);
+    assertNumDocs(10, TARGET_COLLECTION);
+
+    // Start again CDCR, the source cluster should reinitialise its log readers
+    // with the latest checkpoints
+
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2);
+
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    commit(TARGET_COLLECTION);
+
+    assertNumDocs(110, SOURCE_COLLECTION);
+    assertNumDocs(110, TARGET_COLLECTION);
+  }
+
+  /**
+   * Check that the replication manager is properly restarted after a node failure.
+   */
+  @Test
+  @ShardsFixed(num = 4)
+  public void testReplicationAfterRestart() throws Exception {
+    log.info("Starting CDCR");
+
+    // send start action to first shard
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2);
+
+    log.info("Indexing 10 documents");
+
+    int start = 0;
+    List<SolrInputDocument> docs = new ArrayList<>();
+    for (; start < 10; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    log.info("Querying source collection");
+
+    assertNumDocs(10, SOURCE_COLLECTION);
+
+    log.info("Waiting for replication");
+
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    log.info("Querying target collection");
+
+    commit(TARGET_COLLECTION);
+    assertNumDocs(10, TARGET_COLLECTION);
+
+    log.info("Restarting shard1");
+
+    this.restartServers(shardToJetty.get(SOURCE_COLLECTION).get(SHARD1));
+
+    log.info("Indexing 100 documents");
+
+    docs.clear();
+    for (; start < 110; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    log.info("Querying source collection");
+
+    assertNumDocs(110, SOURCE_COLLECTION);
+
+    log.info("Waiting for replication");
+
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2);
+
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    log.info("Querying target collection");
+
+    commit(TARGET_COLLECTION);
+    assertNumDocs(110, TARGET_COLLECTION);
+  }
+
+  /**
+   * Check that the replication manager is properly started after a change of leader.
+   * This test also checks that the log readers on the new leaders are initialised with
+   * the target's checkpoint.
+   */
+  @Test
+  @ShardsFixed(num = 4)
+  public void testReplicationAfterLeaderChange() throws Exception {
+    log.info("Starting CDCR");
+
+    // send start action to first shard
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2);
+
+    log.info("Indexing 10 documents");
+
+    int start = 0;
+    List<SolrInputDocument> docs = new ArrayList<>();
+    for (; start < 10; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    log.info("Querying source collection");
+
+    assertNumDocs(10, SOURCE_COLLECTION);
+
+    log.info("Waiting for replication");
+
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    log.info("Querying target collection");
+
+    commit(TARGET_COLLECTION);
+    assertNumDocs(10, TARGET_COLLECTION);
+
+    log.info("Restarting target leaders");
+
+    // Close all the leaders, then restart them
+    this.restartServer(shardToLeaderJetty.get(TARGET_COLLECTION).get(SHARD1));
+    this.restartServer(shardToLeaderJetty.get(TARGET_COLLECTION).get(SHARD2));
+
+    log.info("Restarting source leaders");
+
+    // Close all the leaders, then restart them
+    this.restartServer(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1));
+    this.restartServer(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD2));
+
+    log.info("Checking queue size of new source leaders");
+
+    // If the log readers of the new leaders are initialised with the target's checkpoint, the
+    // queue size must be inferior to the current number of documents indexed.
+    // The queue might be not completely empty since the new target checkpoint is probably not the
+    // last document received
+    assertTrue(this.getQueueSize(SOURCE_COLLECTION, SHARD1) < 10);
+    assertTrue(this.getQueueSize(SOURCE_COLLECTION, SHARD2) < 10);
+
+    log.info("Indexing 100 documents");
+
+    docs.clear();
+    for (; start < 110; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    log.info("Querying source collection");
+
+    assertNumDocs(110, SOURCE_COLLECTION);
+
+    log.info("Waiting for replication");
+
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    log.info("Querying target collection");
+
+    commit(TARGET_COLLECTION);
+    assertNumDocs(110, TARGET_COLLECTION);
+  }
+
+  /**
+   * Check that the update logs are synchronised between leader and non-leader nodes
+   * when CDCR is on and buffer is disabled
+   */
+  @Test
+  @ShardsFixed(num = 4)
+  public void testUpdateLogSynchronisation() throws Exception {
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    for (int i = 0; i < 100; i++) {
+      // will perform a commit for every document and will create one tlog file per commit
+      index(SOURCE_COLLECTION, getDoc(id, Integer.toString(i)));
+    }
+
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2);
+
+    // wait a bit for the replication to complete
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    commit(TARGET_COLLECTION);
+
+    // Check that the replication was done properly
+    assertNumDocs(100, SOURCE_COLLECTION);
+    assertNumDocs(100, TARGET_COLLECTION);
+
+    // Get the number of tlog files on the replicas (should be equal to the number of documents indexed)
+    int nTlogs = getNumberOfTlogFilesOnReplicas(SOURCE_COLLECTION);
+
+    // Disable the buffer - ulog synch should start on non-leader nodes
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.DISABLEBUFFER);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    int cnt = 15; // timeout after 15 seconds
+    while (cnt > 0) {
+      // Index a new document with a commit to trigger update log cleaning
+      index(SOURCE_COLLECTION, getDoc(id, Integer.toString(50)));
+
+      // Check the update logs on non-leader nodes, the number of tlog files should decrease
+      int n = getNumberOfTlogFilesOnReplicas(SOURCE_COLLECTION);
+      if (n < nTlogs) return;
+
+      cnt--;
+      Thread.sleep(1000);
+    }
+
+    throw new AssertionError("Timeout while trying to assert update logs @ source_collection");
+  }
+
+  /**
+   * Check that the buffer is always activated on non-leader nodes.
+   */
+  @Test
+  @ShardsFixed(num = 4)
+  public void testBufferOnNonLeader() throws Exception {
+    // buffering is enabled by default, so disable it
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.DISABLEBUFFER);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    // Start CDCR
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    // Index documents
+    for (int i = 0; i < 200; i++) {
+      index(SOURCE_COLLECTION, getDoc(id, Integer.toString(i))); // will perform a commit for every document
+    }
+
+    // And immediately, close all the leaders, then restart them. It is likely that the replication will not be
+    // performed fully, and therefore be continued by the new leader
+    // At this stage, the new leader must have been elected
+    this.restartServer(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1));
+    this.restartServer(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD2));
+
+    // wait a bit for the replication to complete
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    // Commit to make the documents visible on the target
+    commit(TARGET_COLLECTION);
+
+    // If the non-leader node were buffering updates, then the replication must be complete
+    assertNumDocs(200, SOURCE_COLLECTION);
+    assertNumDocs(200, TARGET_COLLECTION);
+  }
+
+  /**
+   * Check the ops statistics.
+   */
+  @Test
+  @ShardsFixed(num = 4)
+  public void testOps() throws Exception {
+    // Index documents
+    List<SolrInputDocument> docs = new ArrayList<>();
+    for (int i = 0; i < 200; i++) {
+      docs.add(getDoc(id, Integer.toString(i)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    // Start CDCR
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    // wait a bit for the replication to complete
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    NamedList rsp = this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.OPS);
+    NamedList collections = (NamedList) ((NamedList) rsp.get(CdcrParams.OPERATIONS_PER_SECOND)).getVal(0);
+    NamedList ops = (NamedList) collections.get(TARGET_COLLECTION);
+    double opsAll = (Double) ops.get(CdcrParams.COUNTER_ALL);
+    double opsAdds = (Double) ops.get(CdcrParams.COUNTER_ADDS);
+    assertTrue(opsAll > 0);
+    assertEquals(opsAll, opsAdds, 0);
+
+    double opsDeletes = (Double) ops.get(CdcrParams.COUNTER_DELETES);
+    assertEquals(0, opsDeletes, 0);
+  }
+
+  /**
+   * Check that batch updates with deletes
+   */
+  @Test
+  @ShardsFixed(num = 4)
+  public void testBatchAddsWithDelete() throws Exception {
+    // Index 50 documents
+    int start = 0;
+    List<SolrInputDocument> docs = new ArrayList<>();
+    for (; start < 50; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    // Delete 10 documents: 10-19
+    List<String> ids = new ArrayList<>();
+    for (int id = 10; id < 20; id++) {
+      ids.add(Integer.toString(id));
+    }
+    deleteById(SOURCE_COLLECTION, ids);
+
+    // Index 10 documents
+    docs = new ArrayList<>();
+    for (; start < 60; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    // Delete 1 document: 50
+    ids = new ArrayList<>();
+    ids.add(Integer.toString(50));
+    deleteById(SOURCE_COLLECTION, ids);
+
+    // Index 10 documents
+    docs = new ArrayList<>();
+    for (; start < 70; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    // Start CDCR
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+    this.waitForBootstrapToComplete(TARGET_COLLECTION, SHARD1);
+    this.waitForBootstrapToComplete(TARGET_COLLECTION, SHARD2);
+
+    // wait a bit for the replication to complete
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    commit(TARGET_COLLECTION);
+
+    // If the non-leader node were buffering updates, then the replication must be complete
+    assertNumDocs(59, SOURCE_COLLECTION);
+    assertNumDocs(59, TARGET_COLLECTION);
+  }
+
+  /**
+   * Checks that batches are correctly constructed when batch boundaries are reached.
+   */
+  @Test
+  @ShardsFixed(num = 4)
+  public void testBatchBoundaries() throws Exception {
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    log.info("Indexing documents");
+
+    List<SolrInputDocument> docs = new ArrayList<>();
+    for (int i = 0; i < 128; i++) { // should create two full batches (default batch = 64)
+      docs.add(getDoc(id, Integer.toString(i)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    assertNumDocs(128, SOURCE_COLLECTION);
+
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2);
+
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    commit(TARGET_COLLECTION);
+
+    assertNumDocs(128, SOURCE_COLLECTION);
+    assertNumDocs(128, TARGET_COLLECTION);
+  }
+
+  /**
+   * Check resilience of replication with delete by query executed on targets
+   */
+  @Test
+  @ShardsFixed(num = 4)
+  public void testResilienceWithDeleteByQueryOnTarget() throws Exception {
+    // Index 50 documents
+    int start = 0;
+    List<SolrInputDocument> docs = new ArrayList<>();
+    for (; start < 50; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    // Start CDCR
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2);
+
+    // wait a bit for the replication to complete
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    commit(TARGET_COLLECTION);
+
+    // If the non-leader node were buffering updates, then the replication must be complete
+    assertNumDocs(50, SOURCE_COLLECTION);
+    assertNumDocs(50, TARGET_COLLECTION);
+
+    deleteByQuery(SOURCE_COLLECTION, "*:*");
+    deleteByQuery(TARGET_COLLECTION, "*:*");
+
+    assertNumDocs(0, SOURCE_COLLECTION);
+    assertNumDocs(0, TARGET_COLLECTION);
+
+    docs.clear();
+    for (; start < 100; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    // wait a bit for the replication to complete
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    commit(TARGET_COLLECTION);
+
+    assertNumDocs(50, SOURCE_COLLECTION);
+    assertNumDocs(50, TARGET_COLLECTION);
+
+    deleteByQuery(TARGET_COLLECTION, "*:*");
+
+    assertNumDocs(50, SOURCE_COLLECTION);
+    assertNumDocs(0, TARGET_COLLECTION);
+
+    // Restart CDCR
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.STOP);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+    this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START);
+    this.waitForCdcrStateReplication(SOURCE_COLLECTION);
+
+    docs.clear();
+    for (; start < 150; start++) {
+      docs.add(getDoc(id, Integer.toString(start)));
+    }
+    index(SOURCE_COLLECTION, docs);
+
+    // wait a bit for the replication to complete
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1);
+    this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2);
+
+    commit(TARGET_COLLECTION);
+
+    assertNumDocs(100, SOURCE_COLLECTION);
+    assertNumDocs(50, TARGET_COLLECTION);
+  }
+
+  private int numberOfFiles(String dir) {
+    File file = new File(dir);
+    if (!file.isDirectory()) {
+      assertTrue("Path to tlog " + dir + " does not exists or it's not a directory.", false);
+    }
+    log.debug("Update log dir {} contains: {}", dir, file.listFiles());
+    return file.listFiles().length;
+  }
+
+  private int getNumberOfTlogFilesOnReplicas(String collection) throws Exception {
+    CollectionInfo info = collectInfo(collection);
+    Map<String, List<CollectionInfo.CoreInfo>> shardToCoresMap = info.getShardToCoresMap();
+
+    int count = 0;
+
+    for (String shard : shardToCoresMap.keySet()) {
+      for (int i = 0; i < replicationFactor - 1; i++) {
+        count += numberOfFiles(info.getReplicas(shard).get(i).ulogDir);
+      }
+    }
+
+    return count;
+  }
+
+}
+