You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2017/05/10 22:09:59 UTC

[5/6] lucene-solr:jira/solr-10233: Removed OnlyLeaderIndexesTest.java (test is TestAppendReplica now) and back to using UpdateHandler stats in TestAppend and TestPassive

Removed OnlyLeaderIndexesTest.java (test is TestAppendReplica now) and back to using UpdateHandler stats in TestAppend and TestPassive


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1c53272e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1c53272e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1c53272e

Branch: refs/heads/jira/solr-10233
Commit: 1c53272e947801bdc1f54ab9ef89fe1e98edf432
Parents: 6271229
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Wed May 10 13:17:02 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Wed May 10 13:17:02 2017 -0700

----------------------------------------------------------------------
 .../solr/cloud/OnlyLeaderIndexesTest.java       | 488 -------------------
 .../apache/solr/cloud/TestAppendReplica.java    |  30 +-
 .../apache/solr/cloud/TestPassiveReplica.java   |   7 +-
 3 files changed, 6 insertions(+), 519 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1c53272e/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java b/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java
deleted file mode 100644
index 6297408..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/OnlyLeaderIndexesTest.java
+++ /dev/null
@@ -1,488 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.concurrent.Semaphore;
-
-import org.apache.lucene.index.IndexWriter;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.CollectionStatePredicate;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.update.DirectUpdateHandler2;
-import org.apache.solr.update.SolrIndexWriter;
-import org.apache.solr.update.UpdateHandler;
-import org.apache.solr.update.UpdateLog;
-import org.apache.solr.util.RefCounted;
-import org.apache.zookeeper.KeeperException;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class OnlyLeaderIndexesTest extends SolrCloudTestCase {
-  private static final String COLLECTION = "collection1";
-
-  @BeforeClass
-  public static void setupCluster() throws Exception {
-    System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
-    System.setProperty("solr.ulog.numRecordsToKeep", "1000");
-
-    configureCluster(3)
-        .addConfig("config", TEST_PATH().resolve("configsets")
-        .resolve("cloud-minimal-inplace-updates").resolve("conf"))
-        .configure();
-    
-    CollectionAdminRequest.ClusterProp clusterPropRequest = CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, "false");
-    CollectionAdminResponse response = clusterPropRequest.process(cluster.getSolrClient());
-    assertEquals(0, response.getStatus());
-
-    CollectionAdminRequest
-        .createCollection(COLLECTION, "config", 1, 0, 3, 0)
-        .setMaxShardsPerNode(1)
-        .process(cluster.getSolrClient());
-    AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
-        false, true, 30);
-  }
-
-  @Test
-  public void test() throws Exception {
-    assertNumberOfReplicas(0, 3, 0, false, true);
-    basicTest();
-    recoveryTest();
-    dbiTest();
-    basicLeaderElectionTest();
-    outOfOrderDBQWithInPlaceUpdatesTest();
-  }
-
-  public void basicTest() throws Exception {
-    CloudSolrClient cloudClient = cluster.getSolrClient();
-    new UpdateRequest()
-        .add(sdoc("id", "1"))
-        .add(sdoc("id", "2"))
-        .add(sdoc("id", "3"))
-        .add(sdoc("id", "4"))
-        .process(cloudClient, COLLECTION);
-
-    {
-      UpdateHandler updateHandler = getSolrCore(true).get(0).getUpdateHandler();
-      RefCounted<IndexWriter> iwRef = updateHandler.getSolrCoreState().getIndexWriter(null);
-      assertTrue("IndexWriter at leader must see updates ", iwRef.get().hasUncommittedChanges());
-      iwRef.decref();
-    }
-
-    for (SolrCore solrCore : getSolrCore(false)) {
-      RefCounted<IndexWriter> iwRef = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
-      assertFalse("IndexWriter at replicas must not see updates ", iwRef.get().hasUncommittedChanges());
-      iwRef.decref();
-    }
-
-    checkRTG(1, 4, cluster.getJettySolrRunners());
-
-    new UpdateRequest()
-        .deleteById("1")
-        .deleteByQuery("id:2")
-        .process(cloudClient, COLLECTION);
-
-    // The DBQ is not processed at replicas, so we still can get doc2 and other docs by RTG
-    checkRTG(2,4, getSolrRunner(false));
-
-    new UpdateRequest()
-        .commit(cloudClient, COLLECTION);
-
-    checkShardConsistency(2, 1);
-
-    // Update log roll over
-    for (SolrCore solrCore : getSolrCore(false)) {
-      UpdateLog updateLog = solrCore.getUpdateHandler().getUpdateLog();
-      assertFalse(updateLog.hasUncommittedChanges());
-    }
-
-    // UpdateLog copy over old updates
-    for (int i = 15; i <= 150; i++) {
-      cloudClient.add(COLLECTION, sdoc("id",String.valueOf(i)));
-      if (random().nextInt(100) < 15 & i != 150) {
-        cloudClient.commit(COLLECTION);
-      }
-    }
-    checkRTG(120,150, cluster.getJettySolrRunners());
-    waitForReplicasCatchUp(20);
-  }
-
-  public void recoveryTest() throws Exception {
-    CloudSolrClient cloudClient = cluster.getSolrClient();
-    new UpdateRequest()
-        .deleteByQuery("*:*")
-        .commit(cluster.getSolrClient(), COLLECTION);
-    new UpdateRequest()
-        .add(sdoc("id", "3"))
-        .add(sdoc("id", "4"))
-        .commit(cloudClient, COLLECTION);
-    // Replica recovery
-    new UpdateRequest()
-        .add(sdoc("id", "5"))
-        .process(cloudClient, COLLECTION);
-    JettySolrRunner solrRunner = getSolrRunner(false).get(0);
-    ChaosMonkey.stop(solrRunner);
-    new UpdateRequest()
-        .add(sdoc("id", "6"))
-        .process(cloudClient, COLLECTION);
-    ChaosMonkey.start(solrRunner);
-    AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
-        false, true, 30);
-    // We skip peerSync, so replica will always trigger commit on leader
-    checkShardConsistency(4, 20);
-
-    // LTR can be kicked off, so waiting for replicas recovery
-    new UpdateRequest()
-        .add(sdoc("id", "7"))
-        .commit(cloudClient, COLLECTION);
-    AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
-        false, true, 30);
-    checkShardConsistency(5, 20);
-
-    // More Replica recovery testing
-    new UpdateRequest()
-        .add(sdoc("id", "8"))
-        .process(cloudClient, COLLECTION);
-    checkRTG(3,8, cluster.getJettySolrRunners());
-    DirectUpdateHandler2.commitOnClose = false;
-    ChaosMonkey.stop(solrRunner);
-    DirectUpdateHandler2.commitOnClose = true;
-    ChaosMonkey.start(solrRunner);
-    AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
-        false, true, 30);
-    checkRTG(3,8, cluster.getJettySolrRunners());
-    checkShardConsistency(6, 20);
-
-    // Test replica recovery apply buffer updates
-    Semaphore waitingForBufferUpdates = new Semaphore(0);
-    Semaphore waitingForReplay = new Semaphore(0);
-    RecoveryStrategy.testing_beforeReplayBufferingUpdates = () -> {
-      try {
-        waitingForReplay.release();
-        waitingForBufferUpdates.acquire();
-      } catch (InterruptedException e) {
-        e.printStackTrace();
-      }
-    };
-    ChaosMonkey.stop(solrRunner);
-    ChaosMonkey.start(solrRunner);
-    waitingForReplay.acquire();
-    new UpdateRequest()
-        .add(sdoc("id", "9"))
-        .add(sdoc("id", "10"))
-        .process(cloudClient, COLLECTION);
-    waitingForBufferUpdates.release();
-    RecoveryStrategy.testing_beforeReplayBufferingUpdates = null;
-    AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
-        false, true, 30);
-    checkRTG(3,10, cluster.getJettySolrRunners());
-    checkShardConsistency(6, 20);
-    for (SolrCore solrCore : getSolrCore(false)) {
-      RefCounted<IndexWriter> iwRef = solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);
-      assertFalse("IndexWriter at replicas must not see updates ", iwRef.get().hasUncommittedChanges());
-      iwRef.decref();
-    }
-  }
-
-  public void dbiTest() throws Exception{
-    CloudSolrClient cloudClient = cluster.getSolrClient();
-    new UpdateRequest()
-        .deleteByQuery("*:*")
-        .commit(cluster.getSolrClient(), COLLECTION);
-    new UpdateRequest()
-        .add(sdoc("id", "1"))
-        .commit(cloudClient, COLLECTION);
-    checkShardConsistency(1, 1);
-    new UpdateRequest()
-        .deleteById("1")
-        .process(cloudClient, COLLECTION);
-    try {
-      checkRTG(1, 1, cluster.getJettySolrRunners());
-    } catch (AssertionError e) {
-      return;
-    }
-    fail("Doc1 is deleted but it's still exist");
-  }
-
-  public void basicLeaderElectionTest() throws Exception {
-    CloudSolrClient cloudClient = cluster.getSolrClient();
-    new UpdateRequest()
-        .deleteByQuery("*:*")
-        .commit(cluster.getSolrClient(), COLLECTION);
-    new UpdateRequest()
-        .add(sdoc("id", "1"))
-        .add(sdoc("id", "2"))
-        .process(cloudClient, COLLECTION);
-    String oldLeader = getLeader();
-    JettySolrRunner oldLeaderJetty = getSolrRunner(true).get(0);
-    ChaosMonkey.kill(oldLeaderJetty);
-    for (int i = 0; i < 60; i++) { // wait till leader is changed
-      if (!oldLeader.equals(getLeader())) {
-        break;
-      }
-      Thread.sleep(100);
-    }
-    new UpdateRequest()
-        .add(sdoc("id", "3"))
-        .add(sdoc("id", "4"))
-        .process(cloudClient, COLLECTION);
-    ChaosMonkey.start(oldLeaderJetty);
-    waitForState("Replica not removed", "collection1", activeReplicaCount(0, 3, 0));
-    AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
-        false, true, 60);
-    checkRTG(1,4, cluster.getJettySolrRunners());
-    new UpdateRequest()
-        .commit(cloudClient, COLLECTION);
-    checkShardConsistency(4,1);
-  }
-
-  private String getLeader() throws InterruptedException {
-    ZkNodeProps props = cluster.getSolrClient().getZkStateReader().getLeaderRetry("collection1", "shard1", 30000);
-    return props.getStr(ZkStateReader.NODE_NAME_PROP);
-  }
-
-  public void outOfOrderDBQWithInPlaceUpdatesTest() throws Exception {
-    new UpdateRequest()
-        .deleteByQuery("*:*")
-        .commit(cluster.getSolrClient(), COLLECTION);
-    List<UpdateRequest> updates = new ArrayList<>();
-    updates.add(simulatedUpdateRequest(null, "id", 1, "title_s", "title0_new", "inplace_updatable_int", 5, "_version_", Long.MAX_VALUE-100)); // full update
-    updates.add(simulatedDBQ("inplace_updatable_int:5", Long.MAX_VALUE-98));
-    updates.add(simulatedUpdateRequest(Long.MAX_VALUE-100, "id", 1, "inplace_updatable_int", 6, "_version_", Long.MAX_VALUE-99));
-    for (JettySolrRunner solrRunner: getSolrRunner(false)) {
-      try (SolrClient client = solrRunner.newClient()) {
-        for (UpdateRequest up : updates) {
-          up.process(client, COLLECTION);
-        }
-      }
-    }
-    JettySolrRunner oldLeaderJetty = getSolrRunner(true).get(0);
-    ChaosMonkey.kill(oldLeaderJetty);
-    AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
-        false, true, 30);
-    ChaosMonkey.start(oldLeaderJetty);
-    AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
-        false, true, 30);
-    new UpdateRequest()
-        .add(sdoc("id", "2"))
-        .commit(cluster.getSolrClient(), COLLECTION);
-    checkShardConsistency(2,20);
-    SolrDocument doc = cluster.getSolrClient().getById(COLLECTION,"1");
-    assertNotNull(doc.get("title_s"));
-  }
-
-  private UpdateRequest simulatedUpdateRequest(Long prevVersion, Object... fields) throws SolrServerException, IOException {
-    SolrInputDocument doc = sdoc(fields);
-
-    // get baseUrl of the leader
-    String baseUrl = getBaseUrl();
-
-    UpdateRequest ur = new UpdateRequest();
-    ur.add(doc);
-    ur.setParam("update.distrib", "FROMLEADER");
-    if (prevVersion != null) {
-      ur.setParam("distrib.inplace.prevversion", String.valueOf(prevVersion));
-      ur.setParam("distrib.inplace.update", "true");
-    }
-    ur.setParam("distrib.from", baseUrl);
-    return ur;
-  }
-
-  private UpdateRequest simulatedDBQ(String query, long version) throws SolrServerException, IOException {
-    String baseUrl = getBaseUrl();
-
-    UpdateRequest ur = new UpdateRequest();
-    ur.deleteByQuery(query);
-    ur.setParam("_version_", ""+version);
-    ur.setParam("update.distrib", "FROMLEADER");
-    ur.setParam("distrib.from", baseUrl);
-    return ur;
-  }
-
-  private String getBaseUrl() {
-    DocCollection collection = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION);
-    Slice slice = collection.getSlice("shard1");
-    return slice.getLeader().getCoreUrl();
-  }
-
-  private void checkRTG(int from, int to, List<JettySolrRunner> solrRunners) throws Exception{
-
-    for (JettySolrRunner solrRunner: solrRunners) {
-      try (SolrClient client = solrRunner.newClient()) {
-        for (int i = from; i <= to; i++) {
-          SolrQuery query = new SolrQuery("*:*");
-          query.set("distrib", false);
-          query.setRequestHandler("/get");
-          query.set("id",i);
-          QueryResponse res = client.query(COLLECTION, query);
-          assertNotNull("Can not find doc "+ i + " in " + solrRunner.getBaseUrl(),res.getResponse().get("doc"));
-        }
-      }
-    }
-
-  }
-
-  private void checkShardConsistency(int expected, int numTry) throws Exception{
-    String replicaNotInSync = null;
-    for (int i = 0; i < numTry; i++) {
-      boolean inSync = true;
-      for (JettySolrRunner solrRunner: cluster.getJettySolrRunners()) {
-        try (SolrClient client = solrRunner.newClient()) {
-          SolrQuery query = new SolrQuery("*:*");
-          query.set("distrib", false);
-          long results = client.query(COLLECTION, query).getResults().getNumFound();
-          if (expected != results) {
-            inSync = false;
-            replicaNotInSync = solrRunner.getNodeName();
-            break;
-          }
-        }
-      }
-      if (inSync) return;
-      Thread.sleep(500);
-    }
-
-    fail("Some replicas are not in sync with leader: " + replicaNotInSync);
-  }
-
-  private void waitForReplicasCatchUp(int numTry) throws IOException, InterruptedException {
-    String leaderTimeCommit = getSolrCore(true).get(0).getDeletionPolicy().getLatestCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
-    if (leaderTimeCommit == null) return;
-    for (int i = 0; i < numTry; i++) {
-      boolean inSync = true;
-      for (SolrCore solrCore : getSolrCore(false)) {
-        String replicateTimeCommit = solrCore.getDeletionPolicy().getLatestCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
-        if (!leaderTimeCommit.equals(replicateTimeCommit)) {
-          inSync = false;
-          Thread.sleep(500);
-          break;
-        }
-      }
-      if (inSync) return;
-    }
-
-    fail("Some replicas are not in sync with leader");
-
-  }
-
-  private List<SolrCore> getSolrCore(boolean isLeader) {
-    List<SolrCore> rs = new ArrayList<>();
-
-    CloudSolrClient cloudClient = cluster.getSolrClient();
-    DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(COLLECTION);
-
-    for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
-      if (solrRunner.getCoreContainer() == null) continue;
-      for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
-        CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor();
-        Slice slice = docCollection.getSlice(cloudDescriptor.getShardId());
-        Replica replica = docCollection.getReplica(cloudDescriptor.getCoreNodeName());
-        if (slice.getLeader() == replica && isLeader) {
-          rs.add(solrCore);
-        } else if (slice.getLeader() != replica && !isLeader) {
-          rs.add(solrCore);
-        }
-      }
-    }
-    return rs;
-  }
-
-  private List<JettySolrRunner> getSolrRunner(boolean isLeader) {
-    List<JettySolrRunner> rs = new ArrayList<>();
-
-    CloudSolrClient cloudClient = cluster.getSolrClient();
-    DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(COLLECTION);
-
-    for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
-      if (solrRunner.getCoreContainer() == null) continue;
-      for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
-        CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor();
-        Slice slice = docCollection.getSlice(cloudDescriptor.getShardId());
-        Replica replica = docCollection.getReplica(cloudDescriptor.getCoreNodeName());
-        if (slice.getLeader() == replica && isLeader) {
-          rs.add(solrRunner);
-        } else if (slice.getLeader() != replica && !isLeader) {
-          rs.add(solrRunner);
-        }
-      }
-    }
-    return rs;
-  }
-  
-  // TODO: This is copy/paste from TestPassiveReplica, refactor
-  private DocCollection assertNumberOfReplicas(int numWriter, int numActive, int numPassive, boolean updateCollection, boolean activeOnly) throws KeeperException, InterruptedException {
-    if (updateCollection) {
-      cluster.getSolrClient().getZkStateReader().forceUpdateCollection("collection1");
-    }
-    DocCollection docCollection = getCollectionState("collection1");
-    assertNotNull(docCollection);
-    assertEquals("Unexpected number of writer replicas: " + docCollection, numWriter, 
-        docCollection.getReplicas(EnumSet.of(Replica.Type.REALTIME)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
-    assertEquals("Unexpected number of passive replicas: " + docCollection, numPassive, 
-        docCollection.getReplicas(EnumSet.of(Replica.Type.PASSIVE)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
-    assertEquals("Unexpected number of active replicas: " + docCollection, numActive, 
-        docCollection.getReplicas(EnumSet.of(Replica.Type.APPEND)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
-    return docCollection;
-  }
-  
-  private CollectionStatePredicate activeReplicaCount(int numWriter, int numActive, int numPassive) {
-    return (liveNodes, collectionState) -> {
-      int writersFound = 0, activesFound = 0, passivesFound = 0;
-      if (collectionState == null)
-        return false;
-      for (Slice slice : collectionState) {
-        for (Replica replica : slice) {
-          if (replica.isActive(liveNodes))
-            switch (replica.getType()) {
-              case APPEND:
-                activesFound++;
-                break;
-              case PASSIVE:
-                passivesFound++;
-                break;
-              case REALTIME:
-                writersFound++;
-                break;
-              default:
-                throw new AssertionError("Unexpected replica type");
-            }
-        }
-      }
-      return numWriter == writersFound && numActive == activesFound && numPassive == passivesFound;
-    };
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1c53272e/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
index b56c5a7..23dde01 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestAppendReplica.java
@@ -24,6 +24,7 @@ import java.util.Collection;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -50,6 +51,7 @@ import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.NamedList;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.update.DirectUpdateHandler2;
 import org.apache.solr.update.SolrIndexWriter;
@@ -114,28 +116,6 @@ public class TestAppendReplica extends SolrCloudTestCase {
     super.tearDown();
   }
   
-  // Just to compare test time, nocommit
-  @Ignore
-  public void testCreateDelete2() throws Exception {
-    try {
-      CollectionAdminRequest.createCollection(collectionName, "conf", 1, 8, 0, 0).process(cluster.getSolrClient());
-      DocCollection docCollection = getCollectionState(collectionName);
-      assertNotNull(docCollection);
-//      assertEquals("Expecting 4 relpicas per shard",
-//          8, docCollection.getReplicas().size());
-//      assertEquals("Expecting 6 passive replicas, 3 per shard",
-//          6, docCollection.getReplicas(EnumSet.of(Replica.Type.PASSIVE)).size());
-//      assertEquals("Expecting 2 writer replicas, one per shard",
-//          2, docCollection.getReplicas(EnumSet.of(Replica.Type.WRITER)).size());
-//      for (Slice s:docCollection.getSlices()) {
-//        // read-only replicas can never become leaders
-//        assertFalse(s.getLeader().isReadOnly());
-//      }
-    } finally {
-      zkClient().printLayoutToStdOut();
-    }
-  }
-  
   /**
    * Asserts that Update logs exist for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#REALTIME}, but not
    * for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#PASSIVE}
@@ -186,6 +166,7 @@ public class TestAppendReplica extends SolrCloudTestCase {
     }
   }
   
+  @SuppressWarnings("unchecked")
   public void testAddDocs() throws Exception {
     int numAppendReplicas = 1 + random().nextInt(3);
     DocCollection docCollection = createAndWaitForCollection(1, 0, numAppendReplicas, 0);
@@ -212,9 +193,8 @@ public class TestAppendReplica extends SolrCloudTestCase {
                 "qt", "/admin/plugins",
                 "stats", "true");
             QueryResponse statsResponse = appendReplicaClient.query(req);
-//            TODO: uncomment when SOLR-10569 is fixed
-//            assertEquals("Append replicas should recive all updates. Replica: " + r + ", response: " + statsResponse, 
-//                1L, ((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats", "cumulative_adds"));
+            assertEquals("Append replicas should recive all updates. Replica: " + r + ", response: " + statsResponse, 
+                1L, ((Map<String, Object>)((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get("UPDATE.updateHandler.cumulativeAdds.count"));
             break;
           } catch (AssertionError e) {
             if (t.hasTimedOut()) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1c53272e/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
index e1f597b..70f2627 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPassiveReplica.java
@@ -63,9 +63,6 @@ public class TestPassiveReplica extends SolrCloudTestCase {
   
   private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
   
-  // TODO: Backup/Snapshot should not work on passive replicas 
-  // TODO: ADDSHARD operation
-  
   private String collectionName = null;
   private final static int REPLICATION_TIMEOUT_SECS = 10;
   
@@ -231,10 +228,8 @@ public class TestPassiveReplica extends SolrCloudTestCase {
             "qt", "/admin/plugins",
             "stats", "true");
         QueryResponse statsResponse = readOnlyReplicaClient.query(req);
-//        assertEquals("Replicas shouldn't process the add document request: " + statsResponse, 
-//            0L, ((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats", "adds"));
         assertEquals("Replicas shouldn't process the add document request: " + statsResponse, 
-            0L, ((Map<String, Object>)((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "/update", "stats")).get("UPDATE./update.requests"));
+            0L, ((Map<String, Object>)((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get("UPDATE.updateHandler.adds"));
       }
     }
     assertUlogPresence(docCollection);