You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ab...@apache.org on 2023/10/16 12:08:25 UTC

[solr-sandbox] branch main updated: Add support for mirroring plain commit requests. (#74)

This is an automated email from the ASF dual-hosted git repository.

ab pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-sandbox.git


The following commit(s) were added to refs/heads/main by this push:
     new 66d2a82  Add support for mirroring plain commit requests. (#74)
66d2a82 is described below

commit 66d2a8226e6e75c998cea1a22a37497ee8936c29
Author: Andrzej BiaƂecki <ab...@apache.org>
AuthorDate: Mon Oct 16 14:08:18 2023 +0200

    Add support for mirroring plain commit requests. (#74)
    
    * Add a config knob to turn on commit mirroring (default is off).
---
 .../solr/crossdc/common/KafkaCrossDcConf.java      |   4 +
 .../update/processor/MirroringUpdateProcessor.java |  78 ++++++++++++++-
 .../MirroringUpdateRequestProcessorFactory.java    |   3 +-
 .../apache/solr/crossdc/DeleteByQueryToIdTest.java |   2 +-
 .../processor/MirroringUpdateProcessorTest.java    | 111 ++++++++++++++++++++-
 5 files changed, 191 insertions(+), 7 deletions(-)

diff --git a/crossdc-commons/src/main/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java b/crossdc-commons/src/main/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java
index d9f4fcd..3147643 100644
--- a/crossdc-commons/src/main/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java
+++ b/crossdc-commons/src/main/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java
@@ -61,6 +61,8 @@ public class KafkaCrossDcConf extends CrossDcConf {
 
   private static final String DEFAULT_MIRROR_COLLECTIONS = "";
 
+  private static final String DEFAULT_MIRROR_COMMITS = "false";
+
   public static final String TOPIC_NAME = "topicName";
 
   public static final String BOOTSTRAP_SERVERS = "bootstrapServers";
@@ -115,6 +117,7 @@ public class KafkaCrossDcConf extends CrossDcConf {
 
   public static final String MIRROR_COLLECTIONS = "mirror.collections";
 
+  public static final String MIRROR_COMMITS = "mirrorCommits";
 
   public static final List<ConfigProperty> CONFIG_PROPERTIES;
   private static final Map<String, ConfigProperty> CONFIG_PROPERTIES_MAP;
@@ -150,6 +153,7 @@ public class KafkaCrossDcConf extends CrossDcConf {
             new ConfigProperty(SESSION_TIMEOUT_MS, DEFAULT_SESSION_TIMEOUT_MS),
 
             new ConfigProperty(MIRROR_COLLECTIONS, DEFAULT_MIRROR_COLLECTIONS),
+            new ConfigProperty(MIRROR_COMMITS, DEFAULT_MIRROR_COMMITS),
 
             new ConfigProperty(MAX_PARTITION_FETCH_BYTES, DEFAULT_MAX_PARTITION_FETCH_BYTES),
             new ConfigProperty(MAX_POLL_RECORDS, DEFAULT_MAX_POLL_RECORDS),
diff --git a/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateProcessor.java b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateProcessor.java
index dc2e561..0fcdbd5 100644
--- a/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateProcessor.java
+++ b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateProcessor.java
@@ -29,6 +29,7 @@ import java.util.Collection;
 import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 
 import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
 
@@ -56,6 +57,11 @@ public class MirroringUpdateProcessor extends UpdateRequestProcessor {
    */
   private final boolean indexUnmirrorableDocs;
 
+  /**
+   * If true then commit commands are mirrored, otherwise they are processed only locally.
+   */
+  private final boolean mirrorCommits;
+
   private final long maxMirroringDocSizeBytes;
 
 
@@ -71,6 +77,7 @@ public class MirroringUpdateProcessor extends UpdateRequestProcessor {
 
   public MirroringUpdateProcessor(final UpdateRequestProcessor next, boolean doMirroring,
       final boolean indexUnmirrorableDocs,
+      final boolean mirrorCommits,
       final long maxMirroringBatchSizeBytes,
       final SolrParams mirroredReqParams,
       final DistributedUpdateProcessor.DistribPhase distribPhase,
@@ -78,6 +85,7 @@ public class MirroringUpdateProcessor extends UpdateRequestProcessor {
     super(next);
     this.doMirroring = doMirroring;
     this.indexUnmirrorableDocs = indexUnmirrorableDocs;
+    this.mirrorCommits = mirrorCommits;
     this.maxMirroringDocSizeBytes = maxMirroringBatchSizeBytes;
     this.mirrorParams = mirroredReqParams;
     this.distribPhase = distribPhase;
@@ -89,8 +97,8 @@ public class MirroringUpdateProcessor extends UpdateRequestProcessor {
 
   UpdateRequest createMirrorRequest() {
     UpdateRequest mirrorRequest = new UpdateRequest();
-      mirrorRequest.setParams(new ModifiableSolrParams(mirrorParams));
-      return mirrorRequest;
+    mirrorRequest.setParams(new ModifiableSolrParams(mirrorParams));
+    return mirrorRequest;
   }
 
   @Override public void processAdd(final AddUpdateCommand cmd) throws IOException {
@@ -268,11 +276,77 @@ public class MirroringUpdateProcessor extends UpdateRequestProcessor {
     // TODO: We can't/shouldn't support this ?
   }
 
+  private boolean shouldMirrorCommit(SolrQueryRequest req) {
+    CloudDescriptor cd = req.getCore().getCoreDescriptor().getCloudDescriptor();
+    if (cd != null) {
+      String shardId = cd.getShardId();
+      ClusterState clusterState =
+          req.getCore().getCoreContainer().getZkController().getClusterState();
+      DocCollection coll = clusterState.getCollection(cd.getCollectionName());
+      String firstShard = new TreeMap<>(coll.getSlicesMap()).keySet().iterator().next();
+      if (!shardId.equals(firstShard)) {
+        return false;
+      }
+      Replica leaderReplica;
+      try {
+        leaderReplica = req.getCore().getCoreContainer().getZkController().getZkStateReader().getLeaderRetry(cd.getCollectionName(), shardId);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+      }
+      return leaderReplica.getName().equals(cd.getCoreNodeName());
+    } else {
+      return false;
+    }
+  }
+
   public void processCommit(CommitUpdateCommand cmd) throws IOException {
     log.debug("process commit cmd={}", cmd);
     if (next != null) next.processCommit(cmd);
+    if (!mirrorCommits) {
+      return;
+    }
+    UpdateRequest req = createMirrorRequest();
+    // mirror only once from the first shard leader
+    boolean shouldMirror = shouldMirrorCommit(cmd.getReq());
+    if (doMirroring && shouldMirror) {
+      req.setParam(UpdateParams.COMMIT, "true");
+      // transfer other params
+      if (cmd.optimize) {
+        req.setParam(UpdateParams.OPTIMIZE, "true");
+      }
+      if (cmd.softCommit) {
+        req.setParam(UpdateParams.SOFT_COMMIT, "true");
+      }
+      if (cmd.prepareCommit) {
+        req.setParam(UpdateParams.PREPARE_COMMIT, "true");
+      }
+      if (cmd.waitSearcher) {
+        req.setParam(UpdateParams.WAIT_SEARCHER, "true");
+      }
+      if (cmd.openSearcher) {
+        req.setParam(UpdateParams.OPEN_SEARCHER, "true");
+      }
+      if (cmd.expungeDeletes) {
+        req.setParam(UpdateParams.EXPUNGE_DELETES, "true");
+      }
+      if (cmd.maxOptimizeSegments != 0) {
+        req.setParam(UpdateParams.MAX_OPTIMIZE_SEGMENTS, Integer.toString(cmd.maxOptimizeSegments));
+      }
+      log.debug(" --doMirroring commit req={}", req);
+      try {
+        requestMirroringHandler.mirror(req);
+      } catch (Exception e) {
+        log.error("mirror submit failed", e);
+        throw new SolrException(SERVER_ERROR, "mirror submit failed", e);
+      }
+
+    } else {
+      log.debug(" -- skip commit mirroring, doMirroring={}, shouldMirror={}", doMirroring, shouldMirror);
+    }
   }
 
+
   @Override public final void finish() throws IOException {
     super.finish();
   }
diff --git a/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
index d14f49e..0077339 100644
--- a/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
+++ b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
@@ -203,6 +203,7 @@ public class MirroringUpdateRequestProcessorFactory extends UpdateRequestProcess
 
         // Check if mirroring is disabled in request params, defaults to true
         boolean doMirroring = req.getParams().getBool(SERVER_SHOULD_MIRROR, true);
+        boolean mirrorCommits = conf.getBool(MIRROR_COMMITS);
         final long maxMirroringBatchSizeBytes = conf.getInt(MAX_REQUEST_SIZE_BYTES);
         Boolean indexUnmirrorableDocs = conf.getBool(INDEX_UNMIRRORABLE_DOCS);
 
@@ -235,7 +236,7 @@ public class MirroringUpdateRequestProcessorFactory extends UpdateRequestProcess
             log.trace("Create MirroringUpdateProcessor with mirroredParams={}", mirroredParams);
         }
 
-        return new MirroringUpdateProcessor(next, doMirroring, indexUnmirrorableDocs, maxMirroringBatchSizeBytes, mirroredParams,
+        return new MirroringUpdateProcessor(next, doMirroring, indexUnmirrorableDocs, mirrorCommits, maxMirroringBatchSizeBytes, mirroredParams,
                 DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM)), doMirroring ? mirroringHandler : null);
     }
 
diff --git a/crossdc-producer/src/test/java/org/apache/solr/crossdc/DeleteByQueryToIdTest.java b/crossdc-producer/src/test/java/org/apache/solr/crossdc/DeleteByQueryToIdTest.java
index d57b547..140686d 100644
--- a/crossdc-producer/src/test/java/org/apache/solr/crossdc/DeleteByQueryToIdTest.java
+++ b/crossdc-producer/src/test/java/org/apache/solr/crossdc/DeleteByQueryToIdTest.java
@@ -187,7 +187,7 @@ import java.util.Properties;
       if (results.getResults().getNumFound() == 1) {
         foundUpdates = true;
       } else {
-        Thread.sleep(500);
+        Thread.sleep(1000);
       }
     }
 
diff --git a/crossdc-producer/src/test/java/org/apache/solr/update/processor/MirroringUpdateProcessorTest.java b/crossdc-producer/src/test/java/org/apache/solr/update/processor/MirroringUpdateProcessorTest.java
index bd23393..78abb00 100644
--- a/crossdc-producer/src/test/java/org/apache/solr/update/processor/MirroringUpdateProcessorTest.java
+++ b/crossdc-producer/src/test/java/org/apache/solr/update/processor/MirroringUpdateProcessorTest.java
@@ -11,18 +11,22 @@ import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.*;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.SolrQueryRequestBase;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.CommitUpdateCommand;
 import org.apache.solr.update.DeleteUpdateCommand;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import java.io.IOException;
+import java.util.Map;
 
 public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {
 
@@ -31,6 +35,7 @@ public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {
     private RequestMirroringHandler requestMirroringHandler;
     private AddUpdateCommand addUpdateCommand;
     private DeleteUpdateCommand deleteUpdateCommand;
+    private CommitUpdateCommand commitUpdateCommand;
     private SolrQueryRequestBase req;
     UpdateRequest requestMock;
     private UpdateRequestProcessor nextProcessor;
@@ -38,6 +43,8 @@ public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {
     private HttpSolrClient.Builder builder = Mockito.mock(HttpSolrClient.Builder.class);
     private HttpSolrClient client = Mockito.mock(HttpSolrClient.class);
     private CloudDescriptor cloudDesc;
+    private ZkStateReader zkStateReader;
+    private Replica replica;
 
     @Before
     public void setUp() throws Exception {
@@ -59,6 +66,15 @@ public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {
         deleteUpdateCommand = new DeleteUpdateCommand(req);
         deleteUpdateCommand.query = "*:*";
 
+        // set all flags to verify they are passed on
+        commitUpdateCommand = new CommitUpdateCommand(req, true);
+        commitUpdateCommand.prepareCommit = true;
+        commitUpdateCommand.maxOptimizeSegments = 10;
+        commitUpdateCommand.softCommit = true;
+        commitUpdateCommand.expungeDeletes = true;
+        commitUpdateCommand.openSearcher = true;
+        commitUpdateCommand.waitSearcher = true;
+
         next = Mockito.mock(UpdateRequestProcessor.class);
         requestMirroringHandler = Mockito.mock(RequestMirroringHandler.class);
         processor =
@@ -66,6 +82,7 @@ public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {
                         next,
                         true,
                         true,
+                        true,
                         1000L,
                         new ModifiableSolrParams(),
                         DistributedUpdateProcessor.DistribPhase.NONE,
@@ -78,14 +95,16 @@ public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {
         core = Mockito.mock(SolrCore.class);
         CoreDescriptor coreDesc = Mockito.mock(CoreDescriptor.class);
         cloudDesc = Mockito.mock(CloudDescriptor.class);
+        Mockito.when(cloudDesc.getShardId()).thenReturn("shard1");
         CoreContainer coreContainer = Mockito.mock(CoreContainer.class);
         ZkController zkController = Mockito.mock(ZkController.class);
         ClusterState clusterState = Mockito.mock(ClusterState.class);
         DocCollection docCollection = Mockito.mock(DocCollection.class);
         DocRouter docRouter = Mockito.mock(DocRouter.class);
         Slice slice = Mockito.mock(Slice.class);
-        ZkStateReader zkStateReader = Mockito.mock(ZkStateReader.class);
-        Replica replica = Mockito.mock(Replica.class);
+        Mockito.when(slice.getName()).thenReturn("shard1");
+        zkStateReader = Mockito.mock(ZkStateReader.class);
+        replica = Mockito.mock(Replica.class);
 
         Mockito.when(replica.getName()).thenReturn("replica1");
         Mockito.when(zkStateReader.getLeaderRetry(Mockito.any(), Mockito.any()))
@@ -102,6 +121,7 @@ public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {
                                 Mockito.any(),
                                 Mockito.any()))
                 .thenReturn(slice);
+        Mockito.when(docCollection.getSlicesMap()).thenReturn(Map.of("shard1", slice));
         Mockito.when(zkController.getClusterState()).thenReturn(clusterState);
         Mockito.when(coreContainer.getZkController()).thenReturn(zkController);
         Mockito.when(core.getCoreContainer()).thenReturn(coreContainer);
@@ -156,6 +176,91 @@ public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {
         }
     }
 
+    @Test
+    public void processCommitOnlyAnotherShard() {
+        try {
+            // should skip if processing in other shard than the first
+            Mockito.when(cloudDesc.getShardId()).thenReturn("shard2");
+            processor.processCommit(commitUpdateCommand);
+            Mockito.verify(next).processCommit(commitUpdateCommand);
+            Mockito.verify(requestMirroringHandler, Mockito.times(0)).mirror(requestMock);
+        } catch (Exception e) {
+            fail("IOException should not be thrown: " + e);
+        }
+    }
+
+    @Test
+    public void processCommitOnlyNonLeader() {
+        try {
+            // should skip if processing on non-leader replica
+            Mockito.when(replica.getName()).thenReturn("foobar");
+            Mockito.when(cloudDesc.getCoreNodeName()).thenReturn("replica1");
+            processor.processCommit(commitUpdateCommand);
+            Mockito.verify(next).processCommit(commitUpdateCommand);
+            Mockito.verify(requestMirroringHandler, Mockito.times(0)).mirror(requestMock);
+        } catch (Exception e) {
+            fail("IOException should not be thrown: " + e);
+        }
+    }
+
+    @Test
+    public void processCommitOnlyLeader() {
+        try {
+            Mockito.when(cloudDesc.getCoreNodeName()).thenReturn("replica1");
+            // don't override createMirrorRequest, call actual method
+            processor =
+                new MirroringUpdateProcessor(
+                    next,
+                    true,
+                    true,
+                    true,
+                    1000L,
+                    new ModifiableSolrParams(),
+                    DistributedUpdateProcessor.DistribPhase.NONE,
+                    requestMirroringHandler);
+            ArgumentCaptor<UpdateRequest> captor = ArgumentCaptor.forClass(UpdateRequest.class);
+            processor.processCommit(commitUpdateCommand);
+            Mockito.verify(next).processCommit(commitUpdateCommand);
+            Mockito.verify(requestMirroringHandler, Mockito.times(1)).mirror(captor.capture());
+            UpdateRequest req = captor.getValue();
+            assertNotNull(req.getParams());
+            SolrParams params = req.getParams();
+            assertEquals("true", params.get(UpdateParams.COMMIT));
+            assertEquals("true", params.get(UpdateParams.OPTIMIZE));
+            assertEquals("true", params.get(UpdateParams.SOFT_COMMIT));
+            assertEquals("true", params.get(UpdateParams.PREPARE_COMMIT));
+            assertEquals("true", params.get(UpdateParams.WAIT_SEARCHER));
+            assertEquals("true", params.get(UpdateParams.OPEN_SEARCHER));
+            assertEquals("true", params.get(UpdateParams.EXPUNGE_DELETES));
+            assertEquals("10", params.get(UpdateParams.MAX_OPTIMIZE_SEGMENTS));
+        } catch (Exception e) {
+            fail("Exception should not be thrown: " + e);
+        }
+    }
+
+    @Test
+    public void processCommitNoMirroring() {
+        try {
+            Mockito.when(cloudDesc.getCoreNodeName()).thenReturn("replica1");
+            // don't override createMirrorRequest, call actual method
+            processor =
+                new MirroringUpdateProcessor(
+                    next,
+                    true,
+                    true,
+                    false,
+                    1000L,
+                    new ModifiableSolrParams(),
+                    DistributedUpdateProcessor.DistribPhase.NONE,
+                    requestMirroringHandler);
+            processor.processCommit(commitUpdateCommand);
+            Mockito.verify(next).processCommit(commitUpdateCommand);
+            Mockito.verify(requestMirroringHandler, Mockito.times(0)).mirror(requestMock);
+        } catch (Exception e) {
+            fail("Exception should not be thrown: " + e);
+        }
+    }
+
     @Test
     public void testProcessAddWithinLimit() throws Exception {
         Mockito.when(cloudDesc.getCoreNodeName()).thenReturn("replica1");
@@ -187,7 +292,7 @@ public class MirroringUpdateProcessorTest extends SolrTestCaseJ4 {
 
         SolrParams mirrorParams = new ModifiableSolrParams();
         MirroringUpdateProcessor mirroringUpdateProcessorWithLimit = new MirroringUpdateProcessor(nextProcessor, true, false, // indexUnmirrorableDocs set to false
-                50000, mirrorParams, DistributedUpdateProcessor.DistribPhase.NONE, requestMirroringHandler);
+                true, 50000, mirrorParams, DistributedUpdateProcessor.DistribPhase.NONE, requestMirroringHandler);
 
         assertThrows(SolrException.class, () -> mirroringUpdateProcessorWithLimit.processAdd(addUpdateCommand));
     }