You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mk...@apache.org on 2023/04/19 07:41:37 UTC

[jackrabbit-oak] branch trunk updated: OAK-10183: Large Bulk size for BulkProcessor sometimes lead to Socket… (#892)

This is an automated email from the ASF dual-hosted git repository.

mkataria pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 394a24d9a0 OAK-10183: Large Bulk size for BulkProcessor sometimes lead to Socket… (#892)
394a24d9a0 is described below

commit 394a24d9a0f9af1d6ca14fa8e4d7ab6eabb16299
Author: Mohit Kataria <mk...@apache.org>
AuthorDate: Wed Apr 19 13:11:30 2023 +0530

    OAK-10183: Large Bulk size for BulkProcessor sometimes lead to Socket… (#892)
    
    * OAK-10183: Large Bulk size for BulkProcessor sometimes lead to SocketTimeoutException
---
 .../index/elastic/ElasticIndexDefinition.java      |  2 +-
 .../index/elastic/ElasticPropertyIndexTest.java    | 53 +++++++++++++++++++++-
 2 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
index 114a655026..15b2773a4d 100644
--- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
+++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
@@ -45,7 +45,7 @@ public class ElasticIndexDefinition extends IndexDefinition {
     public static final int BULK_ACTIONS_DEFAULT = 250;
 
     public static final String BULK_SIZE_BYTES = "bulkSizeBytes";
-    public static final long BULK_SIZE_BYTES_DEFAULT = 2 * 1024 * 1024; // 2MB
+    public static final long BULK_SIZE_BYTES_DEFAULT = 1 * 1024 * 1024; // 1MB
 
     public static final String BULK_FLUSH_INTERVAL_MS = "bulkFlushIntervalMs";
     public static final long BULK_FLUSH_INTERVAL_MS_DEFAULT = 3000;
diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticPropertyIndexTest.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticPropertyIndexTest.java
index 4f5120b92c..afa4dafd9f 100644
--- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticPropertyIndexTest.java
+++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticPropertyIndexTest.java
@@ -16,10 +16,14 @@
  */
 package org.apache.jackrabbit.oak.plugins.index.elastic;
 
+import ch.qos.logback.classic.Level;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.jackrabbit.oak.api.CommitFailedException;
 import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
 import org.apache.jackrabbit.oak.plugins.index.elastic.util.ElasticIndexDefinitionBuilder;
 import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder;
+import org.junit.Assert;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -32,7 +36,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
 public class ElasticPropertyIndexTest extends ElasticAbstractQueryTest {
 
     @Test
-    public void testBulkProcessorFlushLimit() throws Exception {
+    public void testBulkProcessorEventsFlushLimit() throws Exception {
         setIndex("test1", createIndex("propa"));
 
         Tree test = root.getTree("/").addChild("test");
@@ -63,6 +67,51 @@ public class ElasticPropertyIndexTest extends ElasticAbstractQueryTest {
         });
     }
 
+    @Test
+    public void testBulkProcessorSizeFlushLimit() throws Exception {
+        LogCustomizer customLogger = LogCustomizer
+                .forLogger(
+                        "org.apache.jackrabbit.oak.plugins.index.elastic.index.ElasticBulkProcessorHandler")
+                .enable(Level.DEBUG).create();
+        try {
+            customLogger.starting();
+        /*
+        Below are the conditions to flush data from bulk processor.
+        1. Based on events by default 250 events.
+        2. Based on size of estimated bulk request size.
+        3. When index writer is closed.
+        To trigger flush on bulk request size, we will load large documents so that
+         instead of event, flush is triggered because of bulk request size.
+         */
+            setIndex("test1", createIndex("propa", "propb"));
+            long bulkSize = ElasticIndexDefinition.BULK_SIZE_BYTES_DEFAULT;
+            int docSize = 1024 * 16;
+            // +1 at end leads to bulk size breach, leading to two bulkIds.
+            long docCountBreachingBulkSize = (bulkSize / docSize) + 1;
+            // 250 is the default flush limit for bulk processor
+            Assert.assertTrue(docCountBreachingBulkSize < 250);
+            String random = RandomStringUtils.random(docSize, true, true);
+
+            Tree test = root.getTree("/").addChild("test");
+            for (int i = 1; i <= docCountBreachingBulkSize; i++) {
+                test.addChild("a" + i).setProperty("propa", "foo" + i);
+                test.addChild("a" + i).setProperty("propb", random + i);
+            }
+            root.commit();
+
+            String propaQuery = "select [jcr:path] from [nt:base] where [propa] = 'foo" + docCountBreachingBulkSize + "'";
+            assertEventually(() -> {
+                assertThat(explain(propaQuery), containsString("elasticsearch:test1"));
+                assertQuery(propaQuery, List.of("/test/a" + docCountBreachingBulkSize));
+            });
+
+            Assert.assertEquals(1, customLogger.getLogs().stream().filter(n -> n.contains("Bulk with id 2 processed with status OK in")).count());
+            Assert.assertEquals(0, customLogger.getLogs().stream().filter(n -> n.contains("Bulk with id 3 processed with status OK in")).count());
+        } finally {
+            customLogger.finished();
+        }
+    }
+
     @Test
     public void indexSelection() throws Exception {
         setIndex("test1", createIndex("propa", "propb"));
@@ -181,7 +230,7 @@ public class ElasticPropertyIndexTest extends ElasticAbstractQueryTest {
 
         Tree test = root.getTree("/").addChild("test");
         for (int i = 0; i < 10; i++) {
-            test.addChild("node-" + i).setProperty("propa", (double)i);
+            test.addChild("node-" + i).setProperty("propa", (double) i);
         }
         root.commit();