You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mk...@apache.org on 2023/04/19 07:41:37 UTC
[jackrabbit-oak] branch trunk updated: OAK-10183: Large Bulk size for BulkProcessor sometimes lead to Socket… (#892)
This is an automated email from the ASF dual-hosted git repository.
mkataria pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 394a24d9a0 OAK-10183: Large Bulk size for BulkProcessor sometimes lead to Socket… (#892)
394a24d9a0 is described below
commit 394a24d9a0f9af1d6ca14fa8e4d7ab6eabb16299
Author: Mohit Kataria <mk...@apache.org>
AuthorDate: Wed Apr 19 13:11:30 2023 +0530
OAK-10183: Large Bulk size for BulkProcessor sometimes lead to Socket… (#892)
* OAK-10183: Large Bulk size for BulkProcessor sometimes lead to SocketTimeoutException
---
.../index/elastic/ElasticIndexDefinition.java | 2 +-
.../index/elastic/ElasticPropertyIndexTest.java | 53 +++++++++++++++++++++-
2 files changed, 52 insertions(+), 3 deletions(-)
diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
index 114a655026..15b2773a4d 100644
--- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
+++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
@@ -45,7 +45,7 @@ public class ElasticIndexDefinition extends IndexDefinition {
public static final int BULK_ACTIONS_DEFAULT = 250;
public static final String BULK_SIZE_BYTES = "bulkSizeBytes";
- public static final long BULK_SIZE_BYTES_DEFAULT = 2 * 1024 * 1024; // 2MB
+ public static final long BULK_SIZE_BYTES_DEFAULT = 1 * 1024 * 1024; // 1MB
public static final String BULK_FLUSH_INTERVAL_MS = "bulkFlushIntervalMs";
public static final long BULK_FLUSH_INTERVAL_MS_DEFAULT = 3000;
diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticPropertyIndexTest.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticPropertyIndexTest.java
index 4f5120b92c..afa4dafd9f 100644
--- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticPropertyIndexTest.java
+++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticPropertyIndexTest.java
@@ -16,10 +16,14 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elastic;
+import ch.qos.logback.classic.Level;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
import org.apache.jackrabbit.oak.plugins.index.elastic.util.ElasticIndexDefinitionBuilder;
import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder;
+import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
@@ -32,7 +36,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
public class ElasticPropertyIndexTest extends ElasticAbstractQueryTest {
@Test
- public void testBulkProcessorFlushLimit() throws Exception {
+ public void testBulkProcessorEventsFlushLimit() throws Exception {
setIndex("test1", createIndex("propa"));
Tree test = root.getTree("/").addChild("test");
@@ -63,6 +67,51 @@ public class ElasticPropertyIndexTest extends ElasticAbstractQueryTest {
});
}
+ @Test
+ public void testBulkProcessorSizeFlushLimit() throws Exception {
+ LogCustomizer customLogger = LogCustomizer
+ .forLogger(
+ "org.apache.jackrabbit.oak.plugins.index.elastic.index.ElasticBulkProcessorHandler")
+ .enable(Level.DEBUG).create();
+ try {
+ customLogger.starting();
+ /*
+ Below are the conditions to flush data from bulk processor.
+ 1. Based on events by default 250 events.
+ 2. Based on size of estimated bulk request size.
+ 3. When index writer is closed.
+ To trigger flush on bulk request size, we will load large documents so that
+ instead of event, flush is triggered because of bulk request size.
+ */
+ setIndex("test1", createIndex("propa", "propb"));
+ long bulkSize = ElasticIndexDefinition.BULK_SIZE_BYTES_DEFAULT;
+ int docSize = 1024 * 16;
+ // +1 at end leads to bulk size breach, leading to two bulkIds.
+ long docCountBreachingBulkSize = (bulkSize / docSize) + 1;
+ // 250 is the default flush limit for bulk processor
+ Assert.assertTrue(docCountBreachingBulkSize < 250);
+ String random = RandomStringUtils.random(docSize, true, true);
+
+ Tree test = root.getTree("/").addChild("test");
+ for (int i = 1; i <= docCountBreachingBulkSize; i++) {
+ test.addChild("a" + i).setProperty("propa", "foo" + i);
+ test.addChild("a" + i).setProperty("propb", random + i);
+ }
+ root.commit();
+
+ String propaQuery = "select [jcr:path] from [nt:base] where [propa] = 'foo" + docCountBreachingBulkSize + "'";
+ assertEventually(() -> {
+ assertThat(explain(propaQuery), containsString("elasticsearch:test1"));
+ assertQuery(propaQuery, List.of("/test/a" + docCountBreachingBulkSize));
+ });
+
+ Assert.assertEquals(1, customLogger.getLogs().stream().filter(n -> n.contains("Bulk with id 2 processed with status OK in")).count());
+ Assert.assertEquals(0, customLogger.getLogs().stream().filter(n -> n.contains("Bulk with id 3 processed with status OK in")).count());
+ } finally {
+ customLogger.finished();
+ }
+ }
+
@Test
public void indexSelection() throws Exception {
setIndex("test1", createIndex("propa", "propb"));
@@ -181,7 +230,7 @@ public class ElasticPropertyIndexTest extends ElasticAbstractQueryTest {
Tree test = root.getTree("/").addChild("test");
for (int i = 0; i < 10; i++) {
- test.addChild("node-" + i).setProperty("propa", (double)i);
+ test.addChild("node-" + i).setProperty("propa", (double) i);
}
root.commit();