You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2020/02/17 07:21:45 UTC
[camel] branch master updated: CAMEL-12219: Solr - Add support for
soft commits (#3582)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/master by this push:
new 7cc8b59 CAMEL-12219: Solr - Add support for soft commits (#3582)
7cc8b59 is described below
commit 7cc8b59b4b1dc39bcabf3fc591a6455e21cf61ba
Author: Denis Istomin <is...@gmail.com>
AuthorDate: Mon Feb 17 12:21:22 2020 +0500
CAMEL-12219: Solr - Add support for soft commits (#3582)
---
.../camel-solr/src/main/docs/solr-component.adoc | 2 ++
.../apache/camel/component/solr/SolrConstants.java | 1 +
.../apache/camel/component/solr/SolrEndpoint.java | 22 ++++++++++++++--------
.../apache/camel/component/solr/SolrProducer.java | 4 ++++
.../camel/component/solr/JettySolrFactory.java | 2 +-
.../camel/component/solr/SolrCloudFixture.java | 4 +++-
.../apache/camel/component/solr/SolrFixtures.java | 5 +++--
.../camel/component/solr/SolrTransactionsTest.java | 14 ++++++++++++++
8 files changed, 42 insertions(+), 12 deletions(-)
diff --git a/components/camel-solr/src/main/docs/solr-component.adoc b/components/camel-solr/src/main/docs/solr-component.adoc
index ec0c4b1..eba8ed8 100644
--- a/components/camel-solr/src/main/docs/solr-component.adoc
+++ b/components/camel-solr/src/main/docs/solr-component.adoc
@@ -137,6 +137,8 @@ bean]
|COMMIT |n/a |performs a commit on any pending index changes
+|SOFT_COMMIT |n/a |performs a `soft commit` (without guarantee that Lucene index files are written to stable storage; useful for Near Real Time operations) on any pending index changes
+
|ROLLBACK |n/a |performs a rollback on any pending index changes
|OPTIMIZE |n/a |performs a commit on any pending index changes and then runs the
diff --git a/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrConstants.java b/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrConstants.java
index 065fcc2..13ea976 100644
--- a/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrConstants.java
+++ b/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrConstants.java
@@ -22,6 +22,7 @@ public final class SolrConstants {
public static final String OPERATION = "SolrOperation";
public static final String PARAM = "SolrParam.";
public static final String OPERATION_COMMIT = "COMMIT";
+ public static final String OPERATION_SOFT_COMMIT = "SOFT_COMMIT";
public static final String OPERATION_ROLLBACK = "ROLLBACK";
public static final String OPERATION_OPTIMIZE = "OPTIMIZE";
public static final String OPERATION_INSERT = "INSERT";
diff --git a/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrEndpoint.java b/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrEndpoint.java
index 99e7d73..7949926 100644
--- a/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrEndpoint.java
+++ b/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrEndpoint.java
@@ -19,6 +19,7 @@ package org.apache.camel.component.solr;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLDecoder;
+import java.util.Optional;
import org.apache.camel.Consumer;
import org.apache.camel.Processor;
@@ -116,7 +117,7 @@ public class SolrEndpoint extends DefaultEndpoint {
private CloudSolrClient getCloudServer() {
CloudSolrClient rVal = null;
if (this.getZkHost() != null && this.getCollection() != null) {
- rVal = new CloudSolrClient.Builder().withZkHost(zkHost).build();
+ rVal = new CloudSolrClient.Builder(java.util.Arrays.asList(zkHost), Optional.empty()).build();
rVal.setDefaultCollection(this.getCollection());
}
return rVal;
@@ -131,19 +132,24 @@ public class SolrEndpoint extends DefaultEndpoint {
ref = new SolrComponent.SolrServerReference();
CloudSolrClient cloudServer = getCloudServer();
if (cloudServer == null) {
- HttpSolrClient solrServer = new HttpSolrClient.Builder(url).build();
- ConcurrentUpdateSolrClient solrStreamingServer = new ConcurrentUpdateSolrClient.Builder(url).withQueueSize(streamingQueueSize).withThreadCount(streamingThreadCount).build();
-
- // set the properties on the solr server
+ HttpSolrClient.Builder solrServerBuilder = new HttpSolrClient.Builder(url);
+ ConcurrentUpdateSolrClient solrStreamingServer = new ConcurrentUpdateSolrClient.Builder(url)
+ .withQueueSize(streamingQueueSize)
+ .withThreadCount(streamingThreadCount)
+ .build();
if (soTimeout != null) {
- solrServer.setSoTimeout(soTimeout);
+ solrServerBuilder.withSocketTimeout(soTimeout);
}
if (connectionTimeout != null) {
- solrServer.setConnectionTimeout(connectionTimeout);
+ solrServerBuilder.withConnectionTimeout(connectionTimeout);
}
+
+ HttpSolrClient solrServer = solrServerBuilder.build();
+ // set the properties on the solr server
if (followRedirects != null) {
solrServer.setFollowRedirects(followRedirects);
- }
+ }
+
ref.setSolrServer(solrServer);
ref.setUpdateSolrServer(solrStreamingServer);
}
diff --git a/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrProducer.java b/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrProducer.java
index 582112f..c9fcf9c 100644
--- a/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrProducer.java
+++ b/components/camel-solr/src/main/java/org/apache/camel/component/solr/SolrProducer.java
@@ -98,6 +98,10 @@ public class SolrProducer extends DefaultProducer {
UpdateRequest updateRequest = createUpdateRequest();
updateRequest.setAction(ACTION.COMMIT, true, true);
updateRequest.process(serverToUse);
+ } else if (operation.equalsIgnoreCase(SolrConstants.OPERATION_SOFT_COMMIT)) {
+ UpdateRequest updateRequest = createUpdateRequest();
+ updateRequest.setAction(ACTION.COMMIT, true, true, true);
+ updateRequest.process(serverToUse);
} else if (operation.equalsIgnoreCase(SolrConstants.OPERATION_ROLLBACK)) {
UpdateRequest updateRequest = createUpdateRequest();
updateRequest.rollback();
diff --git a/components/camel-solr/src/test/java/org/apache/camel/component/solr/JettySolrFactory.java b/components/camel-solr/src/test/java/org/apache/camel/component/solr/JettySolrFactory.java
index 68c5b4c..bd8d239 100644
--- a/components/camel-solr/src/test/java/org/apache/camel/component/solr/JettySolrFactory.java
+++ b/components/camel-solr/src/test/java/org/apache/camel/component/solr/JettySolrFactory.java
@@ -27,8 +27,8 @@ import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
-import org.apache.http.conn.ssl.SSLContextBuilder;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.ssl.SSLContextBuilder;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.embedded.SSLConfig;
diff --git a/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrCloudFixture.java b/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrCloudFixture.java
index f5ebd38..c32c9a2 100644
--- a/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrCloudFixture.java
+++ b/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrCloudFixture.java
@@ -22,10 +22,12 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import org.apache.camel.util.IOHelper;
import org.apache.log4j.Logger;
@@ -82,7 +84,7 @@ public class SolrCloudFixture {
}
}
- solrClient = new CloudSolrClient.Builder().withZkHost(zkAddr).build();
+ solrClient = new CloudSolrClient.Builder(Arrays.asList(zkAddr), Optional.empty()).build();
solrClient.connect();
createCollection(solrClient, "collection1", 1, 1, "conf1");
diff --git a/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrFixtures.java b/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrFixtures.java
index 0ec2fcf..f6b4f68 100644
--- a/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrFixtures.java
+++ b/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrFixtures.java
@@ -76,8 +76,9 @@ public class SolrFixtures {
solrHttpsRunner = JettySolrFactory.createJettyTestFixture(true);
httpsPort = solrHttpsRunner.getLocalPort();
log.info("Started Https Test Server: " + solrHttpsRunner.getBaseUrl());
- solrHttpsServer = new HttpSolrClient.Builder("https://127.0.0.1:" + httpsPort + "/solr").build();
- solrHttpsServer.setConnectionTimeout(60000);
+ solrHttpsServer = new HttpSolrClient.Builder("https://127.0.0.1:" + httpsPort + "/solr")
+ .withConnectionTimeout(60000)
+ .build();
solrRunner = JettySolrFactory.createJettyTestFixture(false);
port = solrRunner.getLocalPort();
diff --git a/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrTransactionsTest.java b/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrTransactionsTest.java
index a9427b4..de3e5b4 100644
--- a/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrTransactionsTest.java
+++ b/components/camel-solr/src/test/java/org/apache/camel/component/solr/SolrTransactionsTest.java
@@ -54,6 +54,20 @@ public class SolrTransactionsTest extends SolrComponentTestSupport {
}
@Test
+ public void testSoftCommit() throws Exception {
+
+ //insert and verify
+ solrInsertTestEntry();
+ assertEquals("wrong number of entries found", 0, executeSolrQuery("id:" + TEST_ID).getResults().getNumFound());
+
+ //commit
+ template.sendBodyAndHeader("direct:start", null, SolrConstants.OPERATION, SolrConstants.OPERATION_SOFT_COMMIT);
+
+ //verify exists after commit
+ assertEquals("wrong number of entries found", 1, executeSolrQuery("id:" + TEST_ID).getResults().getNumFound());
+ }
+
+ @Test
public void testRollback() throws Exception {
if (SolrFixtures.TestServerType.USE_CLOUD == this.solrServerType) {