You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2017/09/26 06:36:05 UTC
svn commit: r1809685 - in /manifoldcf/trunk: ./ connectors/solr/
connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/
framework/buildfiles/
Author: kwright
Date: Tue Sep 26 06:36:05 2017
New Revision: 1809685
URL: http://svn.apache.org/viewvc?rev=1809685&view=rev
Log:
Fix for CONNECTORS-1458.
Modified:
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/build.xml
manifoldcf/trunk/connectors/solr/build.xml
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedLBHttpSolrClient.java
manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/trunk/framework/buildfiles/connector-build.xml
manifoldcf/trunk/pom.xml
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1809685&r1=1809684&r2=1809685&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Tue Sep 26 06:36:05 2017
@@ -3,6 +3,11 @@ $Id$
======================= 2.9-dev =====================
+CONNECTORS-1458: Update to use SolrJ 7.0.0. This required some
+revision to the connector, and also to make zookeeper.jar available
+to all connectors as a root dependency.
+(Karl Wright)
+
CONNECTORS-1453: Integrate poi 3.17, which seems to fix the
classloader issue in 3.15. Reverted the reassignment of poi and
dependencies to root-level classpath accordingly.
Modified: manifoldcf/trunk/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/build.xml?rev=1809685&r1=1809684&r2=1809685&view=diff
==============================================================================
--- manifoldcf/trunk/build.xml (original)
+++ manifoldcf/trunk/build.xml Tue Sep 26 06:36:05 2017
@@ -35,7 +35,7 @@
<property name="httpmime.version" value="4.5.3"/>
<property name="httpcore.version" value="4.4.6"/>
<property name="xerces.version" value="2.10.0"/>
- <property name="zookeeper.version" value="3.4.8"/>
+ <property name="zookeeper.version" value="3.4.10"/>
<property name="mongodb.version" value="2.11.3"/>
<property name="postgresql.version" value="42.1.3"/>
<property name="axis.version" value="1.4"/>
@@ -66,6 +66,9 @@
<property name="jaxb-api.version" value="2.1"/>
<property name="jaxb-impl.version" value="2.1.11"/>
<property name="jaxws.version" value="2.2.6"/>
+ <property name="solrj.version" value="7.0.0"/>
+ <property name="noggit.version" value="0.8"/>
+ <property name="commons-math3.version" value="3.4.1"/>
<property name="chemistry.version" value="0.11.0"/>
<property name="glassfish.version" value="2.1.v20091210"/>
<property name="junit.version" value="4.12"/>
@@ -2156,17 +2159,24 @@ Use Apache Forrest version forrest-0.9-d
<antcall target="download-via-maven">
<param name="target" value="lib"/>
<param name="project-path" value="org/apache/solr"/>
- <param name="artifact-version" value="6.3.0"/>
+ <param name="artifact-version" value="${solrj.version}"/>
<param name="artifact-name" value="solr-solrj"/>
<param name="artifact-type" value="jar"/>
</antcall>
<antcall target="download-via-maven">
<param name="target" value="lib"/>
<param name="project-path" value="org/noggit"/>
- <param name="artifact-version" value="0.6"/>
+ <param name="artifact-version" value="${noggit.version}"/>
<param name="artifact-name" value="noggit"/>
<param name="artifact-type" value="jar"/>
</antcall>
+ <antcall target="download-via-maven">
+ <param name="target" value="lib"/>
+ <param name="project-path" value="org/apache/commons"/>
+ <param name="artifact-version" value="${commons-math3.version}"/>
+ <param name="artifact-name" value="commons-math3"/>
+ <param name="artifact-type" value="jar"/>
+ </antcall>
</target>
<target name="download-zookeeper">
Modified: manifoldcf/trunk/connectors/solr/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/build.xml?rev=1809685&r1=1809684&r2=1809685&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/build.xml (original)
+++ manifoldcf/trunk/connectors/solr/build.xml Tue Sep 26 06:36:05 2017
@@ -34,6 +34,7 @@
<path refid="mcf-connector-build.connector-classpath"/>
<fileset dir="../../lib">
<include name="solr-solrj*.jar"/>
+ <include name="commons-math3*.jar"/>
<include name="noggit*.jar"/>
<include name="httpmime*.jar"/>
<include name="woodstox-core-asl*.jar"/>
@@ -47,6 +48,7 @@
<copy todir="dist/lib">
<fileset dir="../../lib">
<include name="solr-solrj*.jar"/>
+ <include name="commons-math3*.jar"/>
<include name="noggit*.jar"/>
<include name="httpmime*.jar"/>
<include name="woodstox-core-asl*.jar"/>
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1809685&r1=1809684&r2=1809685&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java Tue Sep 26 06:36:05 2017
@@ -134,7 +134,7 @@ public class HttpPoster
String originalSizeAttributeName, String modifiedDateAttributeName, String createdDateAttributeName, String indexedDateAttributeName,
String fileNameAttributeName, String mimeTypeAttributeName, String contentAttributeName,
Long maxDocumentLength,
- String commitWithin, boolean useExtractUpdateHandler)
+ String commitWithin, boolean useExtractUpdateHandler, boolean allowCompression)
throws ManifoldCFException
{
// These are the paths to the handlers in Solr that deal with the actions we need to do
@@ -162,7 +162,7 @@ public class HttpPoster
{
CloudSolrClient cloudSolrServer = new CloudSolrClient.Builder()
.withZkHost(zookeeperHosts)
- .withLBHttpSolrClient(new ModifiedLBHttpSolrClient(HttpClientUtil.createClient(null)))
+ .withLBHttpSolrClient(new ModifiedLBHttpSolrClient(HttpClientUtil.createClient(null), allowCompression))
.build();
cloudSolrServer.setZkClientTimeout(zkClientTimeout);
cloudSolrServer.setZkConnectTimeout(zkConnectTimeout);
@@ -186,7 +186,7 @@ public class HttpPoster
String originalSizeAttributeName, String modifiedDateAttributeName, String createdDateAttributeName, String indexedDateAttributeName,
String fileNameAttributeName, String mimeTypeAttributeName, String contentAttributeName,
IKeystoreManager keystoreManager, Long maxDocumentLength,
- String commitWithin, boolean useExtractUpdateHandler)
+ String commitWithin, boolean useExtractUpdateHandler, boolean allowCompression)
throws ManifoldCFException
{
// These are the paths to the handlers in Solr that deal with the actions we need to do
@@ -276,7 +276,7 @@ public class HttpPoster
String httpSolrServerUrl = protocol + "://" + server + ":" + port + location;
- solrServer = new ModifiedHttpSolrClient(httpSolrServerUrl, localClient, new XMLResponseParser());
+ solrServer = new ModifiedHttpSolrClient(httpSolrServerUrl, localClient, new XMLResponseParser(), allowCompression);
}
/** Shut down the poster.
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java?rev=1809685&r1=1809684&r2=1809685&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedHttpSolrClient.java Tue Sep 26 06:36:05 2017
@@ -97,8 +97,8 @@ public class ModifiedHttpSolrClient exte
private final HttpClient httpClient;
private final boolean useMultiPartPost = true;
- public ModifiedHttpSolrClient(String baseURL, HttpClient client, ResponseParser parser) {
- super(baseURL, client, parser);
+ public ModifiedHttpSolrClient(String baseURL, HttpClient client, ResponseParser parser, boolean allowCompression) {
+ super(baseURL, client, parser, allowCompression);
httpClient = client;
}
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedLBHttpSolrClient.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedLBHttpSolrClient.java?rev=1809685&r1=1809684&r2=1809685&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedLBHttpSolrClient.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/ModifiedLBHttpSolrClient.java Tue Sep 26 06:36:05 2017
@@ -33,28 +33,30 @@ public class ModifiedLBHttpSolrClient ex
{
private final HttpClient httpClient;
private final ResponseParser parser;
+ private final boolean allowCompression;
- public ModifiedLBHttpSolrClient(String... solrServerUrls) throws MalformedURLException {
- this(null, solrServerUrls);
+ public ModifiedLBHttpSolrClient(boolean allowCompression, String... solrServerUrls) throws MalformedURLException {
+ this(null, allowCompression, solrServerUrls);
}
/** The provided httpClient should use a multi-threaded connection manager */
- public ModifiedLBHttpSolrClient(HttpClient httpClient, String... solrServerUrl)
+ public ModifiedLBHttpSolrClient(HttpClient httpClient, boolean allowCompression, String... solrServerUrl)
throws MalformedURLException {
- this(httpClient, new BinaryResponseParser(), solrServerUrl);
+ this(httpClient, new BinaryResponseParser(), allowCompression, solrServerUrl);
}
/** The provided httpClient should use a multi-threaded connection manager */
- public ModifiedLBHttpSolrClient(HttpClient httpClient, ResponseParser parser, String... solrServerUrl)
+ public ModifiedLBHttpSolrClient(HttpClient httpClient, ResponseParser parser, boolean allowCompression, String... solrServerUrl)
throws MalformedURLException {
super(httpClient, parser, solrServerUrl);
this.httpClient = httpClient;
this.parser = parser;
+ this.allowCompression = allowCompression;
}
@Override
protected HttpSolrClient makeSolrClient(String server) {
- HttpSolrClient client = new ModifiedHttpSolrClient(server, httpClient, parser);
+ HttpSolrClient client = new ModifiedHttpSolrClient(server, httpClient, parser, allowCompression);
if (getRequestWriter() != null) {
client.setRequestWriter(getRequestWriter());
}
Modified: manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1809685&r1=1809684&r2=1809685&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java (original)
+++ manifoldcf/trunk/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java Tue Sep 26 06:36:05 2017
@@ -94,7 +94,9 @@ public class SolrConnector extends org.a
protected String contentAttributeName = null;
/** Use extractiing update handler? */
protected boolean useExtractUpdateHandler = true;
-
+ /** Allow compression? Currently static */
+ protected final static boolean allowCompression = true;
+
/** Whether or not to commit */
protected boolean doCommits = false;
@@ -363,7 +365,7 @@ public class SolrConnector extends org.a
allowAttributeName,denyAttributeName,idAttributeName,
originalSizeAttributeName,modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
fileNameAttributeName,mimeTypeAttributeName,contentAttributeName,
- keystoreManager,maxDocumentLength,commitWithin,useExtractUpdateHandler);
+ keystoreManager,maxDocumentLength,commitWithin,useExtractUpdateHandler,allowCompression);
}
catch (NumberFormatException e)
@@ -412,14 +414,14 @@ public class SolrConnector extends org.a
{
int zkClientTimeout = Integer.parseInt(zkClientTimeoutString) * 1000;
int zkConnectTimeout = Integer.parseInt(zkConnectTimeoutString) * 1000;
-
+
poster = new HttpPoster(zookeeperHost,collection,
zkClientTimeout,zkConnectTimeout,
updatePath,removePath,statusPath,
allowAttributeName,denyAttributeName,idAttributeName,
originalSizeAttributeName,modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
fileNameAttributeName,mimeTypeAttributeName,contentAttributeName,
- maxDocumentLength,commitWithin,useExtractUpdateHandler);
+ maxDocumentLength,commitWithin,useExtractUpdateHandler,allowCompression);
}
catch (NumberFormatException e)
Modified: manifoldcf/trunk/framework/buildfiles/connector-build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/buildfiles/connector-build.xml?rev=1809685&r1=1809684&r2=1809685&view=diff
==============================================================================
--- manifoldcf/trunk/framework/buildfiles/connector-build.xml (original)
+++ manifoldcf/trunk/framework/buildfiles/connector-build.xml Tue Sep 26 06:36:05 2017
@@ -115,6 +115,7 @@
<include name="mcf-pull-agent.jar"/>
</fileset>
<fileset dir="${mcf-dist}/lib">
+ <include name="zookeeper*.jar"/>
<include name="mail*.jar"/>
<include name="commons-exec*.jar"/>
<include name="guava*.jar"/>
Modified: manifoldcf/trunk/pom.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/pom.xml?rev=1809685&r1=1809684&r2=1809685&view=diff
==============================================================================
--- manifoldcf/trunk/pom.xml (original)
+++ manifoldcf/trunk/pom.xml Tue Sep 26 06:36:05 2017
@@ -65,8 +65,8 @@
<httpcomponent.httpclient.version>4.5.3</httpcomponent.httpclient.version>
<httpcomponent.httpcore.version>4.4.6</httpcomponent.httpcore.version>
<httpcomponent.httpmime.version>4.5.3</httpcomponent.httpmime.version>
- <solr.version>6.3.0</solr.version>
- <noggit.version>0.6</noggit.version>
+ <solr.version>7.0.0</solr.version>
+ <noggit.version>0.8</noggit.version>
<commons-el.version>1.0</commons-el.version>
<commons-lang.version>2.6</commons-lang.version>
<xalan.version>2.7.1</xalan.version>
@@ -81,7 +81,7 @@
<velocity.version>1.7</velocity.version>
<slf4j.version>1.7.7</slf4j.version>
<jaxb.version>2.2.6</jaxb.version>
- <zookeeper.version>3.4.8</zookeeper.version>
+ <zookeeper.version>3.4.10</zookeeper.version>
<xmlbeans.version>2.6.0</xmlbeans.version>
<poi.version>3.17</poi.version>
<tika.version>1.14</tika.version>