You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2020/05/03 11:59:42 UTC
svn commit: r1877321 - in /manifoldcf/trunk: ./ CHANGES.txt
connectors/elasticsearch/build.xml
connectors/elasticsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/elasticsearch/tests/BaseITHSQLDB.java
Author: kwright
Date: Sun May 3 11:59:42 2020
New Revision: 1877321
URL: http://svn.apache.org/viewvc?rev=1877321&view=rev
Log:
Fix CONNECTORS-1639.
Modified:
manifoldcf/trunk/ (props changed)
manifoldcf/trunk/CHANGES.txt
manifoldcf/trunk/connectors/elasticsearch/build.xml
manifoldcf/trunk/connectors/elasticsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/elasticsearch/tests/BaseITHSQLDB.java
Propchange: manifoldcf/trunk/
------------------------------------------------------------------------------
Merged /manifoldcf/branches/CONNECTORS-1639:r1876717-1877320
Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1877321&r1=1877320&r2=1877321&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Sun May 3 11:59:42 2020
@@ -3,6 +3,9 @@ $Id$
======================= 2.16-dev =====================
+CONNECTORS-1639: Bring the version of the ES service used for testing up to date.
+(Karl Wright, Michael Cizmar)
+
CONNECTORS-1637: Add Confluence V6+ connector. Thanks to Julien Massiera
for the contribution!
(Julien Massiera)
Modified: manifoldcf/trunk/connectors/elasticsearch/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/build.xml?rev=1877321&r1=1877320&r2=1877321&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/build.xml (original)
+++ manifoldcf/trunk/connectors/elasticsearch/build.xml Sun May 3 11:59:42 2020
@@ -30,31 +30,42 @@
<import file="${mcf-dist}/connector-build.xml"/>
- <property name="elasticsearch.version" value="1.0.1"/>
+ <property name="elasticsearch.version" value="7.6.2"/>
+ <property name="elasticsearch.mapper-attachments.version" value="3.1.2"/>
- <target name="download-dependencies">
-
+ <condition property="isWindows">
+ <os family="windows" />
+ </condition>
+ <condition property="isUnix">
+ <os family="unix" />
+ </condition>
+
+ <target name="download-elasticsearch-unix" if="isUnix">
<mkdir dir="build/download"/>
- <get src="http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${elasticsearch.version}.tar.gz"
+ <get src="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${elasticsearch.version}-linux-x86_64.tar.gz"
dest="build/download/elasticsearch.tar.gz"/>
<gunzip src="build/download/elasticsearch.tar.gz"/>
- <untar src="build/download/elasticsearch.tar" dest="test-materials"/>
-
- <!-- download and install the attachment plugin -->
- <mkdir dir="test-materials/elasticsearch-${elasticsearch.version}/plugins"/>
-
- <get src="https://github.com/elasticsearch/elasticsearch-mapper-attachments/archive/v2.0.0.RC1.zip"
- dest="test-materials/elasticsearch-${elasticsearch.version}/plugins/mapper-attachments.zip"/>
-
- <unzip src="test-materials/elasticsearch-${elasticsearch.version}/plugins/mapper-attachments.zip" dest="test-materials/elasticsearch-${elasticsearch.version}/plugins/mapper-attachments"/>
-
- <chmod dir="test-materials/elasticsearch-${elasticsearch.version}" perm="ugo+rxw">
+ <mkdir dir="test-materials/unix"/>
+ <untar src="build/download/elasticsearch.tar" dest="test-materials/unix"/>
+ <chmod dir="test-materials/unix/elasticsearch-${elasticsearch.version}" perm="ugo+rxw">
<fileset dir="**/*"/>
</chmod>
- <chmod file="test-materials/elasticsearch-${elasticsearch.version}/bin/plugin" perm="ugo+rx"/>
- <chmod file="test-materials/elasticsearch-${elasticsearch.version}/bin/elasticsearch" perm="ugo+rx"/>
- <chmod file="test-materials/elasticsearch-${elasticsearch.version}/bin/elasticsearch.in.sh" perm="ugo+rx"/>
-
+ <chmod file="test-materials/unix/elasticsearch-${elasticsearch.version}/bin/elasticsearch" perm="ugo+rx"/>
+
+ </target>
+
+ <target name="download-elasticsearch-windows" if="isWindows">
+ <mkdir dir="build/download"/>
+ <get src="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${elasticsearch.version}-windows-x86_64.zip"
+ dest="build/download/elasticsearch.zip"/>
+ <mkdir dir="test-materials/windows"/>
+ <unzip src="build/download/elasticsearch.zip" dest="test-materials/windows"/>
+ </target>
+
+ <target name="download-elasticsearch" depends="download-elasticsearch-unix,download-elasticsearch-windows">
+ </target>
+
+ <target name="download-dependencies" depends="download-elasticsearch">
</target>
<target name="download-cleanup">
@@ -74,22 +85,21 @@
</copy>
</target>
- <target name="calculate-testcode-condition">
- <available file="test-materials/elasticsearch-${elasticsearch.version}" property="tests-present"/>
+ <target name="calculate-testcode-condition-unix" if="isUnix">
+ <available file="test-materials/unix/elasticsearch-${elasticsearch.version}" property="tests-present"/>
+ </target>
+
+ <target name="calculate-testcode-condition-windows" if="isWindows">
+ <available file="test-materials/windows/elasticsearch-${elasticsearch.version}" property="tests-present"/>
+ </target>
+
+ <target name="calculate-testcode-condition" depends="calculate-testcode-condition-unix,calculate-testcode-condition-windows">
</target>
<target name="pretest-warn" depends="calculate-testcode-condition" unless="tests-present">
<echo message="Need ElasticSearch download in order to compile tests - run ant make-deps"/>
</target>
- <path id="connector-test-classpath">
- <path refid="mcf-connector-build.connector-test-classpath"/>
- <fileset dir="test-materials/elasticsearch-${elasticsearch.version}">
- <include name="lib/*.jar"/>
- <include name="plugins/mapper-attachments/*.jar"/>
- </fileset>
- </path>
-
<target name="deliver-connector" depends="mcf-connector-build.deliver-connector">
<antcall target="general-add-output-connector">
<param name="connector-label" value="ElasticSearch"/>
Modified: manifoldcf/trunk/connectors/elasticsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/elasticsearch/tests/BaseITHSQLDB.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/connectors/elasticsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/elasticsearch/tests/BaseITHSQLDB.java?rev=1877321&r1=1877320&r2=1877321&view=diff
==============================================================================
--- manifoldcf/trunk/connectors/elasticsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/elasticsearch/tests/BaseITHSQLDB.java (original)
+++ manifoldcf/trunk/connectors/elasticsearch/connector/src/test/java/org/apache/manifoldcf/agents/output/elasticsearch/tests/BaseITHSQLDB.java Sun May 3 11:59:42 2020
@@ -18,7 +18,6 @@
*/
package org.apache.manifoldcf.agents.output.elasticsearch.tests;
-import org.elasticsearch.node.Node;
import org.junit.After;
import org.junit.Before;
@@ -26,7 +25,15 @@ import org.eclipse.jetty.server.handler.
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
-import static org.elasticsearch.node.NodeBuilder.*;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.HttpEntity;
+import org.apache.http.util.EntityUtils;
+import org.apache.http.impl.client.HttpClients;
+import java.io.IOException;
+import java.io.File;
+
/**
* Base integration tests class for Elastic Search tested against a CMIS repository
@@ -35,7 +42,16 @@ import static org.elasticsearch.node.Nod
* */
public class BaseITHSQLDB extends org.apache.manifoldcf.crawler.tests.BaseITHSQLDB
{
- protected Node node = null;
+
+ final static boolean isUnix;
+ static {
+ final String os = System.getProperty("os.name").toLowerCase();
+ if (os.contains("win")) {
+ isUnix = false;
+ } else {
+ isUnix = true;
+ }
+ }
protected String[] getConnectorNames()
{
@@ -57,22 +73,73 @@ public class BaseITHSQLDB extends org.ap
return new String[]{"org.apache.manifoldcf.agents.output.elasticsearch.ElasticSearchConnector"};
}
+ Process esTestProcess = null;
+
@Before
public void setupElasticSearch()
throws Exception
{
- //Initialize ElasticSearch server
- //the default port is 9200
+ final ProcessBuilder pb = new ProcessBuilder();
+
+ final File absFile = new File(".").getAbsoluteFile();
+ System.out.println("ES working directory is '"+absFile+"'");
+ pb.directory(absFile);
+
+ if (isUnix) {
+ pb.command("bash", "-c", "../test-materials/unix/elasticsearch-7.6.2/bin/elasticsearch -q");
+ System.out.println("Unix process");
+ } else {
+ pb.command("cmd.exe", "/c", "..\\test-materials\\windows\\elasticsearch-7.6.2\\bin\\elasticsearch.bat -q");
+ System.out.println("Windows process");
+ }
+
+ File log = new File("es.log");
+ pb.redirectErrorStream(true);
+ pb.redirectOutput(ProcessBuilder.Redirect.appendTo(log));
+ esTestProcess = pb.start();
System.out.println("ElasticSearch is starting...");
- node = nodeBuilder().local(true).node();
+ //the default port is 9200
+
+ waitForElasticSearch();
+
System.out.println("ElasticSearch is started on port 9200");
}
+ public void waitForElasticSearch() {
+ for (int i = 0 ; i < 10000 ; i++) {
+ CloseableHttpClient httpclient = HttpClients.createDefault();
+ HttpGet httpGet = new HttpGet("http://127.0.0.1:9200");
+ try {
+ CloseableHttpResponse response1 = httpclient.execute(httpGet);
+ try {
+ System.out.println("Response from ES: "+response1.getStatusLine());
+ HttpEntity entity1 = response1.getEntity();
+ // do something useful with the response body
+ // and ensure it is fully consumed
+ EntityUtils.consume(entity1);
+ } finally {
+ response1.close();
+ }
+ System.out.println("ES came up!");
+ return;
+ } catch (IOException e) {
+ // Wait 500ms and try again
+ System.out.println("Didn't reach ES; waiting...");
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e1) {
+ break;
+ }
+ }
+ }
+ throw new IllegalStateException("ES didn't come up.");
+ }
@After
public void cleanUpElasticSearch(){
- if(node!=null)
- node.close();
+ if (esTestProcess != null) {
+ esTestProcess.destroy();
+ }
}
}