You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jg...@apache.org on 2022/03/02 15:04:55 UTC
[nifi] branch main updated: NIFI-9678 update Elasticsearch REST API processor integration-tests for Elasticsearch 8.x
This is an automated email from the ASF dual-hosted git repository.
jgresock pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 4cc20e6 NIFI-9678 update Elasticsearch REST API processor integration-tests for Elasticsearch 8.x
4cc20e6 is described below
commit 4cc20e6c0687277394f021ca1f3ececd430c5a0e
Author: Chris Sampson <ch...@gmail.com>
AuthorDate: Tue Mar 1 23:24:35 2022 +0000
NIFI-9678 update Elasticsearch REST API processor integration-tests for Elasticsearch 8.x
Signed-off-by: Joe Gresock <jg...@gmail.com>
This closes #5822.
---
.../elasticsearch/ElasticSearchClientService.java | 8 ++++
.../nifi-elasticsearch-client-service/README.md | 18 +++++++-
.../nifi-elasticsearch-client-service/pom.xml | 48 +++++++++++++++++-----
.../ElasticSearchClientServiceImpl.java | 17 +++++++-
.../ElasticSearchClientService_IT.groovy | 9 +++-
.../TestElasticSearchClientService.groovy | 4 ++
.../src/test/resources/setup-5.script | 2 -
.../src/test/resources/setup-6.script | 2 -
.../src/test/resources/setup-7.script | 2 -
.../resources/{setup-7.script => setup-8.script} | 2 -
.../TestElasticsearchClientService.groovy | 4 ++
.../mock/AbstractMockElasticsearchClient.groovy | 4 ++
12 files changed, 99 insertions(+), 21 deletions(-)
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java
index 138d0a9..d1dc33a 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java
@@ -196,6 +196,14 @@ public interface ElasticSearchClientService extends ControllerService {
UpdateOperationResponse updateByQuery(String query, String index, String type, Map<String, String> requestParameters);
/**
+ * Refresh index/indices.
+ *
+ * @param index The index to target, if omitted then all indices will be updated.
+ * @param requestParameters A collection of URL request parameters. Optional.
+ */
+ void refresh(final String index, final Map<String, String> requestParameters);
+
+ /**
* Get a document by ID.
*
* @param index The index that holds the document.
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/README.md b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/README.md
index 5ca1e4b..023544d 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/README.md
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/README.md
@@ -43,7 +43,13 @@ mvn -P integration-tests,elasticsearch-oss,elasticsearch-6 clean verify
### Elasticsearch 7
-Test integration with Elasticsearch 7.x:
+[elasticsearch-oss](https://www.elastic.co/downloads/past-releases#elasticsearch-oss) was discontinued after `7.10.2`,
+so the use of `elasticsearch-oss` is unnecessary for newer versions.
+
+For 7.x, we have two separate profiles:
+
+1. `elasticsearch-7` that can be used with `oss` (no X-Pack) and `default` (with X-Pack) flavours
+2. `elasticsearch-7-no-oss` that can only be used with the `default` flavour (using a newer version of [elasticsearch](https://www.elastic.co/downloads/past-releases#elasticsearch))
#### With X-Pack
@@ -51,6 +57,8 @@ Allows for testing of some X-Pack only features such as "Point in Time" querying
```bash
mvn -P integration-tests,elasticsearch-default,elasticsearch-7 clean verify
+sleep 2
+mvn -P integration-tests,elasticsearch-default,elasticsearch-7-no-oss clean verify
```
#### Without X-Pack
@@ -58,3 +66,11 @@ mvn -P integration-tests,elasticsearch-default,elasticsearch-7 clean verify
```bash
mvn -P integration-tests,elasticsearch-oss,elasticsearch-7 clean verify
```
+
+### Elasticsearch 8
+
+Test integration with Elasticsearch 8.x (with X-Pack):
+
+```bash
+mvn -P integration-tests,elasticsearch-default,elasticsearch-8 clean verify
+```
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/pom.xml b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/pom.xml
index 0f97fa9..050e2ad 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/pom.xml
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/pom.xml
@@ -28,6 +28,7 @@
<properties>
<!-- use with integration-tests only -->
<es.int.version>5.6.16</es.int.version>
+ <alexcojocaru.plugin.version>6.19</alexcojocaru.plugin.version>
<es.int.script.name>setup-5.script</es.int.script.name>
<es.int.type.name>faketype</es.int.type.name>
<es.int.path.conf>src/test/resources/conf-5/</es.int.path.conf>
@@ -214,11 +215,6 @@
<!-- use 3.0.0-M3 due to a classpath/class loader issue in -M5, expected to be fixed in M6+ -->
<version>3.0.0-M3</version>
</plugin>
- <plugin>
- <groupId>com.github.alexcojocaru</groupId>
- <artifactId>elasticsearch-maven-plugin</artifactId>
- <version>6.19</version>
- </plugin>
</plugins>
</pluginManagement>
</build>
@@ -231,7 +227,8 @@
<activeByDefault>false</activeByDefault>
</activation>
<properties>
- <es.int.version>6.8.19</es.int.version>
+ <es.int.version>6.8.23</es.int.version>
+ <alexcojocaru.plugin.version>6.19</alexcojocaru.plugin.version>
<es.int.type.name>_doc</es.int.type.name>
<es.int.script.name>setup-6.script</es.int.script.name>
<es.int.path.conf />
@@ -245,14 +242,43 @@
</activation>
<properties>
<es.int.version>7.10.2</es.int.version>
+ <alexcojocaru.plugin.version>6.19</alexcojocaru.plugin.version>
<es.int.script.name>setup-7.script</es.int.script.name>
<es.int.type.name />
<es.int.path.conf />
</properties>
</profile>
+ <profile>
+ <!-- use with elasticsearch-default -->
+ <id>elasticsearch-7-no-oss</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <es.int.version>7.17.1</es.int.version>
+ <alexcojocaru.plugin.version>6.19</alexcojocaru.plugin.version>
+ <es.int.script.name>setup-7.script</es.int.script.name>
+ <es.int.type.name />
+ <es.int.path.conf />
+ </properties>
+ </profile>
+ <profile>
+ <!-- use with elasticsearch-default -->
+ <id>elasticsearch-8</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <es.int.version>8.0.1</es.int.version>
+ <alexcojocaru.plugin.version>6.20</alexcojocaru.plugin.version>
+ <es.int.script.name>setup-8.script</es.int.script.name>
+ <es.int.type.name />
+ <es.int.path.conf />
+ </properties>
+ </profile>
<profile>
- <!-- OSS Elasticsearch (no XPack features); required for ES 5.x or < 6.3-->
+ <!-- OSS Elasticsearch (no XPack features); required for ES 5.x or < 6.3 and option for 7.x up to 7.10.2 -->
<id>elasticsearch-oss</id>
<activation>
<activeByDefault>false</activeByDefault>
@@ -273,6 +299,7 @@
<plugin>
<groupId>com.github.alexcojocaru</groupId>
<artifactId>elasticsearch-maven-plugin</artifactId>
+ <version>${alexcojocaru.plugin.version}</version>
<configuration>
<clusterName>${es.int.clusterName}</clusterName>
<transportPort>${es.int.transportPort}</transportPort>
@@ -280,7 +307,7 @@
<version>${es.int.version}</version>
<timeout>${es.int.timeout}</timeout>
<logLevel>${es.int.logLevel}</logLevel>
- <pathInitScript>${project.basedir}/src/test/resources/${es.int.script.name}</pathInitScript>
+ <pathInitScript>${project.basedir}/src/test/resources/${es.int.script.name}</pathInitScript>
<keepExistingData>false</keepExistingData>
<pathConf>${es.int.path.conf}</pathConf>
</configuration>
@@ -306,7 +333,7 @@
</profile>
<profile>
- <!-- Elasticsearch (default) with XPack (only for ES 6.3+ although there are XPack permission problems in 6.x startup) -->
+ <!-- Elasticsearch (default) with XPack (only for ES 6.3+ & 7.x although there are XPack permission problems in 6.x startup) -->
<id>elasticsearch-default</id>
<activation>
<activeByDefault>false</activeByDefault>
@@ -327,6 +354,7 @@
<plugin>
<groupId>com.github.alexcojocaru</groupId>
<artifactId>elasticsearch-maven-plugin</artifactId>
+ <version>${alexcojocaru.plugin.version}</version>
<configuration>
<flavour>default</flavour>
<clusterName>${es.int.clusterName}</clusterName>
@@ -335,7 +363,7 @@
<version>${es.int.version}</version>
<timeout>${es.int.timeout}</timeout>
<logLevel>${es.int.logLevel}</logLevel>
- <pathInitScript>${project.basedir}/src/test/resources/${es.int.script.name}</pathInitScript>
+ <pathInitScript>${project.basedir}/src/test/resources/${es.int.script.name}</pathInitScript>
<keepExistingData>false</keepExistingData>
</configuration>
<executions>
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java
index 202c873..58dc99e 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java
@@ -389,7 +389,7 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im
return new DeleteOperationResponse(watch.getDuration(TimeUnit.MILLISECONDS));
}
-
+ @Override
public UpdateOperationResponse updateByQuery(final String query, final String index, final String type, final Map<String, String> requestParameters) {
final long start = System.currentTimeMillis();
final Response response = runQuery("_update_by_query", query, index, type, requestParameters);
@@ -401,6 +401,21 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im
return new UpdateOperationResponse(end - start);
}
+ @Override
+ public void refresh(final String index, final Map<String, String> requestParameters) {
+ try {
+ final StringBuilder endpoint = new StringBuilder();
+ if (StringUtils.isNotBlank(index) && !"/".equals(index)) {
+ endpoint.append(index);
+ }
+ endpoint.append("/_refresh");
+ final Response response = performRequest("POST", endpoint.toString(), requestParameters, null);
+ parseResponseWarningHeaders(response);
+ } catch (final Exception ex) {
+ throw new ElasticsearchException(ex);
+ }
+ }
+
@SuppressWarnings("unchecked")
@Override
public Map<String, Object> get(final String index, final String type, final String id, final Map<String, String> requestParameters) {
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.groovy
index aefc38a..5c77cc1 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.groovy
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.groovy
@@ -58,6 +58,7 @@ class ElasticSearchClientService_IT {
static final ComparableVersion VERSION = new ComparableVersion(System.getProperty("es_version", "0.0.0"))
static final ComparableVersion ES_7_10 = new ComparableVersion("7.10")
+ static final ComparableVersion ES_8_0 = new ComparableVersion("8.0")
static final String FLAVOUR = System.getProperty("es_flavour")
static final String DEFAULT = "default"
@@ -117,6 +118,8 @@ class ElasticSearchClientService_IT {
ex.printStackTrace()
throw ex
}
+
+ service.refresh(null, null);
}
@After
@@ -223,9 +226,13 @@ class ElasticSearchClientService_IT {
@Test
void testSearchWarnings() {
+ Assume.assumeTrue("Requires version <8.0 (no search API deprecations yet for 8.x)", VERSION < ES_8_0)
+
String query
String type = TYPE
- if (VERSION.toString().startsWith("7.")) {
+ if (VERSION.toString().startsWith("8.")) {
+ // TODO: something that's deprecated when the 8.x branch progresses to include search-API deprecations
+ } else if (VERSION.toString().startsWith("7.")) {
// querying with _type in ES 7.x is deprecated
query = prettyPrint(toJson([size: 1, query: [match_all: [:]]]))
type = "a-type"
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/TestElasticSearchClientService.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/TestElasticSearchClientService.groovy
index ffb3e39..4e0f425 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/TestElasticSearchClientService.groovy
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/groovy/org/apache/nifi/elasticsearch/integration/TestElasticSearchClientService.groovy
@@ -69,6 +69,10 @@ class TestElasticSearchClientService extends AbstractControllerService implement
}
@Override
+ void refresh(final String index, final Map<String, String> requestParameters) {
+ }
+
+ @Override
Map<String, Object> get(String index, String type, String id, Map<String, String> requestParameters) {
return data
}
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-5.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-5.script
index 429a74d..c2cffc4 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-5.script
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-5.script
@@ -46,5 +46,3 @@ PUT:nested/faketype/1:{"msg":"Hello, world","subField":{"longField":150000,"date
PUT:nested/faketype/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}}
PUT:nested/faketype/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}}
-# refresh all indices before testing
-POST:_refresh:{}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script
index 739c0f2..c3522f5 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script
@@ -46,5 +46,3 @@ PUT:nested/_doc/1:{"msg":"Hello, world","subField":{"longField":150000,"dateFiel
PUT:nested/_doc/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}}
PUT:nested/_doc/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}}
-# refresh all indices before testing
-POST:_refresh:{}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script
index 4328390..d7cfcc9 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script
@@ -46,5 +46,3 @@ POST:nested/_doc/1:{"msg":"Hello, world","subField":{"longField":150000,"dateFie
POST:nested/_doc/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}}
POST:nested/_doc/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}}
-# refresh all indices before testing
-POST:_refresh:{}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script
similarity index 98%
copy from nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script
copy to nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script
index 4328390..d7cfcc9 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script
@@ -46,5 +46,3 @@ POST:nested/_doc/1:{"msg":"Hello, world","subField":{"longField":150000,"dateFie
POST:nested/_doc/2:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Hello, world!","deepest":{"super_secret":"I could tell, but then I would have to kill you"}}}}
POST:nested/_doc/3:{"msg":"Hello, world","subField":{"longField":150000,"dateField":"2018-08-14T10:08:00Z","deeper":{"secretz":"Buongiorno, mondo!!","deepest":{"super_secret":"The sky is blue"}}}}
-# refresh all indices before testing
-POST:_refresh:{}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy
index 7410380..02eed95 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy
@@ -98,6 +98,10 @@ class TestElasticsearchClientService extends AbstractControllerService implement
}
@Override
+ void refresh(final String index, final Map<String, String> requestParameters) {
+ }
+
+ @Override
Map<String, Object> get(String index, String type, String id, Map<String, String> requestParameters) {
common(throwErrorInGet || throwNotFoundInGet, requestParameters)
return [ "msg": "one" ]
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy
index dca1f85..52a0316 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy
@@ -66,6 +66,10 @@ class AbstractMockElasticsearchClient extends AbstractControllerService implemen
}
@Override
+ void refresh(final String index, final Map<String, String> requestParameters) {
+ }
+
+ @Override
Map<String, Object> get(String index, String type, String id, Map<String, String> requestParameters) {
return null
}