You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jg...@apache.org on 2023/05/03 09:03:40 UTC
[nifi] branch support/nifi-1.x updated: NIFI-10067: enable use of script when updating documents in Elasticsearch via PutElasticsearchJson or PutElasticsearchRecord NIFI-3262: enable use of dynamic_templates in Elasticsearch _bulk operations
This is an automated email from the ASF dual-hosted git repository.
jgresock pushed a commit to branch support/nifi-1.x
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/support/nifi-1.x by this push:
new 9bc05c098c NIFI-10067: enable use of script when updating documents in Elasticsearch via PutElasticsearchJson or PutElasticsearchRecord NIFI-3262: enable use of dynamic_templates in Elasticsearch _bulk operations
9bc05c098c is described below
commit 9bc05c098c0baba7e89c3f47b95f41dd714d107f
Author: Chris Sampson <ch...@naimuri.com>
AuthorDate: Thu Oct 27 21:47:36 2022 +0100
NIFI-10067: enable use of script when updating documents in Elasticsearch via PutElasticsearchJson or PutElasticsearchRecord
NIFI-3262: enable use of dynamic_templates in Elasticsearch _bulk operations
Signed-off-by: Joe Gresock <jg...@gmail.com>
This closes #6628.
---
.../nifi-elasticsearch-bundle/README.md | 24 ++
.../elasticsearch/ElasticSearchClientService.java | 15 +-
.../nifi/elasticsearch/IndexOperationRequest.java | 37 ++-
.../ElasticSearchClientServiceImpl.java | 113 ++++---
.../TestElasticSearchClientService.java | 5 +
.../integration/AbstractElasticsearch_IT.java | 2 +-
.../integration/ElasticSearchClientService_IT.java | 269 +++++++++++++----
.../src/test/resources/setup-6.script | 2 +-
.../src/test/resources/setup-7.script | 3 +-
.../src/test/resources/setup-8.script | 3 +-
.../AbstractByQueryElasticsearch.java | 2 +-
.../AbstractPaginatedJsonQueryElasticsearch.java | 2 +-
.../elasticsearch/AbstractPutElasticsearch.java | 30 +-
.../elasticsearch/ElasticsearchRestProcessor.java | 18 +-
.../processors/elasticsearch/GetElasticsearch.java | 15 +-
.../elasticsearch/JsonQueryElasticsearch.java | 2 +-
.../elasticsearch/PutElasticsearchJson.java | 195 ++++++++----
.../elasticsearch/PutElasticsearchRecord.java | 327 +++++++++++++++++----
.../elasticsearch/UpdateByQueryElasticsearch.java | 3 +-
.../additionalDetails.html | 87 +++++-
.../additionalDetails.html | 100 ++++++-
.../AbstractJsonQueryElasticsearchTest.groovy | 3 +-
.../elasticsearch/GetElasticsearchTest.groovy | 1 -
.../elasticsearch/PutElasticsearchJsonTest.groovy | 94 +++++-
.../PutElasticsearchRecordTest.groovy | 166 ++++++++++-
.../TestElasticsearchClientService.groovy | 8 +-
.../mock/AbstractMockElasticsearchClient.groovy | 5 +
.../integration/AbstractElasticsearchITBase.java | 36 +--
28 files changed, 1250 insertions(+), 317 deletions(-)
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/README.md b/nifi-nar-bundles/nifi-elasticsearch-bundle/README.md
index 3f3d139ff5..ba4983ec12 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/README.md
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/README.md
@@ -67,6 +67,30 @@ done
- [Elasticsearch Client Service](nifi-elasticsearch-client-service)
- [Elasticsearch REST API Processors](nifi-elasticsearch-restapi-processors)
+## Running on Mac
+
+Testcontainers support for "Mac OS X - Docker for Mac" is currently [best-efforts](https://www.testcontainers.org/supported_docker_environment/).
+
+It may be necessary to do the following to run the tests successfully:
+
+- Link the Docker Unix Socket to the standard location
+
+ ```sh
+ sudo ln -s $HOME/.docker/run/docker.sock /var/run/docker.sock
+ ```
+
+- Set the `TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE` environment variable
+
+ ```sh
+ export TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE=/var/run/docker.sock
+ ```
+
+## Running in IDEs
+
+- Edit "Run Configurations" to:
+ - enable Testcontainers via the system property, i.e. `-Delasticsearch.testcontainers.enabled=true`
+ - set the `TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE` environment variable if required
+
## Misc
Integration Tests with Testcontainers currently only uses the `amd64` Docker Images.
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java
index 76e5772eaf..54b3780dab 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java
@@ -38,7 +38,8 @@ public interface ElasticSearchClientService extends ControllerService, Verifiabl
PropertyDescriptor HTTP_HOSTS = new PropertyDescriptor.Builder()
.name("el-cs-http-hosts")
.displayName("HTTP Hosts")
- .description("A comma-separated list of HTTP hosts that host Elasticsearch query nodes.")
+ .description("A comma-separated list of HTTP hosts that host Elasticsearch query nodes. " +
+ "Note that the Host is included in requests as a header (typically including domain and port, e.g. elasticsearch:9200).")
.required(true)
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
@@ -245,11 +246,21 @@ public interface ElasticSearchClientService extends ControllerService, Verifiabl
/**
* Check whether an index exists.
*
- * @param index The index to target, if omitted then all indices will be updated.
+ * @param index The index to check.
* @param requestParameters A collection of URL request parameters. Optional.
*/
boolean exists(final String index, final Map<String, String> requestParameters);
+ /**
+ * Check whether a document exists.
+ *
+ * @param index The index that holds the document.
+ * @param type The document type. Optional. Will not be used in future versions of Elasticsearch.
+ * @param id The document ID
+ * @param requestParameters A collection of URL request parameters. Optional.
+ */
+ boolean documentExists(final String index, final String type, final String id, final Map<String, String> requestParameters);
+
/**
* Get a document by ID.
*
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/IndexOperationRequest.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/IndexOperationRequest.java
index 07cdb3cad1..22cc9873b0 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/IndexOperationRequest.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/IndexOperationRequest.java
@@ -31,13 +31,24 @@ public class IndexOperationRequest {
private final String id;
private final Map<String, Object> fields;
private final Operation operation;
+ private final Map<String, Object> script;
- public IndexOperationRequest(final String index, final String type, final String id, final Map<String, Object> fields, final Operation operation) {
+ private final boolean scriptedUpsert;
+ private final Map<String, Object> dynamicTemplates;
+ private final Map<String, String> headerFields;
+
+ public IndexOperationRequest(final String index, final String type, final String id, final Map<String, Object> fields,
+ final Operation operation, final Map<String, Object> script, final boolean scriptedUpsert,
+ final Map<String, Object> dynamicTemplates, final Map<String, String> headerFields) {
this.index = index;
this.type = type;
this.id = id;
this.fields = fields;
this.operation = operation;
+ this.script = script;
+ this.scriptedUpsert = scriptedUpsert;
+ this.dynamicTemplates = dynamicTemplates;
+ this.headerFields = headerFields;
}
public String getIndex() {
@@ -60,6 +71,22 @@ public class IndexOperationRequest {
return operation;
}
+ public Map<String, Object> getScript() {
+ return script;
+ }
+
+ public boolean isScriptedUpsert() {
+ return scriptedUpsert;
+ }
+
+ public Map<String, Object> getDynamicTemplates() {
+ return dynamicTemplates;
+ }
+
+ public Map<String, String> getHeaderFields() {
+ return headerFields;
+ }
+
public enum Operation {
Create("create"),
Delete("delete"),
@@ -81,10 +108,6 @@ public class IndexOperationRequest {
.filter(o -> o.getValue().equalsIgnoreCase(value)).findFirst()
.orElseThrow(() -> new IllegalArgumentException(String.format("Unknown Index Operation %s", value)));
}
-
- public static String[] allValues() {
- return Arrays.stream(Operation.values()).map(Operation::getValue).sorted().toArray(String[]::new);
- }
}
@Override
@@ -95,6 +118,10 @@ public class IndexOperationRequest {
", id='" + id + '\'' +
", fields=" + fields +
", operation=" + operation +
+ ", script=" + script +
+ ", scriptedUpsert=" + scriptedUpsert +
+ ", dynamicTemplates=" + dynamicTemplates +
+ ", headerFields=" + headerFields +
'}';
}
}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java
index ce0b200d2b..139eb4a805 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientServiceImpl.java
@@ -415,23 +415,28 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im
final String operation = request.getOperation().equals(IndexOperationRequest.Operation.Upsert)
? "update"
: request.getOperation().getValue();
- return buildBulkHeader(operation, request.getIndex(), request.getType(), request.getId());
+ return buildBulkHeader(operation, request.getIndex(), request.getType(), request.getId(), request.getDynamicTemplates(), request.getHeaderFields());
}
- private String buildBulkHeader(final String operation, final String index, final String type, final String id) throws JsonProcessingException {
- final Map<String, Object> header = new HashMap<String, Object>() {{
- put(operation, new HashMap<String, Object>() {{
- put("_index", index);
- if (StringUtils.isNotBlank(id)) {
- put("_id", id);
- }
- if (StringUtils.isNotBlank(type)) {
- put("_type", type);
- }
- }});
- }};
+ private String buildBulkHeader(final String operation, final String index, final String type, final String id,
+ final Map<String, Object> dynamicTemplates, final Map<String, String> headerFields) throws JsonProcessingException {
+ final Map<String, Object> operationBody = new HashMap<>();
+ operationBody.put("_index", index);
+ if (StringUtils.isNotBlank(id)) {
+ operationBody.put("_id", id);
+ }
+ if (StringUtils.isNotBlank(type)) {
+ operationBody.put("_type", type);
+ }
+ if (dynamicTemplates != null && !dynamicTemplates.isEmpty()) {
+ operationBody.put("dynamic_templates", dynamicTemplates);
+ }
+ if (headerFields != null && !headerFields.isEmpty()) {
+ headerFields.entrySet().stream().filter(e -> StringUtils.isNotBlank(e.getValue()))
+ .forEach(e -> operationBody.putIfAbsent(e.getKey(), e.getValue()));
+ }
- return flatten(mapper.writeValueAsString(header));
+ return flatten(mapper.writeValueAsString(Collections.singletonMap(operation, operationBody)));
}
protected void buildRequest(final IndexOperationRequest request, final StringBuilder builder) throws JsonProcessingException {
@@ -445,13 +450,21 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im
break;
case Update:
case Upsert:
- final Map<String, Object> doc = new HashMap<String, Object>() {{
- put("doc", request.getFields());
+ final Map<String, Object> updateBody = new HashMap<>(2, 1);
+ if (request.getScript() != null && !request.getScript().isEmpty()) {
+ updateBody.put("script", request.getScript());
+ if (request.getOperation().equals(IndexOperationRequest.Operation.Upsert)) {
+ updateBody.put("scripted_upsert", request.isScriptedUpsert());
+ updateBody.put("upsert", request.getFields());
+ }
+ } else {
+ updateBody.put("doc", request.getFields());
if (request.getOperation().equals(IndexOperationRequest.Operation.Upsert)) {
- put("doc_as_upsert", true);
+ updateBody.put("doc_as_upsert", true);
}
- }};
- final String update = flatten(mapper.writeValueAsString(doc)).trim();
+ }
+
+ final String update = flatten(mapper.writeValueAsString(updateBody)).trim();
builder.append(update).append("\n");
break;
case Delete:
@@ -510,7 +523,7 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im
try {
final StringBuilder sb = new StringBuilder();
for (final String id : ids) {
- final String header = buildBulkHeader("delete", index, type, id);
+ final String header = buildBulkHeader("delete", index, type, id, null, null);
sb.append(header).append("\n");
}
final HttpEntity entity = new NStringEntity(sb.toString(), ContentType.APPLICATION_JSON);
@@ -592,6 +605,23 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im
}
}
+ @Override
+ public boolean documentExists(final String index, final String type, final String id, final Map<String, String> requestParameters) {
+ boolean exists = true;
+ try {
+ final Map<String, String> existsParameters = requestParameters != null ? new HashMap<>(requestParameters) : new HashMap<>();
+ existsParameters.putIfAbsent("_source", "false");
+ get(index, type, id, existsParameters);
+ } catch (final ElasticsearchException ee) {
+ if (ee.isNotFound()) {
+ exists = false;
+ } else {
+ throw ee;
+ }
+ }
+ return exists;
+ }
+
@SuppressWarnings("unchecked")
@Override
public Map<String, Object> get(final String index, final String type, final String id, final Map<String, String> requestParameters) {
@@ -796,29 +826,28 @@ public class ElasticSearchClientServiceImpl extends AbstractControllerService im
}
if (entity != null) {
request.setEntity(entity);
- }
- if (getLogger().isDebugEnabled()) {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- entity.writeTo(out);
- out.close();
-
- StringBuilder builder = new StringBuilder(1000);
- builder.append("Dumping Elasticsearch REST request...\n")
- .append("HTTP Method: ")
- .append(method)
- .append("\n")
- .append("Endpoint: ")
- .append(endpoint)
- .append("\n")
- .append("Parameters: ")
- .append(prettyPrintWriter.writeValueAsString(parameters))
- .append("\n")
- .append("Request body: ")
- .append(new String(out.toByteArray()))
- .append("\n");
-
- getLogger().debug(builder.toString());
+ if (getLogger().isDebugEnabled()) {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ entity.writeTo(out);
+ out.close();
+
+ String builder = "Dumping Elasticsearch REST request...\n" +
+ "HTTP Method: " +
+ method +
+ "\n" +
+ "Endpoint: " +
+ endpoint +
+ "\n" +
+ "Parameters: " +
+ prettyPrintWriter.writeValueAsString(parameters) +
+ "\n" +
+ "Request body: " +
+ out +
+ "\n";
+
+ getLogger().debug(builder);
+ }
}
return client.performRequest(request);
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/TestElasticSearchClientService.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/TestElasticSearchClientService.java
index 4ee196a9dc..8b686fc9ed 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/TestElasticSearchClientService.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/TestElasticSearchClientService.java
@@ -87,6 +87,11 @@ public class TestElasticSearchClientService extends AbstractControllerService im
return true;
}
+ @Override
+ public boolean documentExists(String index, String type, String id, Map<String, String> requestParameters) {
+ return true;
+ }
+
@Override
public Map<String, Object> get(String index, String type, String id, Map<String, String> requestParameters) {
return data;
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/AbstractElasticsearch_IT.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/AbstractElasticsearch_IT.java
index 104e504b23..3d3187b3a3 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/AbstractElasticsearch_IT.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/AbstractElasticsearch_IT.java
@@ -31,7 +31,7 @@ abstract class AbstractElasticsearch_IT extends AbstractElasticsearchITBase {
static final List<String> TEST_INDICES =
Arrays.asList("user_details", "complex", "nested", "bulk_a", "bulk_b", "bulk_c", "error_handler", "messages");
- ElasticSearchClientServiceImpl service;
+ ElasticSearchClientService service;
@BeforeEach
void before() throws Exception {
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.java
index d7f0d39874..6aa6970488 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/java/org/apache/nifi/elasticsearch/integration/ElasticSearchClientService_IT.java
@@ -18,10 +18,14 @@
package org.apache.nifi.elasticsearch.integration;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.nio.entity.NStringEntity;
import org.apache.nifi.components.ConfigVerificationResult;
import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.controller.VerifiableControllerService;
import org.apache.nifi.elasticsearch.AuthorizationScheme;
import org.apache.nifi.elasticsearch.DeleteOperationResponse;
import org.apache.nifi.elasticsearch.ElasticSearchClientService;
@@ -39,11 +43,17 @@ import org.apache.nifi.ssl.StandardSSLContextService;
import org.apache.nifi.util.MockConfigurationContext;
import org.apache.nifi.util.MockControllerServiceLookup;
import org.apache.nifi.util.MockVariableRegistry;
+import org.apache.nifi.util.StringUtils;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -62,12 +72,13 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
@AfterEach
void after() throws Exception {
- service.onDisabled();
+ ((ElasticSearchClientServiceImpl) service).onDisabled();
}
private Map<PropertyDescriptor, String> getClientServiceProperties() {
@@ -77,7 +88,7 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
@Test
void testVerifySuccess() {
- final List<ConfigVerificationResult> results = ((VerifiableControllerService) service).verify(
+ final List<ConfigVerificationResult> results = service.verify(
new MockConfigurationContext(service, getClientServiceProperties(), runner.getProcessContext().getControllerServiceLookup(), new MockVariableRegistry()),
runner.getLogger(),
Collections.emptyMap()
@@ -88,7 +99,7 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
@Test
void testVerifySuccessWithApiKeyAuth() throws IOException {
- final Pair<String, String> apiKey = createApiKeyForIndex("*");
+ final Pair<String, String> apiKey = createApiKeyForIndex();
runner.disableControllerService(service);
runner.setProperty(service, ElasticSearchClientService.AUTHORIZATION_SCHEME, AuthorizationScheme.API_KEY.getValue());
@@ -98,7 +109,7 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
runner.setProperty(service, ElasticSearchClientService.API_KEY, apiKey.getValue());
runner.enableControllerService(service);
- final List<ConfigVerificationResult> results = ((VerifiableControllerService) service).verify(
+ final List<ConfigVerificationResult> results = service.verify(
new MockConfigurationContext(service, getClientServiceProperties(), runner.getProcessContext().getControllerServiceLookup(), new MockVariableRegistry()),
runner.getLogger(),
Collections.emptyMap()
@@ -112,7 +123,7 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
runner.disableControllerService(service);
runner.setProperty(service, ElasticSearchClientService.HTTP_HOSTS, "invalid");
- final List<ConfigVerificationResult> results = ((VerifiableControllerService) service).verify(
+ final List<ConfigVerificationResult> results = service.verify(
new MockConfigurationContext(service, getClientServiceProperties(), runner.getProcessContext().getControllerServiceLookup(), new MockVariableRegistry()),
runner.getLogger(),
Collections.emptyMap()
@@ -141,7 +152,7 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
// expected, ignore
}
- final List<ConfigVerificationResult> results = ((VerifiableControllerService) service).verify(
+ final List<ConfigVerificationResult> results = service.verify(
new MockConfigurationContext(service, getClientServiceProperties(), runner.getProcessContext().getControllerServiceLookup(), new MockVariableRegistry()),
runner.getLogger(),
Collections.emptyMap()
@@ -162,7 +173,7 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
runner.setProperty(service, ElasticSearchClientService.USERNAME, "invalid");
runner.setProperty(service, ElasticSearchClientService.PASSWORD, "not-real");
- final List<ConfigVerificationResult> results = ((VerifiableControllerService) service).verify(
+ final List<ConfigVerificationResult> results = service.verify(
new MockConfigurationContext(service, getClientServiceProperties(), runner.getProcessContext().getControllerServiceLookup(), new MockVariableRegistry()),
runner.getLogger(),
Collections.emptyMap()
@@ -188,7 +199,7 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
runner.setProperty(service, ElasticSearchClientService.API_KEY, "not-real");
runner.enableControllerService(service);
- final List<ConfigVerificationResult> results = ((VerifiableControllerService) service).verify(
+ final List<ConfigVerificationResult> results = service.verify(
new MockConfigurationContext(service, getClientServiceProperties(), runner.getProcessContext().getControllerServiceLookup(), new MockVariableRegistry()),
runner.getLogger(),
Collections.emptyMap()
@@ -583,7 +594,7 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
assertNotNull(doc);
} finally {
// replace the deleted doc
- service.add(new IndexOperationRequest(INDEX, type, "1", originalDoc, IndexOperationRequest.Operation.Index), null);
+ service.add(new IndexOperationRequest(INDEX, type, "1", originalDoc, IndexOperationRequest.Operation.Index, null, false, null, null), null);
waitForIndexRefresh(); // (affects later tests using _search or _bulk)
}
}
@@ -635,7 +646,12 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
// index with nulls
suppressNulls(false);
- IndexOperationResponse response = service.bulk(Collections.singletonList(new IndexOperationRequest("nulls", type, "1", doc, IndexOperationRequest.Operation.Index)), null);
+ IndexOperationResponse response = service.bulk(
+ Collections.singletonList(
+ new IndexOperationRequest("nulls", type, "1", doc, IndexOperationRequest.Operation.Index, null, false, null, null)
+ ),
+ null
+ );
assertNotNull(response);
assertTrue(response.getTook() > 0);
waitForIndexRefresh();
@@ -645,7 +661,7 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
// suppress nulls
suppressNulls(true);
- response = service.bulk(Collections.singletonList(new IndexOperationRequest("nulls", type, "2", doc, IndexOperationRequest.Operation.Index)), null);
+ response = service.bulk(Collections.singletonList(new IndexOperationRequest("nulls", type, "2", doc, IndexOperationRequest.Operation.Index, null, false, null, null)), null);
assertNotNull(response);
assertTrue(response.getTook() > 0);
waitForIndexRefresh();
@@ -675,12 +691,12 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
final String index = x % 2 == 0 ? "bulk_a": "bulk_b";
payload.add(new IndexOperationRequest(index, type, String.valueOf(x), new HashMap<String, Object>(){{
put("msg", "test");
- }}, IndexOperationRequest.Operation.Index));
+ }}, IndexOperationRequest.Operation.Index, null, false, null, null));
}
for (int x = 0; x < 5; x++) {
payload.add(new IndexOperationRequest("bulk_c", type, String.valueOf(x), new HashMap<String, Object>(){{
put("msg", "test");
- }}, IndexOperationRequest.Operation.Index));
+ }}, IndexOperationRequest.Operation.Index, null, false, null, null));
}
final IndexOperationResponse response = service.bulk(payload, createParameters("refresh", "true"));
assertNotNull(response);
@@ -703,27 +719,29 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
assertEquals(10, indexB.intValue());
assertEquals(5, indexC.intValue());
- final Long total = service.count(query, "bulk_*", type, null);
+ final Long total = service.count(query, "bulk_a,bulk_b,bulk_c", type, null);
assertNotNull(total);
assertEquals(25, total.intValue());
}
@Test
- void testBulkRequestParameters() {
+ void testBulkRequestParametersAndBulkHeaders() {
final List<IndexOperationRequest> payload = new ArrayList<>();
for (int x = 0; x < 20; x++) {
final String index = x % 2 == 0 ? "bulk_a": "bulk_b";
- payload.add(new IndexOperationRequest(index, type, String.valueOf(x), new MapBuilder().of("msg", "test").build(), IndexOperationRequest.Operation.Index));
+ payload.add(new IndexOperationRequest(index, type, String.valueOf(x), new MapBuilder().of("msg", "test").build(),
+ IndexOperationRequest.Operation.Index, null, false, null, Collections.singletonMap("retry_on_conflict", "3")));
}
for (int x = 0; x < 5; x++) {
- payload.add(new IndexOperationRequest("bulk_c", type, String.valueOf(x), new MapBuilder().of("msg", "test").build(), IndexOperationRequest.Operation.Index));
+ payload.add(new IndexOperationRequest("bulk_c", type, String.valueOf(x), new MapBuilder().of("msg", "test").build(),
+ IndexOperationRequest.Operation.Index, null, false, null, null));
}
final IndexOperationResponse response = service.bulk(payload, createParameters("refresh", "true"));
assertNotNull(response);
assertTrue(response.getTook() > 0);
/*
- * Now, check to ensure that both indexes got populated and refreshed appropriately.
+ * Now, check to ensure that all indices got populated and refreshed appropriately.
*/
final String query = "{ \"query\": { \"match_all\": {}}}";
final Long indexA = service.count(query, "bulk_a", type, null);
@@ -743,55 +761,141 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
assertEquals(25, total.intValue());
}
+ @Test
+ void testUnknownBulkHeader() {
+ final IndexOperationRequest failingRequest = new IndexOperationRequest("bulk_c", type, "1", new MapBuilder().of("msg", "test").build(),
+ IndexOperationRequest.Operation.Index, null, false, null, Collections.singletonMap("not_exist", "true"));
+ final ElasticsearchException ee = assertThrows(ElasticsearchException.class, () -> service.add(failingRequest, null));
+ assertInstanceOf(ResponseException.class, ee.getCause());
+ assertTrue(ee.getCause().getMessage().contains("Action/metadata line [1] contains an unknown parameter [not_exist]"));
+ }
+
+ @Test
+ void testDynamicTemplates() {
+ assumeFalse(getElasticMajorVersion() == 6, "Requires Elasticsearch > 6");
+
+ final List<IndexOperationRequest> payload = Collections.singletonList(
+ new IndexOperationRequest("dynamo", type, "1", new MapBuilder().of("msg", "test", "hello", "world").build(),
+ IndexOperationRequest.Operation.Index, null, false, new MapBuilder().of("hello", "test_text").build(), null)
+ );
+
+ final IndexOperationResponse response = service.bulk(payload, createParameters("refresh", "true"));
+ assertNotNull(response);
+ assertTrue(response.getTook() > 0);
+
+ /*
+ * Now, check the dynamic_template was applied
+ */
+ final Map<String, Object> fieldMapping = getIndexFieldMapping(type);
+ assertEquals(1, fieldMapping.size());
+ assertEquals("text", fieldMapping.get("type"));
+ }
+
@Test
void testUpdateAndUpsert() throws InterruptedException {
final String TEST_ID = "update-test";
- final Map<String, Object> doc = new HashMap<>();
- doc.put("msg", "Buongiorno, mondo");
- service.add(new IndexOperationRequest(INDEX, type, TEST_ID, doc, IndexOperationRequest.Operation.Index), createParameters("refresh", "true"));
- Map<String, Object> result = service.get(INDEX, type, TEST_ID, null);
- assertEquals(doc, result, "Not the same");
-
- final Map<String, Object> updates = new HashMap<>();
- updates.put("from", "john.smith");
- final Map<String, Object> merged = new HashMap<>();
- merged.putAll(updates);
- merged.putAll(doc);
- IndexOperationRequest request = new IndexOperationRequest(INDEX, type, TEST_ID, updates, IndexOperationRequest.Operation.Update);
- service.add(request, createParameters("refresh", "true"));
- result = service.get(INDEX, type, TEST_ID, null);
- assertTrue(result.containsKey("from"));
- assertTrue(result.containsKey("msg"));
- assertEquals(merged, result, "Not the same after update.");
-
final String UPSERTED_ID = "upsert-ftw";
- final Map<String, Object> upsertItems = new HashMap<>();
- upsertItems.put("upsert_1", "hello");
- upsertItems.put("upsert_2", 1);
- upsertItems.put("upsert_3", true);
- request = new IndexOperationRequest(INDEX, type, UPSERTED_ID, upsertItems, IndexOperationRequest.Operation.Upsert);
- service.add(request, createParameters("refresh", "true"));
- result = service.get(INDEX, type, UPSERTED_ID, null);
- assertEquals(upsertItems, result);
-
- final List<IndexOperationRequest> deletes = new ArrayList<>();
- deletes.add(new IndexOperationRequest(INDEX, type, TEST_ID, null, IndexOperationRequest.Operation.Delete));
- deletes.add(new IndexOperationRequest(INDEX, type, UPSERTED_ID, null, IndexOperationRequest.Operation.Delete));
- assertFalse(service.bulk(deletes, createParameters("refresh", "true")).hasErrors());
- waitForIndexRefresh(); // wait 1s for index refresh (doesn't prevent GET but affects later tests using _search or _bulk)
- ElasticsearchException ee = assertThrows(ElasticsearchException.class, () -> service.get(INDEX, type, TEST_ID, null) );
- assertTrue(ee.isNotFound());
- ee = assertThrows(ElasticsearchException.class, () -> service.get(INDEX, type, UPSERTED_ID, null));
- assertTrue(ee.isNotFound());
+ final String UPSERT_SCRIPT_ID = "upsert-script";
+ final String SCRIPTED_UPSERT_ID = "scripted-upsert-test";
+ try {
+ final Map<String, Object> doc = new HashMap<>();
+ doc.put("msg", "Buongiorno, mondo");
+ doc.put("counter", 1);
+ service.add(new IndexOperationRequest(INDEX, type, TEST_ID, doc, IndexOperationRequest.Operation.Index, null, false, null, null), createParameters("refresh", "true"));
+ Map<String, Object> result = service.get(INDEX, type, TEST_ID, null);
+ assertEquals(doc, result, "Not the same");
+
+ final Map<String, Object> updates = new HashMap<>();
+ updates.put("from", "john.smith");
+ final Map<String, Object> merged = new HashMap<>();
+ merged.putAll(updates);
+ merged.putAll(doc);
+ IndexOperationRequest request = new IndexOperationRequest(INDEX, type, TEST_ID, updates, IndexOperationRequest.Operation.Update, null, false, null, null);
+ service.add(request, createParameters("refresh", "true"));
+ result = service.get(INDEX, type, TEST_ID, null);
+ assertTrue(result.containsKey("from"));
+ assertTrue(result.containsKey("counter"));
+ assertTrue(result.containsKey("msg"));
+ assertEquals(merged, result, "Not the same after update.");
+
+ final Map<String, Object> upsertItems = new HashMap<>();
+ upsertItems.put("upsert_1", "hello");
+ upsertItems.put("upsert_2", 1);
+ upsertItems.put("upsert_3", true);
+ request = new IndexOperationRequest(INDEX, type, UPSERTED_ID, upsertItems, IndexOperationRequest.Operation.Upsert, null, false, null, null);
+ service.add(request, createParameters("refresh", "true"));
+ result = service.get(INDEX, type, UPSERTED_ID, null);
+ assertEquals(upsertItems, result);
+
+ final Map<String, Object> upsertDoc = new HashMap<>();
+ upsertDoc.put("msg", "Only if doesn't already exist");
+ final Map<String, Object> script = new HashMap<>();
+ script.put("source", "ctx._source.counter += params.count");
+ script.put("lang", "painless");
+ script.put("params", Collections.singletonMap("count", 2));
+ // apply script to existing document
+ request = new IndexOperationRequest(INDEX, type, TEST_ID, upsertDoc, IndexOperationRequest.Operation.Upsert, script, false, null, null);
+ service.add(request, createParameters("refresh", "true"));
+ result = service.get(INDEX, type, TEST_ID, null);
+ assertEquals(doc.get("msg"), result.get("msg"));
+ assertEquals(3, result.get("counter"));
+ // index document that doesn't already exist (don't apply script)
+ request = new IndexOperationRequest(INDEX, type, UPSERT_SCRIPT_ID, upsertDoc, IndexOperationRequest.Operation.Upsert, script, false, null, null);
+ service.add(request, createParameters("refresh", "true"));
+ result = service.get(INDEX, type, UPSERT_SCRIPT_ID, null);
+ assertNull(result.get("counter"));
+ assertEquals(upsertDoc, result);
+
+ final Map<String, Object> emptyUpsertDoc = Collections.emptyMap();
+ final Map<String, Object> upsertScript = new HashMap<>();
+ upsertScript.put("source", "if ( ctx.op == 'create' ) { ctx._source.counter = params.count } else { ctx._source.counter += params.count }");
+ upsertScript.put("lang", "painless");
+ upsertScript.put("params", Collections.singletonMap("count", 2));
+ // no script execution if doc found (without scripted_upsert)
+ request = new IndexOperationRequest(INDEX, type, SCRIPTED_UPSERT_ID, emptyUpsertDoc, IndexOperationRequest.Operation.Upsert, upsertScript, false, null, null);
+ service.add(request, createParameters("refresh", "true"));
+ assertFalse(service.documentExists(INDEX, type, SCRIPTED_UPSERT_ID, null));
+ // script execution with no doc found (with scripted_upsert) - doc not create, no "upsert" doc provided (empty objects suppressed)
+ suppressNulls(true);
+ request = new IndexOperationRequest(INDEX, type, SCRIPTED_UPSERT_ID, emptyUpsertDoc, IndexOperationRequest.Operation.Upsert, upsertScript, true, null, null);
+ service.add(request, createParameters("refresh", "true"));
+ assertFalse(service.documentExists(INDEX, type, SCRIPTED_UPSERT_ID, null));
+ // script execution with no doc found (with scripted_upsert) - doc created, empty "upsert" doc provided
+ suppressNulls(false);
+ request = new IndexOperationRequest(INDEX, type, SCRIPTED_UPSERT_ID, emptyUpsertDoc, IndexOperationRequest.Operation.Upsert, upsertScript, true, null, null);
+ service.add(request, createParameters("refresh", "true"));
+ result = service.get(INDEX, type, SCRIPTED_UPSERT_ID, null);
+ assertEquals(2, result.get("counter"));
+ // script execution with no doc found (with scripted_upsert) - doc updated
+ request = new IndexOperationRequest(INDEX, type, SCRIPTED_UPSERT_ID, emptyUpsertDoc, IndexOperationRequest.Operation.Upsert, upsertScript, true, null, null);
+ service.add(request, createParameters("refresh", "true"));
+ result = service.get(INDEX, type, SCRIPTED_UPSERT_ID, null);
+ assertEquals(4, result.get("counter"));
+ } finally {
+ final List<IndexOperationRequest> deletes = new ArrayList<>();
+ deletes.add(new IndexOperationRequest(INDEX, type, TEST_ID, null, IndexOperationRequest.Operation.Delete, null, false, null, null));
+ deletes.add(new IndexOperationRequest(INDEX, type, UPSERTED_ID, null, IndexOperationRequest.Operation.Delete, null, false, null, null));
+ deletes.add(new IndexOperationRequest(INDEX, type, UPSERT_SCRIPT_ID, null, IndexOperationRequest.Operation.Delete, null, false, null, null));
+ deletes.add(new IndexOperationRequest(INDEX, type, SCRIPTED_UPSERT_ID, null, IndexOperationRequest.Operation.Delete, null, false, null, null));
+ assertFalse(service.bulk(deletes, createParameters("refresh", "true")).hasErrors());
+ waitForIndexRefresh(); // wait 1s for index refresh (doesn't prevent GET but affects later tests using _search or _bulk)
+ assertFalse(service.documentExists(INDEX, type, TEST_ID, null));
+ assertFalse(service.documentExists(INDEX, type, UPSERTED_ID, null));
+ assertFalse(service.documentExists(INDEX, type, UPSERT_SCRIPT_ID, null));
+ assertFalse(service.documentExists(INDEX, type, SCRIPTED_UPSERT_ID, null));
+ }
}
@SuppressWarnings("unchecked")
@Test
void testGetBulkResponsesWithErrors() {
final List<IndexOperationRequest> ops = Arrays.asList(
- new IndexOperationRequest(INDEX, type, "1", new MapBuilder().of("msg", "one", "intField", 1).build(), IndexOperationRequest.Operation.Index), // OK
- new IndexOperationRequest(INDEX, type, "2", new MapBuilder().of("msg", "two", "intField", 1).build(), IndexOperationRequest.Operation.Create), // already exists
- new IndexOperationRequest(INDEX, type, "1", new MapBuilder().of("msg", "one", "intField", "notaninteger").build(), IndexOperationRequest.Operation.Index) // can't parse int field
+ new IndexOperationRequest(INDEX, type, "1", new MapBuilder().of("msg", "one", "intField", 1).build(),
+ IndexOperationRequest.Operation.Index, null, false, null, null), // OK
+ new IndexOperationRequest(INDEX, type, "2", new MapBuilder().of("msg", "two", "intField", 1).build(),
+ IndexOperationRequest.Operation.Create, null, false, null, null), // already exists
+ new IndexOperationRequest(INDEX, type, "1", new MapBuilder().of("msg", "one", "intField", "notaninteger").build(),
+ IndexOperationRequest.Operation.Index, null, false, null, null) // can't parse int field
);
final IndexOperationResponse response = service.bulk(ops, createParameters("refresh", "true"));
assertTrue(response.hasErrors());
@@ -823,4 +927,53 @@ class ElasticSearchClientService_IT extends AbstractElasticsearch_IT {
Thread.sleep(1000);
}
+ protected Pair<String, String> createApiKeyForIndex() throws IOException {
+ final String body = prettyJson(new MapBuilder()
+ .of("name", "test-api-key")
+ .of("role_descriptors", new MapBuilder()
+ .of("test-role", new MapBuilder()
+ .of("cluster", Collections.singletonList("all"))
+ .of("index", Collections.singletonList(new MapBuilder()
+ .of("names", Collections.singletonList("*"))
+ .of("privileges", Collections.singletonList("all"))
+ .build()))
+ .build())
+ .build())
+ .build());
+ final String endpoint = String.format("%s/%s", elasticsearchHost, "_security/api_key");
+ final Request request = new Request("POST", endpoint);
+ final HttpEntity jsonBody = new NStringEntity(body, ContentType.APPLICATION_JSON);
+ request.setEntity(jsonBody);
+
+ final Response response = testDataManagementClient.performRequest(request);
+ final InputStream inputStream = response.getEntity().getContent();
+ final byte[] result = IOUtils.toByteArray(inputStream);
+ inputStream.close();
+ final Map<String, String> ret = MAPPER.readValue(new String(result, StandardCharsets.UTF_8), new TypeReference<Map<String, String>>() {});
+ return Pair.of(ret.get("id"), ret.get("api_key"));
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, Object> getIndexFieldMapping(final String type) {
+ final String index = "dynamo";
+ final String field = "hello";
+
+ final Request request = new Request("GET",
+ String.format("%s/%s/_mapping%s/field/%s", elasticsearchHost, index, StringUtils.isBlank(type) ? "" : "/" + type, field)
+ );
+ try {
+ final Response response = testDataManagementClient.performRequest(request);
+ final InputStream inputStream = response.getEntity().getContent();
+ final byte[] result = IOUtils.toByteArray(inputStream);
+ inputStream.close();
+ final Map<String, Object> parsed = (Map<String, Object>) MAPPER.readValue(new String(result, StandardCharsets.UTF_8), Map.class);
+ final Map<String, Object> parsedIndex = (Map<String, Object>) parsed.get(index);
+ final Map<String, Object> parsedMappings = (Map<String, Object>) (StringUtils.isBlank(type)
+ ? parsedIndex.get("mappings")
+ : ((Map<String, Object>) parsedIndex.get("mappings")).get(type));
+ return (Map<String, Object>) ((Map<String, Object>) ((Map<String, Object>) parsedMappings.get(field)).get("mapping")).get(field);
+ } catch (final IOException ioe) {
+ throw new IllegalStateException(String.format("Error getting field mappings %s [%s]", index, field), ioe);
+ }
+ }
}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script
index 510498ebab..2024dadb42 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-6.script
@@ -15,7 +15,7 @@
#create mapping
PUT:user_details/:{ "mappings":{"_doc":{ "properties":{ "email":{"type":"keyword"},"phone":{"type": "keyword"},"accessKey":{"type": "keyword"}}}}}
-PUT:messages/:{ "mappings":{"_doc":{ "properties":{ "msg":{"type":"keyword"}}}}}
+PUT:messages/:{ "mappings":{"_doc":{ "properties":{ "msg":{"type":"keyword"}, "counter":{"type":"short"}}}}}
PUT:complex/:{"mappings":{"_doc":{"properties":{"msg":{"type":"keyword"},"subField":{"type":"nested","properties":{"longField":{"type":"long"},"dateField":{"type":"date"}}}}}}}
PUT:nested/:{"mappings":{"_doc":{"properties":{"msg":{"type":"keyword"},"subField":{"type":"nested","properties":{"longField":{"type":"long"},"dateField":{"type":"date"},"deeper":{"type":"nested","properties":{"secretz":{"type":"keyword"},"deepest":{"type":"nested","properties":{"super_secret":{"type":"keyword"}}}}}}}}}}}
PUT:bulk_a/:{ "mappings":{"_doc":{ "properties":{ "msg":{"type":"keyword"}}}}}
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script
index 36e04def1d..64e4c85a48 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-7.script
@@ -15,12 +15,13 @@
#create mapping
PUT:user_details/:{ "mappings":{ "properties":{ "email":{"type":"keyword"},"phone":{"type": "keyword"},"accessKey":{"type": "keyword"}}}}
-PUT:messages/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
+PUT:messages/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}, "counter":{"type":"short"}}}}
PUT:complex/:{"mappings":{ "properties":{"msg":{"type":"keyword"},"subField":{"type":"nested","properties":{"longField":{"type":"long"},"dateField":{"type":"date"}}}}}}
PUT:nested/:{"mappings":{ "properties":{"msg":{"type":"keyword"},"subField":{"type":"nested","properties":{"longField":{"type":"long"},"dateField":{"type":"date"},"deeper":{"type":"nested","properties":{"secretz":{"type":"keyword"},"deepest":{"type":"nested","properties":{"super_secret":{"type":"keyword"}}}}}}}}}}
PUT:bulk_a/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
PUT:bulk_b/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
PUT:bulk_c/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
+PUT:dynamo/:{ "mappings":{ "dynamic_templates": [{ "test_text": { "mapping": { "type": "text" } } }], "properties":{ "msg":{"type":"keyword"}}}}
PUT:error_handler:{ "mappings": { "properties": { "msg": { "type": "keyword" }, "intField": { "type": "integer" }}}}
PUT:no_source/:{ "mappings":{ "_source": { "enabled": false }, "properties":{ "msg":{"type":"keyword"}}}}
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script
index 36e04def1d..64e4c85a48 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/src/test/resources/setup-8.script
@@ -15,12 +15,13 @@
#create mapping
PUT:user_details/:{ "mappings":{ "properties":{ "email":{"type":"keyword"},"phone":{"type": "keyword"},"accessKey":{"type": "keyword"}}}}
-PUT:messages/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
+PUT:messages/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}, "counter":{"type":"short"}}}}
PUT:complex/:{"mappings":{ "properties":{"msg":{"type":"keyword"},"subField":{"type":"nested","properties":{"longField":{"type":"long"},"dateField":{"type":"date"}}}}}}
PUT:nested/:{"mappings":{ "properties":{"msg":{"type":"keyword"},"subField":{"type":"nested","properties":{"longField":{"type":"long"},"dateField":{"type":"date"},"deeper":{"type":"nested","properties":{"secretz":{"type":"keyword"},"deepest":{"type":"nested","properties":{"super_secret":{"type":"keyword"}}}}}}}}}}
PUT:bulk_a/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
PUT:bulk_b/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
PUT:bulk_c/:{ "mappings":{ "properties":{ "msg":{"type":"keyword"}}}}
+PUT:dynamo/:{ "mappings":{ "dynamic_templates": [{ "test_text": { "mapping": { "type": "text" } } }], "properties":{ "msg":{"type":"keyword"}}}}
PUT:error_handler:{ "mappings": { "properties": { "msg": { "type": "keyword" }, "intField": { "type": "integer" }}}}
PUT:no_source/:{ "mappings":{ "_source": { "enabled": false }, "properties":{ "msg":{"type":"keyword"}}}}
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractByQueryElasticsearch.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractByQueryElasticsearch.java
index 5d0e5031f1..41ae54fe70 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractByQueryElasticsearch.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractByQueryElasticsearch.java
@@ -137,7 +137,7 @@ public abstract class AbstractByQueryElasticsearch extends AbstractProcessor imp
? context.getProperty(QUERY_ATTRIBUTE).evaluateAttributeExpressions(input).getValue()
: null;
- final OperationResponse or = performOperation(clientService.get(), query, index, type, getUrlQueryParameters(context, input));
+ final OperationResponse or = performOperation(clientService.get(), query, index, type, getDynamicProperties(context, input));
if (input == null) {
input = session.create();
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractPaginatedJsonQueryElasticsearch.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractPaginatedJsonQueryElasticsearch.java
index 42ef9416fb..01e5994c47 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractPaginatedJsonQueryElasticsearch.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractPaginatedJsonQueryElasticsearch.java
@@ -120,7 +120,7 @@ public abstract class AbstractPaginatedJsonQueryElasticsearch extends AbstractJs
if (!newQuery && paginationType == PaginationType.SCROLL) {
response = clientService.get().scroll(queryJson);
} else {
- final Map<String, String> requestParameters = getUrlQueryParameters(context, input);
+ final Map<String, String> requestParameters = getDynamicProperties(context, input);
if (paginationType == PaginationType.SCROLL) {
requestParameters.put("scroll", paginatedJsonQueryParameters.getKeepAlive());
}
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractPutElasticsearch.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractPutElasticsearch.java
index 0ab0a504f7..9ad90cbd02 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractPutElasticsearch.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/AbstractPutElasticsearch.java
@@ -35,6 +35,7 @@ import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.validation.RecordPathValidator;
import java.io.IOException;
import java.io.OutputStream;
@@ -49,6 +50,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Predicate;
+import java.util.stream.Collectors;
public abstract class AbstractPutElasticsearch extends AbstractProcessor implements ElasticsearchRestProcessor {
static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder()
@@ -104,6 +106,10 @@ public abstract class AbstractPutElasticsearch extends AbstractProcessor impleme
private final AtomicReference<Set<Relationship>> relationships = new AtomicReference<>(getBaseRelationships());
+ static final String BULK_HEADER_PREFIX = "BULK:";
+
+ static final ObjectMapper MAPPER = new ObjectMapper();
+
boolean logErrors;
boolean outputErrorResponses;
boolean notFoundIsSuccessful;
@@ -120,13 +126,19 @@ public abstract class AbstractPutElasticsearch extends AbstractProcessor impleme
@Override
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
- return new PropertyDescriptor.Builder()
+ final PropertyDescriptor.Builder builder = new PropertyDescriptor.Builder()
.name(propertyDescriptorName)
.required(false)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .dynamic(true)
- .build();
+ .dynamic(true);
+
+ if (propertyDescriptorName.startsWith(BULK_HEADER_PREFIX)) {
+ builder.addValidator(new RecordPathValidator());
+ } else {
+ builder.addValidator(StandardValidators.NON_EMPTY_VALIDATOR);
+ }
+
+ return builder.build();
}
@Override
@@ -189,6 +201,16 @@ public abstract class AbstractPutElasticsearch extends AbstractProcessor impleme
return validationResults;
}
+ Map<String, String> getRequestURLParameters(final Map<String, String> dynamicProperties) {
+ return dynamicProperties.entrySet().stream().filter(e -> !e.getKey().startsWith(BULK_HEADER_PREFIX))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ }
+
+ Map<String, String> getBulkHeaderParameters(final Map<String, String> dynamicProperties) {
+ return dynamicProperties.entrySet().stream().filter(e -> e.getKey().startsWith(BULK_HEADER_PREFIX))
+ .collect(Collectors.toMap(e -> e.getKey().replace(BULK_HEADER_PREFIX, "").trim(), Map.Entry::getValue));
+ }
+
void transferFlowFilesOnException(final Exception ex, final Relationship rel, final ProcessSession session,
final boolean penalize, final FlowFile... flowFiles) {
for (FlowFile flowFile : flowFiles) {
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/ElasticsearchRestProcessor.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/ElasticsearchRestProcessor.java
index a2e22f8382..bce344869c 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/ElasticsearchRestProcessor.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/ElasticsearchRestProcessor.java
@@ -30,6 +30,7 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.VerifiableProcessor;
import org.apache.nifi.processor.util.JsonValidator;
import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.util.StringUtils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -123,15 +124,18 @@ public interface ElasticsearchRestProcessor extends VerifiableProcessor {
return retVal;
}
- default Map<String, String> getUrlQueryParameters(final ProcessContext context, final FlowFile flowFile) {
- return getUrlQueryParameters(context, flowFile != null ? flowFile.getAttributes() : null);
+ default Map<String, String> getDynamicProperties(final ProcessContext context, final FlowFile flowFile) {
+ return getDynamicProperties(context, flowFile != null ? flowFile.getAttributes() : null);
}
- default Map<String, String> getUrlQueryParameters(final ProcessContext context, final Map<String, String> attributes) {
+ default Map<String, String> getDynamicProperties(final ProcessContext context, final Map<String, String> attributes) {
return context.getProperties().entrySet().stream()
- // filter non-null dynamic properties
- .filter(e -> e.getKey().isDynamic() && e.getValue() != null)
- // convert to Map of URL parameter keys and values
+ // filter non-blank dynamic properties
+ .filter(e -> e.getKey().isDynamic()
+ && StringUtils.isNotBlank(e.getValue())
+ && StringUtils.isNotBlank(context.getProperty(e.getKey()).evaluateAttributeExpressions(attributes).getValue())
+ )
+ // convert to Map keys and evaluated property values
.collect(Collectors.toMap(
e -> e.getKey().getName(),
e -> context.getProperty(e.getKey()).evaluateAttributeExpressions(attributes).getValue()
@@ -152,7 +156,7 @@ public interface ElasticsearchRestProcessor extends VerifiableProcessor {
if (context.getProperty(INDEX).isSet()) {
index = context.getProperty(INDEX).evaluateAttributeExpressions(attributes).getValue();
try {
- if (verifyClientService.exists(index, getUrlQueryParameters(context, attributes))) {
+ if (verifyClientService.exists(index, getDynamicProperties(context, attributes))) {
indexExistsResult.outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
.explanation(String.format("Index [%s] exists", index));
indexExists = true;
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/GetElasticsearch.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/GetElasticsearch.java
index af3f85fe16..13f578fa9b 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/GetElasticsearch.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/GetElasticsearch.java
@@ -180,17 +180,14 @@ public class GetElasticsearch extends AbstractProcessor implements Elasticsearch
final String type = context.getProperty(TYPE).evaluateAttributeExpressions(attributes).getValue();
final String id = context.getProperty(ID).evaluateAttributeExpressions(attributes).getValue();
try {
- final Map<String, String> requestParameters = new HashMap<>(getUrlQueryParameters(context, attributes));
+ final Map<String, String> requestParameters = new HashMap<>(getDynamicProperties(context, attributes));
requestParameters.putIfAbsent("_source", "false");
- verifyClientService.get(index, type, id, requestParameters);
- documentExistsResult.outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
- .explanation(String.format("Document [%s] exists in index [%s]", id, index));
- } catch (final ElasticsearchException ee) {
- if (ee.isNotFound()) {
+ if (verifyClientService.documentExists(index, type, id, requestParameters)) {
documentExistsResult.outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
- .explanation(String.format("Document [%s] does not exist in index [%s]", id, index));
+ .explanation(String.format("Document [%s] exists in index [%s]", id, index));
} else {
- handleDocumentExistsCheckException(ee, documentExistsResult, verificationLogger, index, id);
+ documentExistsResult.outcome(ConfigVerificationResult.Outcome.SUCCESSFUL)
+ .explanation(String.format("Document [%s] does not exist in index [%s]", id, index));
}
} catch (final Exception ex) {
handleDocumentExistsCheckException(ex, documentExistsResult, verificationLogger, index, id);
@@ -233,7 +230,7 @@ public class GetElasticsearch extends AbstractProcessor implements Elasticsearch
}
final StopWatch stopWatch = new StopWatch(true);
- final Map<String, Object> doc = clientService.get().get(index, type, id, getUrlQueryParameters(context, input));
+ final Map<String, Object> doc = clientService.get().get(index, type, id, getDynamicProperties(context, input));
final Map<String, String> attributes = new HashMap<>(4, 1);
attributes.put("filename", id);
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/JsonQueryElasticsearch.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/JsonQueryElasticsearch.java
index d72f13cd27..0b7c49b54b 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/JsonQueryElasticsearch.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/JsonQueryElasticsearch.java
@@ -72,7 +72,7 @@ public class JsonQueryElasticsearch extends AbstractJsonQueryElasticsearch<JsonQ
queryJsonParameters.getQuery(),
queryJsonParameters.getIndex(),
queryJsonParameters.getType(),
- getUrlQueryParameters(context, input)
+ getDynamicProperties(context, input)
);
if (input != null) {
session.getProvenanceReporter().send(
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJson.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJson.java
index f17f015813..36ea292697 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJson.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchJson.java
@@ -20,6 +20,7 @@ package org.apache.nifi.processors.elasticsearch;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.IOUtils;
+import org.apache.nifi.annotation.behavior.DynamicProperties;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.SystemResource;
@@ -30,6 +31,7 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.elasticsearch.ElasticSearchClientService;
import org.apache.nifi.elasticsearch.ElasticsearchException;
import org.apache.nifi.elasticsearch.IndexOperationRequest;
import org.apache.nifi.elasticsearch.IndexOperationResponse;
@@ -38,6 +40,7 @@ import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.util.StringUtils;
@@ -61,13 +64,22 @@ import java.util.stream.Collectors;
description = "The error message if there is an issue parsing the FlowFile, sending the parsed document to Elasticsearch or parsing the Elasticsearch response"),
@WritesAttribute(attribute = "elasticsearch.bulk.error", description = "The _bulk response if there was an error during processing the document within Elasticsearch.")
})
-@DynamicProperty(
- name = "The name of a URL query parameter to add",
- value = "The value of the URL query parameter",
- expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES,
- description = "Adds the specified property name/value as a query parameter in the Elasticsearch URL used for processing. " +
- "These parameters will override any matching parameters in the _bulk request body. " +
- "If FlowFiles are batched, only the first FlowFile in the batch is used to evaluate property values.")
+@DynamicProperties({
+ @DynamicProperty(
+ name = "The name of the Bulk request header",
+ value = "The value of the Bulk request header",
+ expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES,
+ description = "Prefix: " + AbstractPutElasticsearch.BULK_HEADER_PREFIX +
+ " - adds the specified property name/value as a Bulk request header in the Elasticsearch Bulk API body used for processing. " +
+ "These parameters will override any matching parameters in the _bulk request body."),
+ @DynamicProperty(
+ name = "The name of a URL query parameter to add",
+ value = "The value of the URL query parameter",
+ expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES,
+ description = "Adds the specified property name/value as a query parameter in the Elasticsearch URL used for processing. " +
+ "These parameters will override any matching parameters in the _bulk request body. " +
+ "If FlowFiles are batched, only the first FlowFile in the batch is used to evaluate property values.")
+})
@SystemResourceConsideration(
resource = SystemResource.MEMORY,
description = "The Batch of FlowFiles will be stored in memory until the bulk operation is performed.")
@@ -88,6 +100,40 @@ public class PutElasticsearchJson extends AbstractPutElasticsearch {
.addValidator(StandardValidators.ATTRIBUTE_KEY_VALIDATOR)
.build();
+ static final PropertyDescriptor SCRIPT = new PropertyDescriptor.Builder()
+ .name("put-es-json-script")
+ .displayName("Script")
+ .description("The script for the document update/upsert. Only applies to Update/Upsert operations. " +
+ "Must be parsable as JSON Object. " +
+ "If left blank, the FlowFile content will be used for document update/upsert")
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
+ static final PropertyDescriptor SCRIPTED_UPSERT = new PropertyDescriptor.Builder()
+ .name("put-es-json-scripted-upsert")
+ .displayName("Scripted Upsert")
+ .description("Whether to add the scripted_upsert flag to the Upsert Operation. " +
+ "Forces Elasticsearch to execute the Script whether or not the document exists, defaults to false. " +
+ "If the Upsert Document provided (from FlowFile content) will be empty, but sure to set the " +
+ CLIENT_SERVICE.getDisplayName() + " controller service's " + ElasticSearchClientService.SUPPRESS_NULLS.getDisplayName() +
+ " to " + ElasticSearchClientService.NEVER_SUPPRESS.getDisplayName() + " or no \"upsert\" doc will be, " +
+ "included in the request to Elasticsearch and the operation will not create a new document for the script " +
+ "to execute against, resulting in a \"not_found\" error")
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .allowableValues("true", "false")
+ .defaultValue("false")
+ .build();
+
+ static final PropertyDescriptor DYNAMIC_TEMPLATES = new PropertyDescriptor.Builder()
+ .name("put-es-json-dynamic_templates")
+ .displayName("Dynamic Templates")
+ .description("The dynamic_templates for the document. Must be parsable as a JSON Object. " +
+ "Requires Elasticsearch 7+")
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
static final PropertyDescriptor CHARSET = new PropertyDescriptor.Builder()
.name("put-es-json-charset")
.displayName("Character Set")
@@ -125,11 +171,11 @@ public class PutElasticsearchJson extends AbstractPutElasticsearch {
.build();
static final List<PropertyDescriptor> DESCRIPTORS = Collections.unmodifiableList(Arrays.asList(
- ID_ATTRIBUTE, INDEX_OP, INDEX, TYPE, BATCH_SIZE, CHARSET, CLIENT_SERVICE,
+ ID_ATTRIBUTE, INDEX_OP, INDEX, TYPE, SCRIPT, SCRIPTED_UPSERT, DYNAMIC_TEMPLATES, BATCH_SIZE, CHARSET, CLIENT_SERVICE,
LOG_ERROR_RESPONSES, OUTPUT_ERROR_RESPONSES, OUTPUT_ERROR_DOCUMENTS, NOT_FOUND_IS_SUCCESSFUL
));
static final Set<Relationship> BASE_RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
- REL_SUCCESS, REL_FAILURE, REL_RETRY, REL_FAILED_DOCUMENTS
+ REL_SUCCESS, REL_FAILURE, REL_RETRY, REL_FAILED_DOCUMENTS
)));
private boolean outputErrors;
@@ -168,62 +214,14 @@ public class PutElasticsearchJson extends AbstractPutElasticsearch {
final List<FlowFile> originals = new ArrayList<>(flowFiles.size());
final List<IndexOperationRequest> operations = new ArrayList<>(flowFiles.size());
- for (FlowFile input : flowFiles) {
- final String indexOp = context.getProperty(INDEX_OP).evaluateAttributeExpressions(input).getValue();
- final String index = context.getProperty(INDEX).evaluateAttributeExpressions(input).getValue();
- final String type = context.getProperty(TYPE).evaluateAttributeExpressions(input).getValue();
- final String id = StringUtils.isNotBlank(idAttribute) && StringUtils.isNotBlank(input.getAttribute(idAttribute)) ? input.getAttribute(idAttribute) : null;
-
- final String charset = context.getProperty(CHARSET).evaluateAttributeExpressions(input).getValue();
-
- try (final InputStream inStream = session.read(input)) {
- final byte[] result = IOUtils.toByteArray(inStream);
- @SuppressWarnings("unchecked")
- final Map<String, Object> contentMap = objectMapper.readValue(new String(result, charset), Map.class);
-
- final IndexOperationRequest.Operation o = IndexOperationRequest.Operation.forValue(indexOp);
- operations.add(new IndexOperationRequest(index, type, id, contentMap, o));
-
- originals.add(input);
- } catch (final IOException ioe) {
- getLogger().error("Could not read FlowFile content valid JSON.", ioe);
- input = session.putAttribute(input, "elasticsearch.put.error", ioe.getMessage());
- session.penalize(input);
- session.transfer(input, REL_FAILURE);
- } catch (final Exception ex) {
- getLogger().error("Could not index documents.", ex);
- input = session.putAttribute(input, "elasticsearch.put.error", ex.getMessage());
- session.penalize(input);
- session.transfer(input, REL_FAILURE);
- }
+ for (final FlowFile input : flowFiles) {
+ addOperation(operations, originals, idAttribute, context, session, input);
}
if (!originals.isEmpty()) {
try {
final List<FlowFile> errorDocuments = indexDocuments(operations, originals, context, session);
- session.transfer(errorDocuments, REL_FAILED_DOCUMENTS);
- errorDocuments.forEach(e ->
- session.getProvenanceReporter().send(
- e,
- clientService.get().getTransitUrl(
- context.getProperty(INDEX).evaluateAttributeExpressions(e).getValue(),
- context.getProperty(TYPE).evaluateAttributeExpressions(e).getValue()
- ),
- "Elasticsearch _bulk operation error"
- )
- );
-
- final List<FlowFile> successfulDocuments = originals.stream().filter(f -> !errorDocuments.contains(f)).collect(Collectors.toList());
- session.transfer(successfulDocuments, REL_SUCCESS);
- successfulDocuments.forEach(s ->
- session.getProvenanceReporter().send(
- s,
- clientService.get().getTransitUrl(
- context.getProperty(INDEX).evaluateAttributeExpressions(s).getValue(),
- context.getProperty(TYPE).evaluateAttributeExpressions(s).getValue()
- )
- )
- );
+ handleResponse(context, session, errorDocuments, originals);
} catch (final ElasticsearchException ese) {
final String msg = String.format("Encountered a server-side problem with Elasticsearch. %s",
ese.isElastic() ? "Routing to retry." : "Routing to failure");
@@ -244,8 +242,57 @@ public class PutElasticsearchJson extends AbstractPutElasticsearch {
}
}
+ private void addOperation(final List<IndexOperationRequest> operations, final List<FlowFile> originals, final String idAttribute,
+ final ProcessContext context, final ProcessSession session, FlowFile input) {
+ final String indexOp = context.getProperty(INDEX_OP).evaluateAttributeExpressions(input).getValue();
+ final String index = context.getProperty(INDEX).evaluateAttributeExpressions(input).getValue();
+ final String type = context.getProperty(TYPE).evaluateAttributeExpressions(input).getValue();
+ final String id = StringUtils.isNotBlank(idAttribute) && StringUtils.isNotBlank(input.getAttribute(idAttribute)) ? input.getAttribute(idAttribute) : null;
+
+ final Map<String, Object> scriptMap = getMapFromAttribute(SCRIPT, context, input);
+ final boolean scriptedUpsert = context.getProperty(SCRIPTED_UPSERT).evaluateAttributeExpressions(input).asBoolean();
+ final Map<String, Object> dynamicTemplatesMap = getMapFromAttribute(DYNAMIC_TEMPLATES, context, input);
+
+ final Map<String, String> dynamicProperties = getDynamicProperties(context, input);
+ final Map<String, String> bulkHeaderFields = getBulkHeaderParameters(dynamicProperties);
+
+ final String charset = context.getProperty(CHARSET).evaluateAttributeExpressions(input).getValue();
+
+ try (final InputStream inStream = session.read(input)) {
+ final byte[] result = IOUtils.toByteArray(inStream);
+ @SuppressWarnings("unchecked")
+ final Map<String, Object> contentMap = objectMapper.readValue(new String(result, charset), Map.class);
+
+ final IndexOperationRequest.Operation o = IndexOperationRequest.Operation.forValue(indexOp);
+ operations.add(new IndexOperationRequest(index, type, id, contentMap, o, scriptMap, scriptedUpsert, dynamicTemplatesMap, bulkHeaderFields));
+
+ originals.add(input);
+ } catch (final IOException ioe) {
+ getLogger().error("Could not read FlowFile content valid JSON.", ioe);
+ input = session.putAttribute(input, "elasticsearch.put.error", ioe.getMessage());
+ session.penalize(input);
+ session.transfer(input, REL_FAILURE);
+ } catch (final Exception ex) {
+ getLogger().error("Could not index documents.", ex);
+ input = session.putAttribute(input, "elasticsearch.put.error", ex.getMessage());
+ session.penalize(input);
+ session.transfer(input, REL_FAILURE);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, Object> getMapFromAttribute(final PropertyDescriptor propertyDescriptor, final ProcessContext context, final FlowFile input) {
+ final String dynamicTemplates = context.getProperty(propertyDescriptor).evaluateAttributeExpressions(input).getValue();
+ try {
+ return StringUtils.isNotBlank(dynamicTemplates) ? MAPPER.readValue(dynamicTemplates, Map.class) : Collections.emptyMap();
+ } catch (final JsonProcessingException jpe) {
+ throw new ProcessException(propertyDescriptor.getDisplayName() + " must be a String parsable into a JSON Object", jpe);
+ }
+ }
+
private List<FlowFile> indexDocuments(final List<IndexOperationRequest> operations, final List<FlowFile> originals, final ProcessContext context, final ProcessSession session) throws IOException {
- final IndexOperationResponse response = clientService.get().bulk(operations, getUrlQueryParameters(context, originals.get(0)));
+ final Map<String, String> dynamicProperties = getDynamicProperties(context, originals.get(0));
+ final IndexOperationResponse response = clientService.get().bulk(operations, getRequestURLParameters(dynamicProperties));
final Map<Integer, Map<String, Object>> errors = findElasticsearchResponseErrors(response);
final List<FlowFile> errorDocuments = outputErrors ? new ArrayList<>(errors.size()) : Collections.emptyList();
@@ -270,4 +317,30 @@ public class PutElasticsearchJson extends AbstractPutElasticsearch {
return errorDocuments;
}
+
+ private void handleResponse(final ProcessContext context, final ProcessSession session, final List<FlowFile> errorDocuments, final List<FlowFile> originals) {
+ session.transfer(errorDocuments, REL_FAILED_DOCUMENTS);
+ errorDocuments.forEach(e ->
+ session.getProvenanceReporter().send(
+ e,
+ clientService.get().getTransitUrl(
+ context.getProperty(INDEX).evaluateAttributeExpressions(e).getValue(),
+ context.getProperty(TYPE).evaluateAttributeExpressions(e).getValue()
+ ),
+ "Elasticsearch _bulk operation error"
+ )
+ );
+
+ final List<FlowFile> successfulDocuments = originals.stream().filter(f -> !errorDocuments.contains(f)).collect(Collectors.toList());
+ session.transfer(successfulDocuments, REL_SUCCESS);
+ successfulDocuments.forEach(s ->
+ session.getProvenanceReporter().send(
+ s,
+ clientService.get().getTransitUrl(
+ context.getProperty(INDEX).evaluateAttributeExpressions(s).getValue(),
+ context.getProperty(TYPE).evaluateAttributeExpressions(s).getValue()
+ )
+ )
+ );
+ }
}
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecord.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecord.java
index 13fb557756..6801d4bff4 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecord.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecord.java
@@ -17,6 +17,8 @@
package org.apache.nifi.processors.elasticsearch;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.nifi.annotation.behavior.DynamicProperties;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.SystemResource;
@@ -28,6 +30,7 @@ import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.Validator;
+import org.apache.nifi.elasticsearch.ElasticSearchClientService;
import org.apache.nifi.elasticsearch.ElasticsearchException;
import org.apache.nifi.elasticsearch.IndexOperationRequest;
import org.apache.nifi.elasticsearch.IndexOperationResponse;
@@ -86,12 +89,21 @@ import java.util.concurrent.atomic.AtomicLong;
@WritesAttribute(attribute = "elasticsearch.put.error.count", description = "The number of records that generated errors in the Elasticsearch _bulk API."),
@WritesAttribute(attribute = "elasticsearch.put.success.count", description = "The number of records that were successfully processed by the Elasticsearch _bulk API.")
})
-@DynamicProperty(
- name = "The name of a URL query parameter to add",
- value = "The value of the URL query parameter",
- expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES,
- description = "Adds the specified property name/value as a query parameter in the Elasticsearch URL used for processing. " +
- "These parameters will override any matching parameters in the _bulk request body")
+@DynamicProperties({
+ @DynamicProperty(
+ name = "The name of the Bulk request header",
+ value = "The value of the Bulk request header",
+ expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES,
+ description = "Prefix: " + AbstractPutElasticsearch.BULK_HEADER_PREFIX +
+ " - adds the specified property name/value as a Bulk request header in the Elasticsearch Bulk API body used for processing. " +
+ "These parameters will override any matching parameters in the _bulk request body."),
+ @DynamicProperty(
+ name = "The name of a URL query parameter to add",
+ value = "The value of the URL query parameter",
+ expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES,
+ description = "Adds the specified property name/value as a query parameter in the Elasticsearch URL used for processing. " +
+ "These parameters will override any matching parameters in the _bulk request body")
+})
@SystemResourceConsideration(
resource = SystemResource.MEMORY,
description = "The Batch of Records will be stored in memory until the bulk operation is performed.")
@@ -190,6 +202,41 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch {
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
+ static final PropertyDescriptor SCRIPT_RECORD_PATH = new PropertyDescriptor.Builder()
+ .name("put-es-record-script-path")
+ .displayName("Script Record Path")
+ .description("A RecordPath pointing to a field in the record(s) that contains the script for the document update/upsert. " +
+ "Only applies to Update/Upsert operations. Field must be Map-type compatible (e.g. a Map or a Record) " +
+ "or a String parsable into a JSON Object")
+ .addValidator(new RecordPathValidator())
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .build();
+
+ static final PropertyDescriptor SCRIPTED_UPSERT_RECORD_PATH = new PropertyDescriptor.Builder()
+ .name("put-es-record-scripted-upsert-path")
+ .displayName("Scripted Upsert Record Path")
+ .description("A RecordPath pointing to a field in the record(s) that contains the scripted_upsert boolean flag. " +
+ "Whether to add the scripted_upsert flag to the Upsert Operation. " +
+ "Forces Elasticsearch to execute the Script whether or not the document exists, defaults to false. " +
+ "If the Upsert Document provided (from FlowFile content) will be empty, but sure to set the " +
+ CLIENT_SERVICE.getDisplayName() + " controller service's " + ElasticSearchClientService.SUPPRESS_NULLS.getDisplayName() +
+ " to " + ElasticSearchClientService.NEVER_SUPPRESS.getDisplayName() + " or no \"upsert\" doc will be, " +
+ "included in the request to Elasticsearch and the operation will not create a new document for the script " +
+ "to execute against, resulting in a \"not_found\" error")
+ .addValidator(new RecordPathValidator())
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .build();
+
+ static final PropertyDescriptor DYNAMIC_TEMPLATES_RECORD_PATH = new PropertyDescriptor.Builder()
+ .name("put-es-record-dynamic-templates-path")
+ .displayName("Dynamic Templates Record Path")
+ .description("A RecordPath pointing to a field in the record(s) that contains the dynamic_templates for the document. " +
+ "Field must be Map-type compatible (e.g. a Map or Record) or a String parsable into a JSON Object. " +
+ "Requires Elasticsearch 7+")
+ .addValidator(new RecordPathValidator())
+ .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .build();
+
static final PropertyDescriptor RETAIN_AT_TIMESTAMP_FIELD = new PropertyDescriptor.Builder()
.name("put-es-record-retain-at-timestamp-field")
.displayName("Retain @timestamp (Record Path)")
@@ -268,11 +315,11 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch {
static final List<PropertyDescriptor> DESCRIPTORS = Collections.unmodifiableList(Arrays.asList(
INDEX_OP, INDEX, TYPE, AT_TIMESTAMP, CLIENT_SERVICE, RECORD_READER, BATCH_SIZE, ID_RECORD_PATH, RETAIN_ID_FIELD,
INDEX_OP_RECORD_PATH, INDEX_RECORD_PATH, TYPE_RECORD_PATH, AT_TIMESTAMP_RECORD_PATH, RETAIN_AT_TIMESTAMP_FIELD,
- DATE_FORMAT, TIME_FORMAT, TIMESTAMP_FORMAT, LOG_ERROR_RESPONSES, OUTPUT_ERROR_RESPONSES, RESULT_RECORD_WRITER,
- NOT_FOUND_IS_SUCCESSFUL
+ SCRIPT_RECORD_PATH, SCRIPTED_UPSERT_RECORD_PATH, DYNAMIC_TEMPLATES_RECORD_PATH, DATE_FORMAT, TIME_FORMAT,
+ TIMESTAMP_FORMAT, LOG_ERROR_RESPONSES, OUTPUT_ERROR_RESPONSES, RESULT_RECORD_WRITER, NOT_FOUND_IS_SUCCESSFUL
));
static final Set<Relationship> BASE_RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
- REL_SUCCESS, REL_FAILURE, REL_RETRY, REL_FAILED_RECORDS, REL_SUCCESSFUL_RECORDS
+ REL_SUCCESS, REL_FAILURE, REL_RETRY, REL_FAILED_RECORDS, REL_SUCCESSFUL_RECORDS
)));
private RecordPathCache recordPathCache;
@@ -325,27 +372,8 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch {
return;
}
- final String indexOp = context.getProperty(INDEX_OP).evaluateAttributeExpressions(input).getValue();
- final String index = context.getProperty(INDEX).evaluateAttributeExpressions(input).getValue();
- final String type = context.getProperty(TYPE).evaluateAttributeExpressions(input).getValue();
- final String atTimestamp = context.getProperty(AT_TIMESTAMP).evaluateAttributeExpressions(input).getValue();
-
- final String indexOpPath = context.getProperty(INDEX_OP_RECORD_PATH).evaluateAttributeExpressions(input).getValue();
- final String idPath = context.getProperty(ID_RECORD_PATH).evaluateAttributeExpressions(input).getValue();
- final String indexPath = context.getProperty(INDEX_RECORD_PATH).evaluateAttributeExpressions(input).getValue();
- final String typePath = context.getProperty(TYPE_RECORD_PATH).evaluateAttributeExpressions(input).getValue();
- final String atTimestampPath = context.getProperty(AT_TIMESTAMP_RECORD_PATH).evaluateAttributeExpressions(input).getValue();
-
- final RecordPath ioPath = indexOpPath != null ? recordPathCache.getCompiled(indexOpPath) : null;
- final RecordPath path = StringUtils.isNotBlank(idPath) ? recordPathCache.getCompiled(idPath) : null;
- final RecordPath iPath = indexPath != null ? recordPathCache.getCompiled(indexPath) : null;
- final RecordPath tPath = typePath != null ? recordPathCache.getCompiled(typePath) : null;
- final RecordPath atPath = StringUtils.isNotBlank(atTimestampPath) ? recordPathCache.getCompiled(atTimestampPath) : null;
-
- final boolean retainId = context.getProperty(RETAIN_ID_FIELD).evaluateAttributeExpressions(input).asBoolean();
- final boolean retainTimestamp = context.getProperty(RETAIN_AT_TIMESTAMP_FIELD).evaluateAttributeExpressions(input).asBoolean();
+ final IndexOperationParameters indexOperationParameters = new IndexOperationParameters(context, input);
- final int batchSize = context.getProperty(BATCH_SIZE).evaluateAttributeExpressions(input).asInteger();
final List<FlowFile> resultRecords = new ArrayList<>();
final AtomicLong erroredRecords = new AtomicLong(0);
@@ -363,35 +391,17 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch {
Record record;
while ((record = recordSet.next()) != null) {
- final String idx = getFromRecordPath(record, iPath, index, false);
- indices.add(idx);
- final String t = getFromRecordPath(record, tPath, type, false);
- if (StringUtils.isNotBlank(t)) {
- types.add(t);
- }
- final IndexOperationRequest.Operation o = IndexOperationRequest.Operation.forValue(getFromRecordPath(record, ioPath, indexOp, false));
- final String id = getFromRecordPath(record, path, null, retainId);
- final Object timestamp = getTimestampFromRecordPath(record, atPath, atTimestamp, retainTimestamp);
-
- @SuppressWarnings("unchecked")
- final Map<String, Object> contentMap = (Map<String, Object>) DataTypeUtils
- .convertRecordFieldtoObject(record, RecordFieldType.RECORD.getRecordDataType(record.getSchema()));
- formatDateTimeFields(contentMap, record);
- if (timestamp != null) {
- contentMap.putIfAbsent("@timestamp", timestamp);
- }
-
- operationList.add(new IndexOperationRequest(idx, t, id, contentMap, o));
+ addOperation(operationList, record, indexOperationParameters, indices, types);
originals.add(record);
- if (operationList.size() == batchSize || !recordSet.isAnotherRecord()) {
- operate(operationList, originals, reader, context, session, input, resultRecords, erroredRecords, successfulRecords);
+ if (operationList.size() == indexOperationParameters.getBatchSize() || !recordSet.isAnotherRecord()) {
+ operate(operationList, originals, reader, session, input, indexOperationParameters.getRequestParameters(), resultRecords, erroredRecords, successfulRecords);
batches++;
}
}
if (!operationList.isEmpty()) {
- operate(operationList, originals, reader, context, session, input, resultRecords, erroredRecords, successfulRecords);
+ operate(operationList, originals, reader, session, input, indexOperationParameters.getRequestParameters(), resultRecords, erroredRecords, successfulRecords);
batches++;
}
} catch (final ElasticsearchException ese) {
@@ -431,13 +441,46 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch {
session.transfer(input, REL_SUCCESS);
}
+ private void addOperation(final List<IndexOperationRequest> operationList, final Record record, final IndexOperationParameters indexOperationParameters,
+ final Set<String> indices, final Set<String> types) {
+ final String index = getFromRecordPath(record, indexOperationParameters.getIndexPath(), true, indexOperationParameters.getDefaultIndex(), false);
+ indices.add(index);
+ final String type = getFromRecordPath(record, indexOperationParameters.getTypePath(), true, indexOperationParameters.getDefaultType(), false);
+ if (StringUtils.isNotBlank(type)) {
+ types.add(type);
+ }
+ final String op = getFromRecordPath(record, indexOperationParameters.getIndexOpPath(), true, indexOperationParameters.getDefaultIndexOp(), false);
+ final IndexOperationRequest.Operation indexOp = IndexOperationRequest.Operation.forValue(op);
+ final String id = getFromRecordPath(record, indexOperationParameters.getIdPath(), true, null, indexOperationParameters.isRetainId());
+ final Object atTimestamp = getTimestampFromRecordPath(record, indexOperationParameters.getAtTimestampPath(),
+ indexOperationParameters.getDefaultAtTimestamp(), indexOperationParameters.isRetainTimestamp());
+ final Map<String, Object> script = getMapFromRecordPath(record, indexOperationParameters.getScriptPath());
+ final boolean scriptedUpsert = Boolean.parseBoolean(getFromRecordPath(record, indexOperationParameters.getScriptedUpsertPath(), false, "false", false));
+ final Map<String, Object> dynamicTemplates = getMapFromRecordPath(record, indexOperationParameters.getDynamicTypesPath());
+
+ final Map<String, String> bulkHeaderFields = new HashMap<>(indexOperationParameters.getBulkHeaderRecordPaths().size(), 1);
+ for (final Map.Entry<String, RecordPath> entry : indexOperationParameters.getBulkHeaderRecordPaths().entrySet()) {
+ bulkHeaderFields.put(entry.getKey(), getFromRecordPath(record, entry.getValue(), false, null, false));
+ }
+
+ @SuppressWarnings("unchecked")
+ final Map<String, Object> contentMap = (Map<String, Object>) DataTypeUtils
+ .convertRecordFieldtoObject(record, RecordFieldType.RECORD.getRecordDataType(record.getSchema()));
+ formatDateTimeFields(contentMap, record);
+ if (atTimestamp != null) {
+ contentMap.putIfAbsent("@timestamp", atTimestamp);
+ }
+
+ operationList.add(new IndexOperationRequest(index, type, id, contentMap, indexOp, script, scriptedUpsert, dynamicTemplates, bulkHeaderFields));
+ }
+
private void operate(final List<IndexOperationRequest> operationList, final List<Record> originals, final RecordReader reader,
- final ProcessContext context, final ProcessSession session, final FlowFile input,
+ final ProcessSession session, final FlowFile input, final Map<String, String> requestParameters,
final List<FlowFile> resultRecords, final AtomicLong erroredRecords, final AtomicLong successfulRecords)
throws IOException, SchemaNotFoundException, MalformedRecordException {
final BulkOperation bundle = new BulkOperation(operationList, originals, reader.getSchema());
- final ResponseDetails responseDetails = indexDocuments(bundle, context, session, input);
+ final ResponseDetails responseDetails = indexDocuments(bundle, session, input, requestParameters);
if (responseDetails.getErrored() != null) {
resultRecords.add(responseDetails.getErrored());
@@ -461,8 +504,9 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch {
results.clear();
}
- private ResponseDetails indexDocuments(final BulkOperation bundle, final ProcessContext context, final ProcessSession session, final FlowFile input) throws IOException, SchemaNotFoundException {
- final IndexOperationResponse response = clientService.get().bulk(bundle.getOperationList(), getUrlQueryParameters(context, input));
+ private ResponseDetails indexDocuments(final BulkOperation bundle, final ProcessSession session, final FlowFile input,
+ final Map<String, String> requestParameters) throws IOException, SchemaNotFoundException {
+ final IndexOperationResponse response = clientService.get().bulk(bundle.getOperationList(), requestParameters);
final Map<Integer, Map<String, Object>> errors = findElasticsearchResponseErrors(response);
if (!errors.isEmpty()) {
@@ -541,19 +585,19 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch {
}
}
- private String getFromRecordPath(final Record record, final RecordPath path, final String fallback,
- final boolean retain) {
+ private String getFromRecordPath(final Record record, final RecordPath path, final boolean strictString,
+ final String fallback, final boolean retain) {
if (path == null) {
return fallback;
}
final RecordPathResult result = path.evaluate(record);
final Optional<FieldValue> value = result.getSelectedFields().findFirst();
- if (value.isPresent() && value.get().getValue() != null) {
+ if (value.isPresent() && value.get().getValue() != null && DataTypeUtils.isStringTypeCompatible(value.get().getValue())) {
final FieldValue fieldValue = value.get();
- if (!fieldValue.getField().getDataType().getFieldType().equals(RecordFieldType.STRING) ) {
+ if (strictString && !fieldValue.getField().getDataType().getFieldType().equals(RecordFieldType.STRING)) {
throw new ProcessException(
- String.format("Field referenced by %s must be a string.", path.getPath())
+ String.format("Field referenced by %s must be a string", path.getPath())
);
}
@@ -567,6 +611,38 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch {
}
}
+ @SuppressWarnings("unchecked")
+ private Map<String, Object> getMapFromRecordPath(final Record record, final RecordPath path) {
+ if (path == null) {
+ return Collections.emptyMap();
+ }
+
+ final RecordPathResult result = path.evaluate(record);
+ final Optional<FieldValue> value = result.getSelectedFields().findFirst();
+ if (value.isPresent() && value.get().getValue() != null) {
+ final FieldValue fieldValue = value.get();
+ final Map<String, Object> map;
+ if (DataTypeUtils.isMapTypeCompatible(fieldValue.getValue())) {
+ map = DataTypeUtils.toMap(fieldValue.getValue(), path.getPath());
+ } else {
+ try {
+ map = MAPPER.readValue(fieldValue.getValue().toString(), Map.class);
+ } catch (final JsonProcessingException jpe) {
+ getLogger().error("Unable to parse field {} as Map", path.getPath(), jpe);
+ throw new ProcessException(
+ String.format("Field referenced by %s must be Map-type compatible or a String parsable into a JSON Object", path.getPath())
+ );
+ }
+ }
+
+ fieldValue.updateValue(null);
+
+ return map;
+ } else {
+ return Collections.emptyMap();
+ }
+ }
+
private Object getTimestampFromRecordPath(final Record record, final RecordPath path, final String fallback,
final boolean retain) {
if (path == null) {
@@ -683,4 +759,135 @@ public class PutElasticsearchRecord extends AbstractPutElasticsearch {
return successCount;
}
}
+
+ private class IndexOperationParameters {
+ private final String defaultIndexOp;
+ private final String defaultIndex;
+ private final String defaultType;
+ private final String defaultAtTimestamp;
+
+ private final RecordPath indexOpPath;
+ private final RecordPath idPath;
+ private final RecordPath indexPath;
+ private final RecordPath typePath;
+ private final RecordPath atTimestampPath;
+ private final RecordPath scriptPath;
+
+ private final RecordPath scriptedUpsertPath;
+ private final RecordPath dynamicTypesPath;
+
+ private final Map<String, String> requestParameters;
+ private final Map<String, RecordPath> bulkHeaderRecordPaths;
+
+ private final boolean retainId;
+ private final boolean retainTimestamp;
+ private final int batchSize;
+
+ IndexOperationParameters(final ProcessContext context, final FlowFile input) {
+ defaultIndexOp = context.getProperty(INDEX_OP).evaluateAttributeExpressions(input).getValue();
+ defaultIndex = context.getProperty(INDEX).evaluateAttributeExpressions(input).getValue();
+ defaultType = context.getProperty(TYPE).evaluateAttributeExpressions(input).getValue();
+ defaultAtTimestamp = context.getProperty(AT_TIMESTAMP).evaluateAttributeExpressions(input).getValue();
+
+ indexOpPath = compileRecordPathFromProperty(context, INDEX_OP_RECORD_PATH, input);
+ idPath = compileRecordPathFromProperty(context, ID_RECORD_PATH, input);
+ indexPath = compileRecordPathFromProperty(context, INDEX_RECORD_PATH, input);
+ typePath = compileRecordPathFromProperty(context, TYPE_RECORD_PATH, input);
+ atTimestampPath = compileRecordPathFromProperty(context, AT_TIMESTAMP_RECORD_PATH, input);
+ scriptPath = compileRecordPathFromProperty(context, SCRIPT_RECORD_PATH, input);
+ scriptedUpsertPath = compileRecordPathFromProperty(context, SCRIPTED_UPSERT_RECORD_PATH, input);
+ dynamicTypesPath = compileRecordPathFromProperty(context, DYNAMIC_TEMPLATES_RECORD_PATH, input);
+
+ final Map<String, String> dynamicProperties = getDynamicProperties(context, input);
+ requestParameters = getRequestURLParameters(dynamicProperties);
+
+ final Map<String, String> bulkHeaderParameterPaths = getBulkHeaderParameters(dynamicProperties);
+ bulkHeaderRecordPaths = new HashMap<>(bulkHeaderParameterPaths.size(), 1);
+ for (final Map.Entry<String, String> entry : bulkHeaderParameterPaths.entrySet()) {
+ if (StringUtils.isNotBlank(entry.getValue())) {
+ final RecordPath rp = recordPathCache.getCompiled(entry.getValue());
+ if (rp != null) {
+ bulkHeaderRecordPaths.put(entry.getKey(), rp);
+ }
+ }
+ }
+
+ retainId = context.getProperty(RETAIN_ID_FIELD).evaluateAttributeExpressions(input).asBoolean();
+ retainTimestamp = context.getProperty(RETAIN_AT_TIMESTAMP_FIELD).evaluateAttributeExpressions(input).asBoolean();
+ batchSize = context.getProperty(BATCH_SIZE).evaluateAttributeExpressions(input).asInteger();
+ }
+
+ private RecordPath compileRecordPathFromProperty(final ProcessContext context, final PropertyDescriptor property, final FlowFile input) {
+ final String recordPathProperty = context.getProperty(property).evaluateAttributeExpressions(input).getValue();
+ return StringUtils.isNotBlank(recordPathProperty) ? recordPathCache.getCompiled(recordPathProperty) : null;
+ }
+
+ public String getDefaultIndexOp() {
+ return defaultIndexOp;
+ }
+
+ public String getDefaultIndex() {
+ return defaultIndex;
+ }
+
+ public String getDefaultType() {
+ return defaultType;
+ }
+
+ public String getDefaultAtTimestamp() {
+ return defaultAtTimestamp;
+ }
+
+ public RecordPath getIndexOpPath() {
+ return indexOpPath;
+ }
+
+ public RecordPath getIdPath() {
+ return idPath;
+ }
+
+ public RecordPath getIndexPath() {
+ return indexPath;
+ }
+
+ public RecordPath getTypePath() {
+ return typePath;
+ }
+
+ public RecordPath getAtTimestampPath() {
+ return atTimestampPath;
+ }
+
+ public RecordPath getScriptPath() {
+ return scriptPath;
+ }
+
+ public RecordPath getScriptedUpsertPath() {
+ return scriptedUpsertPath;
+ }
+
+ public RecordPath getDynamicTypesPath() {
+ return dynamicTypesPath;
+ }
+
+ public Map<String, String> getRequestParameters() {
+ return requestParameters;
+ }
+
+ public Map<String, RecordPath> getBulkHeaderRecordPaths() {
+ return bulkHeaderRecordPaths;
+ }
+
+ public boolean isRetainId() {
+ return retainId;
+ }
+
+ public boolean isRetainTimestamp() {
+ return retainTimestamp;
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+ }
}
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/UpdateByQueryElasticsearch.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/UpdateByQueryElasticsearch.java
index 77f5a4555c..87ddaffe37 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/UpdateByQueryElasticsearch.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/java/org/apache/nifi/processors/elasticsearch/UpdateByQueryElasticsearch.java
@@ -36,7 +36,8 @@ import java.util.Map;
@InputRequirement(InputRequirement.Requirement.INPUT_ALLOWED)
@Tags({ "elastic", "elasticsearch", "elasticsearch5", "elasticsearch6", "elasticsearch7", "elasticsearch8", "update", "query"})
@CapabilityDescription("Update documents in an Elasticsearch index using a query. The query can be loaded from a flowfile body " +
- "or from the Query parameter.")
+ "or from the Query parameter. The loaded Query can contain any JSON accepted by Elasticsearch's _update_by_query API, " +
+ "for example a \"query\" object to identify what documents are to be updated, plus a \"script\" to define the updates to perform.")
@DynamicProperty(
name = "The name of a URL query parameter to add",
value = "The value of the URL query parameter",
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/resources/docs/org.apache.nifi.processors.elasticsearch.PutElasticsearchJson/additionalDetails.html b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/resources/docs/org.apache.nifi.processors.elasticsearch.PutElasticsearchJson/additionalDetails.html
index 403cb7df8e..649f45b55b 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/resources/docs/org.apache.nifi.processors.elasticsearch.PutElasticsearchJson/additionalDetails.html
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/resources/docs/org.apache.nifi.processors.elasticsearch.PutElasticsearchJson/additionalDetails.html
@@ -38,7 +38,92 @@
<p>
The index, operation and (optional) type fields are configured with default values.
The ID (optional unless the operation is "index") can be set as an attribute on the FlowFile(s).
- The following is an example of a document exercising all of these features:
</p>
+
+<h3>Dynamic Templates</h3>
+<p>
+ Index and Create operations can use Dynamic Templates. The Dynamic Templates property must be parsable as a JSON object.
+</p>
+<h4>Example - Index with Dynamic Templates</h4>
+<pre>
+ {
+ "message": "Hello, world"
+ }
+</pre>
+<p>The Dynamic Templates property below would be parsable:</p>
+<pre>{"message": "keyword_lower"}</pre>
+<p>Would create Elasticsearch action:</p>
+<pre>
+ { "index" : {"_id" : "1", "_index" : "test", "dynamic_templates" : {"message" : "keyword_lower"}} }
+ { "doc" : {"message" : "Hello, world"} }
+</pre>
+
+<h3>Update/Upsert Scripts</h3>
+<p>
+ Update and Upsert operations can use a script. Scripts must contain all the elements required by Elasticsearch, e.g. source and lang.
+ The Script property must be parsable as a JSON object.
+</p>
+<p>
+ If a script is defined for an upset, the Flowfile content will be used as the upsert fields in the Elasticsearch action.
+ If no script is defined, the FlowFile content will be used as the update doc (or doc_as_upsert for upsert operations).
+</p>
+<h4>Example - Update without Script</h4>
+<pre>
+ {
+ "message": "Hello, world",
+ "from": "john.smith"
+ }
+</pre>
+<p>Would create Elasticsearch action:</p>
+<pre>
+ { "update" : {"_id" : "1", "_index" : "test"} }
+ { "doc" : {"message" : "Hello, world", "from" : "john.smith"} }
+</pre>
+<h4>Example - Upsert with Script</h4>
+<pre>
+ {
+ "counter": 1
+ }
+</pre>
+<p>The script property below would be parsable:</p>
+<pre>{"source": "ctx._source.counter += params.param1", "lang": "painless", "params": {"param1": 1}}</pre>
+<p>Would create Elasticsearch action:</p>
+<pre>
+ { "update" : {"_id" : "1", "_index" : "test"} }
+ { "script" : { "source": "ctx._source.counter += params.param1", "lang" : "painless", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
+</pre>
+
+<h3>Bulk Action Header Fields</h3>
+<p>
+ Dynamic Properties can be defined on the processor with <i>BULK:</i> prefixes.
+ Users must ensure that only known Bulk action fields are sent to Elasticsearch for the relevant index operation defined for the FlowFile,
+ Elasticsearch will reject invalid combinations of index operation and Bulk action fields.
+</p>
+<h4>Example - Update with Retry on Conflict</h4>
+<pre>
+ {
+ "message": "Hello, world",
+ "from": "john.smith"
+ }
+</pre>
+<p>With the Dynamic Property below:</p>
+<ul>
+ <li>BULK:retry_on_conflict = 3</li>
+</ul>
+<p>Would create Elasticsearch action:</p>
+<pre>
+ { "update" : {"_id" : "1", "_index" : "test", "retry_on_conflict" : "3"} }
+ { "doc" : {"message" : "Hello, world", "from" : "john.smith"} }
+</pre>
+
+<h3>Index Operations</h3>
+<p>Valid values for "operation" are:</p>
+<ul>
+ <li>create</li>
+ <li>delete</li>
+ <li>index</li>
+ <li>update</li>
+ <li>upsert</li>
+</ul>
</body>
</html>
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/resources/docs/org.apache.nifi.processors.elasticsearch.PutElasticsearchRecord/additionalDetails.html b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/resources/docs/org.apache.nifi.processors.elasticsearch.PutElasticsearchRecord/additionalDetails.html
index 18da9cbff5..231c939e61 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/resources/docs/org.apache.nifi.processors.elasticsearch.PutElasticsearchRecord/additionalDetails.html
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/main/resources/docs/org.apache.nifi.processors.elasticsearch.PutElasticsearchRecord/additionalDetails.html
@@ -37,6 +37,8 @@
not too large for it to handle. When failures do occur, this processor is capable of attempting to write the records
that failed to an output record writer so that only failed records can be processed downstream or replayed.
</p>
+
+<h3>Per-Record Actions</h3>
<p>
The index, operation and (optional) type fields are configured with default values that can be overridden using
record path operations that find an index or type value in the record set.
@@ -44,8 +46,9 @@
the record set.
An "@timestamp" field can be added to the data either using a default or by extracting it from the record set.
This is useful if the documents are being indexed into an Elasticsearch Data Stream.
- The following is an example of a document exercising all of these features:
</p>
+<h4>Example - per-record actions</h4>
+<p>The following is an example of a document exercising all of these features:</p>
<pre>
{
"metadata": {
@@ -77,6 +80,101 @@
<li>metadata/operation</li>
<li>/ts</li>
</ul>
+
+<h3>Dynamic Templates</h3>
+<p>
+ Index and Create operations can use Dynamic Templates from the Record, record path operations can be configured to
+ find the Dynamic Templates from the record set. Dynamic Templates fields in Records must either be a Map,
+ child Record or a string that can be parsable as a JSON object.
+</p>
+<h4>Example - Index with Dynamic Templates</h4>
+<pre>
+ {
+ "message": "Hello, world",
+ "dynamic_templates": "{\"message\": \"keyword_lower\"}"
+ }
+</pre>
+<p>The record path operation below would extract the relevant Dynamic Templates:</p>
+<ul>
+ <li>/dynamic_templates</li>
+</ul>
+<p>Would create Elasticsearch action:</p>
+<pre>
+ { "index" : {"_id" : "1", "_index" : "test", "dynamic_templates" : {"message" : "keyword_lower"}} }
+ { "doc" : {"message" : "Hello, world"} }
+</pre>
+
+<h3>Update/Upsert Scripts</h3>
+<p>
+ Update and Upsert operations can use a script from the Record, record path operations can be configured to
+ find the script from the record set. Scripts must contain all the elements required by Elasticsearch, e.g. source and lang.
+ Script fields in Records must either be a Map, child Record or a string that can be parsable as a JSON object.
+</p>
+<p>
+ If a script is defined for an upset, any fields remaining in the Record will be used as the upsert fields in the Elasticsearch action.
+ If no script is defined, all Record fields will be used as the update doc (or doc_as_upsert for upsert operations).
+</p>
+<h4>Example - Update without Script</h4>
+<pre>
+ {
+ "message": "Hello, world",
+ "from": "john.smith"
+ }
+</pre>
+<p>Would create Elasticsearch action:</p>
+<pre>
+ { "update" : {"_id" : "1", "_index" : "test"} }
+ { "doc" : {"message" : "Hello, world", "from" : "john.smith"} }
+</pre>
+<h4>Example - Upsert with Script</h4>
+<pre>
+ {
+ "counter": 1,
+ "script" {
+ "source": "ctx._source.counter += params.param1",
+ "lang": "painless",
+ "params": {
+ "param1": 1
+ }
+ }
+ }
+</pre>
+<p>The record path operation below would extract the relevant script:</p>
+<ul>
+ <li>/script</li>
+</ul>
+<p>Would create Elasticsearch action:</p>
+<pre>
+ { "update" : {"_id" : "1", "_index" : "test"} }
+ { "script" : { "source": "ctx._source.counter += params.param1", "lang" : "painless", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
+</pre>
+
+<h3>Bulk Action Header Fields</h3>
+<p>
+ Dynamic Properties can be defined on the processor with <i>BULK:</i> prefixes. The value of the Dynamic Property
+ is a record path operation to find the field value from the record set.
+ Users must ensure that only known Bulk action fields are sent to Elasticsearch for the relevant index operation defined for the Record,
+ Elasticsearch will reject invalid combinations of index operation and Bulk action fields.
+</p>
+<h4>Example - Update with Retry on Conflict</h4>
+<pre>
+ {
+ "message": "Hello, world",
+ "from": "john.smith",
+ "retry": 3
+ }
+</pre>
+<p>The Dynamic Property and record path operation below would extract the relevant field:</p>
+<ul>
+ <li>BULK:retry_on_conflict = /retry</li>
+</ul>
+<p>Would create Elasticsearch action:</p>
+<pre>
+ { "update" : {"_id" : "1", "_index" : "test", "retry_on_conflict" : 3} }
+ { "doc" : {"message" : "Hello, world", "from" : "john.smith"} }
+</pre>
+
+<h3>Index Operations</h3>
<p>Valid values for "operation" are:</p>
<ul>
<li>create</li>
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/AbstractJsonQueryElasticsearchTest.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/AbstractJsonQueryElasticsearchTest.groovy
index 670b9599b0..e07d5bf3ff 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/AbstractJsonQueryElasticsearchTest.groovy
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/AbstractJsonQueryElasticsearchTest.groovy
@@ -217,8 +217,7 @@ abstract class AbstractJsonQueryElasticsearchTest<P extends AbstractJsonQueryEla
testCounts(runner, isInput() ? 1 : 0, 0, 0, 0)
assertThat(
runner.getProvenanceEvents().stream().filter({ pe ->
- pe.getEventType() == ProvenanceEventType.RECEIVE &&
- pe.getAttribute("uuid") == hits.getAttribute("uuid")
+ pe.getEventType() == ProvenanceEventType.RECEIVE
}).count(),
is(0L)
)
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/GetElasticsearchTest.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/GetElasticsearchTest.groovy
index 2a459cf035..c9b6fac092 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/GetElasticsearchTest.groovy
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/GetElasticsearchTest.groovy
@@ -29,7 +29,6 @@ import static groovy.json.JsonOutput.toJson
import static org.hamcrest.CoreMatchers.equalTo
import static org.hamcrest.CoreMatchers.is
import static org.hamcrest.MatcherAssert.assertThat
-
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertThrows
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/PutElasticsearchJsonTest.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/PutElasticsearchJsonTest.groovy
index e3c371eea8..6dfc68d77f 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/PutElasticsearchJsonTest.groovy
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/PutElasticsearchJsonTest.groovy
@@ -19,6 +19,7 @@ package org.apache.nifi.processors.elasticsearch
import org.apache.nifi.elasticsearch.IndexOperationRequest
import org.apache.nifi.elasticsearch.IndexOperationResponse
+import org.apache.nifi.processor.exception.ProcessException
import org.apache.nifi.processors.elasticsearch.mock.MockBulkLoadClientService
import org.apache.nifi.provenance.ProvenanceEventType
import org.apache.nifi.util.TestRunner
@@ -30,6 +31,7 @@ import static groovy.json.JsonOutput.toJson
import static org.hamcrest.CoreMatchers.containsString
import static org.hamcrest.MatcherAssert.assertThat
import static org.junit.jupiter.api.Assertions.assertEquals
+import static org.junit.jupiter.api.Assertions.assertInstanceOf
import static org.junit.jupiter.api.Assertions.assertThrows
import static org.junit.jupiter.api.Assertions.assertTrue
@@ -75,10 +77,18 @@ class PutElasticsearchJsonTest extends AbstractPutElasticsearchTest<PutElasticse
int indexCount = items.findAll { it.index == "test_index" }.size()
int typeCount = items.findAll { it.type == "test_type" }.size()
int opCount = items.findAll { it.operation == IndexOperationRequest.Operation.Index }.size()
+ int emptyScriptCount = items.findAll { it.script.isEmpty() }.size()
+ int falseScriptedUpsertCount = items.findAll { !it.scriptedUpsert }.size()
+ int emptyDynamicTemplatesCount = items.findAll { it.dynamicTemplates.isEmpty() }.size()
+ int emptyHeaderFields = items.findAll { it.headerFields.isEmpty() }.size()
assertEquals(1, nullIdCount)
assertEquals(1, indexCount)
assertEquals(1, typeCount)
assertEquals(1, opCount)
+ assertEquals(1, emptyScriptCount)
+ assertEquals(1, falseScriptedUpsertCount)
+ assertEquals(1, emptyDynamicTemplatesCount)
+ assertEquals(1, emptyHeaderFields)
}
basicTest(failure, retry, success, evalClosure)
@@ -122,10 +132,16 @@ class PutElasticsearchJsonTest extends AbstractPutElasticsearchTest<PutElasticse
}
@Test
- void simpleTestWithDocIdAndRequestParameters() {
+ void simpleTestWithDocIdAndRequestParametersAndBulkHeaders() {
runner.setProperty("refresh", "true")
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "routing", "1")
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "version", '${version}')
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "empty", '${empty}')
runner.setProperty("slices", '${slices}')
+ runner.setProperty("another", '${blank}')
runner.setVariable("slices", "auto")
+ runner.setVariable("blank", " ")
+ runner.setVariable("version", "external")
runner.assertValid()
def evalParametersClosure = { Map<String, String> params ->
@@ -141,10 +157,17 @@ class PutElasticsearchJsonTest extends AbstractPutElasticsearchTest<PutElasticse
int indexCount = items.findAll { it.index == "test_index" }.size()
int typeCount = items.findAll { it.type == "test_type" }.size()
int opCount = items.findAll { it.operation == IndexOperationRequest.Operation.Index }.size()
+ int headerFieldsCount = items.findAll { !it.headerFields.isEmpty() }.size()
assertEquals(1, idCount)
assertEquals(1, indexCount)
assertEquals(1, typeCount)
assertEquals(1, opCount)
+ assertEquals(1, headerFieldsCount)
+
+ def headerFields = items.get(0).headerFields
+ assertEquals(2, headerFields.size())
+ assertEquals("1", headerFields.get("routing"))
+ assertEquals("external", headerFields.get("version"))
}
clientService.evalClosure = evalClosure
@@ -153,9 +176,13 @@ class PutElasticsearchJsonTest extends AbstractPutElasticsearchTest<PutElasticse
}
@Test
- void simpleTestWithRequestParametersFlowFileEL() {
+ void simpleTestWithRequestParametersAndBulkHeadersFlowFileEL() {
runner.setProperty("refresh", "true")
runner.setProperty("slices", '${slices}')
+ runner.setVariable("blank", " ")
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "routing", "1")
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "version", '${version}')
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "empty", '${empty}')
runner.assertValid()
def evalParametersClosure = { Map<String, String> params ->
@@ -168,12 +195,67 @@ class PutElasticsearchJsonTest extends AbstractPutElasticsearchTest<PutElasticse
def evalClosure = { List<IndexOperationRequest> items ->
int nullIdCount = items.findAll { it.id == null }.size()
+ int headerFieldsCount = items.findAll { !it.headerFields.isEmpty() }.size()
assertEquals(1, nullIdCount)
+ assertEquals(1, headerFieldsCount)
+
+ def headerFields = items.get(0).headerFields
+ assertEquals(2, headerFields.size())
+ assertEquals("1", headerFields.get("routing"))
+ assertEquals("external", headerFields.get("version"))
}
clientService.evalClosure = evalClosure
- basicTest(0, 0, 1, [slices: "auto", "doc_id": ""])
+ basicTest(0, 0, 1, [slices: "auto", version: "external", blank: " ", "doc_id": ""])
+ }
+
+ @Test
+ void simpleTestWithScriptAndDynamicTemplates() {
+ runner.setProperty(PutElasticsearchJson.SCRIPT, prettyPrint(toJson(
+ [ _source: "some script", language: "painless" ]
+ )))
+ runner.setProperty(PutElasticsearchJson.DYNAMIC_TEMPLATES, prettyPrint(toJson(
+ [ my_field: "keyword", your_field: [type: "text", keyword: [type: "text"]]]
+ )))
+
+ def evalClosure = { List<IndexOperationRequest> items ->
+ int scriptCount = items.findAll { it.script == [ _source: "some script", language: "painless" ] }.size()
+ int falseScriptedUpsertCount = items.findAll { !it.scriptedUpsert }.size()
+ int dynamicTemplatesCount = items.findAll { it.dynamicTemplates == [ my_field: "keyword", your_field: [type: "text", keyword: [type: "text"]]] }.size()
+ assertEquals(1, scriptCount)
+ assertEquals(1, falseScriptedUpsertCount)
+ assertEquals(1, dynamicTemplatesCount)
+ }
+
+ basicTest(0, 0, 1, evalClosure)
+
+ runner.clearTransferState()
+ runner.clearProvenanceEvents()
+
+ runner.setProperty(PutElasticsearchJson.INDEX_OP, IndexOperationRequest.Operation.Upsert.getValue().toLowerCase())
+ runner.setProperty(PutElasticsearchJson.SCRIPTED_UPSERT, "true")
+ evalClosure = { List<IndexOperationRequest> items ->
+ int scriptCount = items.findAll { it.script == [ _source: "some script", language: "painless" ] }.size()
+ int trueScriptedUpsertCount = items.findAll { it.scriptedUpsert }.size()
+ int dynamicTemplatesCount = items.findAll { it.dynamicTemplates == [ my_field: "keyword", your_field: [type: "text", keyword: [type: "text"]]] }.size()
+ assertEquals(1, scriptCount)
+ assertEquals(1, trueScriptedUpsertCount)
+ assertEquals(1, dynamicTemplatesCount)
+ }
+
+ basicTest(0, 0, 1, evalClosure)
+
+ runner.clearTransferState()
+ runner.clearProvenanceEvents()
+
+ runner.setProperty(PutElasticsearchJson.SCRIPT, "not-json")
+ runner.removeProperty(PutElasticsearchJson.DYNAMIC_TEMPLATES)
+
+ runner.enqueue(flowFileContents)
+ final AssertionError ae = assertThrows(AssertionError.class, runner.&run)
+ assertInstanceOf(ProcessException.class, ae.getCause())
+ assertEquals(PutElasticsearchJson.SCRIPT.getDisplayName() + " must be a String parsable into a JSON Object", ae.getCause().getMessage())
}
@Test
@@ -250,7 +332,7 @@ class PutElasticsearchJsonTest extends AbstractPutElasticsearchTest<PutElasticse
runner.assertTransferCount(PutElasticsearchJson.REL_FAILED_DOCUMENTS, 1)
runner.assertTransferCount(PutElasticsearchRecord.REL_ERROR_RESPONSES, 0)
- def failedDoc = runner.getFlowFilesForRelationship(PutElasticsearchJson.REL_FAILED_DOCUMENTS)[0];
+ def failedDoc = runner.getFlowFilesForRelationship(PutElasticsearchJson.REL_FAILED_DOCUMENTS)[0]
assertThat(failedDoc.getContent(), containsString("20abcd"))
failedDoc.assertAttributeExists("elasticsearch.bulk.error")
failedDoc.assertAttributeNotExists("elasticsearch.put.error")
@@ -280,13 +362,13 @@ class PutElasticsearchJsonTest extends AbstractPutElasticsearchTest<PutElasticse
runner.assertTransferCount(PutElasticsearchJson.REL_FAILED_DOCUMENTS, 2)
runner.assertTransferCount(PutElasticsearchRecord.REL_ERROR_RESPONSES, 1)
- failedDoc = runner.getFlowFilesForRelationship(PutElasticsearchJson.REL_FAILED_DOCUMENTS)[0];
+ failedDoc = runner.getFlowFilesForRelationship(PutElasticsearchJson.REL_FAILED_DOCUMENTS)[0]
assertThat(failedDoc.getContent(), containsString("not_found"))
failedDoc.assertAttributeExists("elasticsearch.bulk.error")
failedDoc.assertAttributeNotExists("elasticsearch.put.error")
assertThat(failedDoc.getAttribute("elasticsearch.bulk.error"), containsString("not_found"))
- failedDoc = runner.getFlowFilesForRelationship(PutElasticsearchJson.REL_FAILED_DOCUMENTS)[1];
+ failedDoc = runner.getFlowFilesForRelationship(PutElasticsearchJson.REL_FAILED_DOCUMENTS)[1]
assertThat(failedDoc.getContent(), containsString("20abcd"))
failedDoc.assertAttributeExists("elasticsearch.bulk.error")
failedDoc.assertAttributeNotExists("elasticsearch.put.error")
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecordTest.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecordTest.groovy
index 085844be3a..f3992d79da 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecordTest.groovy
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/PutElasticsearchRecordTest.groovy
@@ -74,13 +74,15 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
type: "record",
fields: [
[ name: "msg", type: "string" ],
- [ name: "from", type: "string" ]
+ [ name: "from", type: "string" ],
+ [ name: "routing", type: "string" ],
+ [ name: "version", type: "string" ]
]
]))
static final List<Map<String, String>> flowFileContentMaps = [
- [ msg: "Hello, world", from: "john.smith" ],
- [ msg: "Hi, back at ya!", from: "jane.doe" ]
+ [ msg: "Hello, world", from: "john.smith", routing: "1", version: " " ],
+ [ msg: "Hi, back at ya!", from: "jane.doe", version: "external" ]
]
static final String flowFileContents = prettyPrint(toJson(flowFileContentMaps))
@@ -126,10 +128,18 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
int indexCount = items.findAll { it.index == "test_index" }.size()
int typeCount = items.findAll { it.type == "test_type" }.size()
int opCount = items.findAll { it.operation == IndexOperationRequest.Operation.Index }.size()
+ int emptyScriptCount = items.findAll { it.script.isEmpty() }.size()
+ int falseScriptedUpsertCount = items.findAll { !it.scriptedUpsert }.size()
+ int emptyDynamicTemplatesCount = items.findAll { it.dynamicTemplates.isEmpty() }.size()
+ int emptyHeaderFields = items.findAll { it.headerFields.isEmpty() }.size()
assertEquals(2, timestampDefaultCount)
assertEquals(2, indexCount)
assertEquals(2, typeCount)
assertEquals(2, opCount)
+ assertEquals(2, emptyScriptCount)
+ assertEquals(2, falseScriptedUpsertCount)
+ assertEquals(2, emptyDynamicTemplatesCount)
+ assertEquals(2, emptyHeaderFields)
}
basicTest(failure, retry, success, evalClosure)
@@ -188,10 +198,16 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
}
@Test
- void simpleTestWithRequestParameters() {
+ void simpleTestWithRequestParametersAndBulkHeaders() {
runner.setProperty("refresh", "true")
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "routing", "/routing")
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "version", '${version}')
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "empty", '${empty}')
runner.setProperty("slices", '${slices}')
+ runner.setProperty("another", '${blank}')
runner.setVariable("slices", "auto")
+ runner.setVariable("blank", " ")
+ runner.setVariable("version", "/version")
runner.assertValid()
def evalParametersClosure = { Map<String, String> params ->
@@ -202,13 +218,26 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
clientService.evalParametersClosure = evalParametersClosure
- basicTest(0, 0, 1)
+ def evalClosure = { List<IndexOperationRequest> items ->
+ int headerFieldsCount = items.findAll { !it.headerFields.isEmpty() }.size()
+ int routingCount = items.findAll { it.headerFields.get("routing") == "1" }.size()
+ int versionCount = items.findAll { it.headerFields.get("version") == "external" }.size()
+ assertEquals(2, headerFieldsCount)
+ assertEquals(1, routingCount)
+ assertEquals(1, versionCount)
+ }
+
+ basicTest(0, 0, 1, evalClosure)
}
@Test
- void simpleTestWithRequestParametersFlowFileEL() {
+ void simpleTestWithRequestParametersAndBulkHeadersFlowFileEL() {
runner.setProperty("refresh", "true")
runner.setProperty("slices", '${slices}')
+ runner.setVariable("blank", " ")
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "routing", "/routing")
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "version", '${version}')
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "empty", '${empty}')
runner.assertValid()
def evalParametersClosure = { Map<String, String> params ->
@@ -219,7 +248,18 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
clientService.evalParametersClosure = evalParametersClosure
- basicTest(0, 0, 1, ["schema.name": "simple", slices: "auto"])
+ def evalClosure = { List<IndexOperationRequest> items ->
+ int headerFieldsCount = items.findAll { !it.headerFields.isEmpty() }.size()
+ int routingCount = items.findAll { it.headerFields.get("routing") == "1" }.size()
+ int versionCount = items.findAll { it.headerFields.get("version") == "external" }.size()
+ assertEquals(2, headerFieldsCount)
+ assertEquals(1, routingCount)
+ assertEquals(1, versionCount)
+ }
+
+ clientService.evalClosure = evalClosure
+
+ basicTest(0, 0, 1, ["schema.name": "simple", version: "/version", slices: "auto", blank: " "])
}
@Test
@@ -263,17 +303,25 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
[ name: "ts", type: [ type: "long", logicalType: "timestamp-millis" ] ],
[ name: "date", type: [ type: "int", logicalType: "date" ] ],
[ name: "time", type: [ type: "int", logicalType: "time-millis" ] ],
- [ name: "code", type: "long" ]
+ [ name: "code", type: "long" ],
+ [ name: "script", type: [ type: "map", values: "string" ] ],
+ [ name: "scripted_upsert", type: ["null", "boolean", "string"] ],
+ [ name: "script_record", type: [ type: "record", name: "nested", fields: [
+ [ name: "source", type: "string" ], [ name: "language", type: "string" ]
+ ] ] ],
+ [ name: "dynamic_templates", type: "string" ]
]
]))
+ def script = [ source: "some script", language: "painless" ]
+ def dynamicTemplates = [ my_field: "keyword", your_field: [type: "text", keyword: [type: "text"]]]
def flowFileContents = prettyPrint(toJson([
[ id: "rec-1", op: "index", index: "bulk_a", type: "message", msg: "Hello", ts: Timestamp.valueOf(LOCAL_DATE_TIME).toInstant().toEpochMilli() ],
[ id: "rec-2", op: "index", index: "bulk_b", type: "message", msg: "Hello" ],
[ id: "rec-3", op: "index", index: "bulk_a", type: "message", msg: "Hello" ],
- [ id: "rec-4", op: "index", index: "bulk_b", type: "message", msg: "Hello" ],
- [ id: "rec-5", op: "index", index: "bulk_a", type: "message", msg: "" ],
- [ id: "rec-6", op: "create", index: "bulk_b", type: "message", msg: null ]
+ [ id: "rec-4", op: "index", index: "bulk_b", type: "message", msg: "Hello", scripted_upsert: null ],
+ [ id: "rec-5", op: "index", index: "bulk_a", type: "message", msg: "", script: script, scripted_upsert: false, dynamic_templates: null],
+ [ id: "rec-6", op: "create", index: "bulk_b", type: "message", msg: null, script: null, scripted_upsert: true, dynamic_templates: prettyPrint(toJson(dynamicTemplates)) ]
]))
def evalClosure = { List<IndexOperationRequest> items ->
@@ -290,6 +338,12 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
int timestampDefault = items.findAll { it.fields.get("@timestamp") == "test_timestamp" }.size()
int ts = items.findAll { it.fields.get("ts") != null }.size()
int id = items.findAll { it.fields.get("id") != null }.size()
+ int emptyScript = items.findAll { it.script.isEmpty() }.size()
+ int falseScriptedUpsertCount = items.findAll { !it.scriptedUpsert }.size()
+ int trueScriptedUpsertCount = items.findAll { it.scriptedUpsert }.size()
+ int s = items.findAll { it.script == script }.size()
+ int emptyDynamicTemplates = items.findAll { it.dynamicTemplates.isEmpty() }.size()
+ int dt = items.findAll { it.dynamicTemplates == dynamicTemplates }.size()
items.each {
assertNotNull(it.id)
assertTrue(it.id.startsWith("rec-"))
@@ -307,6 +361,12 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
assertEquals(5, timestampDefault)
assertEquals(0, ts)
assertEquals(0, id)
+ assertEquals(5, emptyScript)
+ assertEquals(5, falseScriptedUpsertCount)
+ assertEquals(1, trueScriptedUpsertCount)
+ assertEquals(1, s)
+ assertEquals(5, emptyDynamicTemplates)
+ assertEquals(1, dt)
}
clientService.evalClosure = evalClosure
@@ -319,6 +379,9 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
runner.setProperty(PutElasticsearchRecord.INDEX_RECORD_PATH, "/index")
runner.setProperty(PutElasticsearchRecord.TYPE_RECORD_PATH, "/type")
runner.setProperty(PutElasticsearchRecord.AT_TIMESTAMP_RECORD_PATH, "/ts")
+ runner.setProperty(PutElasticsearchRecord.SCRIPT_RECORD_PATH, "/script")
+ runner.setProperty(PutElasticsearchRecord.SCRIPTED_UPSERT_RECORD_PATH, "/scripted_upsert")
+ runner.setProperty(PutElasticsearchRecord.DYNAMIC_TEMPLATES_RECORD_PATH, "/dynamic_templates")
runner.enqueue(flowFileContents, [
"schema.name": "recordPathTest"
])
@@ -339,7 +402,7 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
[ id: "rec-3", op: null, index: null, type: null, msg: "Hello" ],
[ id: "rec-4", op: null, index: null, type: null, msg: "Hello" ],
[ id: "rec-5", op: "update", index: null, type: null, msg: "Hello" ],
- [ id: "rec-6", op: null, index: "bulk_b", type: "message", msg: "Hello" ]
+ [ id: "rec-6", op: null, index: "bulk_b", type: "message", msg: "Hello", script_record: script ]
]))
evalClosure = { List<IndexOperationRequest> items ->
@@ -355,6 +418,8 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
int dateCount = items.findAll { it.fields.get("date") != null }.size()
def idCount = items.findAll { it.fields.get("id") != null }.size()
def defaultCoercedTimestampCount = items.findAll { it.fields.get("@timestamp") == 100L }.size()
+ int emptyScriptCount = items.findAll { it.script.isEmpty() }.size()
+ int scriptCount = items.findAll { it.script == script }.size()
assertEquals(5, testTypeCount)
assertEquals(1, messageTypeCount)
assertEquals(5, testIndexCount)
@@ -365,6 +430,8 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
assertEquals(5, defaultCoercedTimestampCount)
assertEquals(1, dateCount)
assertEquals(6, idCount)
+ assertEquals(5, emptyScriptCount)
+ assertEquals(1, scriptCount)
}
clientService.evalClosure = evalClosure
@@ -375,6 +442,7 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
runner.setProperty(PutElasticsearchRecord.AT_TIMESTAMP_RECORD_PATH, "/date")
runner.setProperty(PutElasticsearchRecord.DATE_FORMAT, "dd/MM/yyyy")
runner.setProperty(PutElasticsearchRecord.RETAIN_AT_TIMESTAMP_FIELD, "true")
+ runner.setProperty(PutElasticsearchRecord.SCRIPT_RECORD_PATH, "/script_record")
runner.enqueue(flowFileContents, [
"schema.name": "recordPathTest",
"operation": "index"
@@ -545,6 +613,73 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
runner.assertTransferCount(PutElasticsearchRecord.REL_FAILED_RECORDS, 0)
runner.assertTransferCount(PutElasticsearchRecord.REL_SUCCESSFUL_RECORDS, 0)
runner.assertTransferCount(PutElasticsearchRecord.REL_ERROR_RESPONSES, 0)
+
+ runner.clearTransferState()
+
+ flowFileContents = prettyPrint(toJson([
+ [ id: "rec-1", op: "index", index: "bulk_a", type: "message", msg: "Hello", dynamic_templates: "" ]
+ ]))
+
+ runner.enqueue(flowFileContents, [
+ "schema.name": "recordPathTest"
+ ])
+ runner.run()
+ runner.assertTransferCount(PutElasticsearchRecord.REL_SUCCESS, 0)
+ runner.assertTransferCount(PutElasticsearchRecord.REL_FAILURE, 1)
+ runner.assertTransferCount(PutElasticsearchRecord.REL_RETRY, 0)
+ runner.assertTransferCount(PutElasticsearchRecord.REL_FAILED_RECORDS, 0)
+ runner.assertTransferCount(PutElasticsearchRecord.REL_SUCCESSFUL_RECORDS, 0)
+ runner.assertTransferCount(PutElasticsearchRecord.REL_ERROR_RESPONSES, 0)
+ def failure = runner.getFlowFilesForRelationship(PutElasticsearchRecord.REL_FAILURE)[0]
+ failure.assertAttributeEquals("elasticsearch.put.error", String.format("Field referenced by %s must be Map-type compatible or a String parsable into a JSON Object","/dynamic_templates"))
+
+ runner.clearTransferState()
+
+ flowFileContents = prettyPrint(toJson([
+ [ id: "rec-1", op: "index", index: "bulk_a", type: "message" ],
+ [ id: null, op: "create", index: "bulk_a", msg: "Hello" ],
+ [ id: "rec-3", op: null, type: "message", msg: "Hello" ],
+ [ id: "rec-4", index: null, type: "message", msg: "Hello" ],
+ [ op: "create", index: "bulk_a", type: null, msg: "Hello" ],
+ [ id: "rec-6", op: "create", index: "bulk_a", type: "message", msg: null ]
+ ]))
+
+ clientService.evalClosure = { List<IndexOperationRequest> items ->
+ int idNotNull = items.findAll { it.id != null }.size()
+ int opIndex = items.findAll { it.operation == IndexOperationRequest.Operation.Index }.size()
+ int opCreate = items.findAll { it.operation == IndexOperationRequest.Operation.Create }.size()
+ int indexA = items.findAll { it.index == "bulk_a" }.size()
+ int indexC = items.findAll { it.index == "bulk_c" }.size()
+ int typeMessage = items.findAll { it.type == "message" }.size()
+ int typeBlah = items.findAll { it.type == "blah" }.size()
+ assertEquals(4, idNotNull)
+ assertEquals(3, opIndex)
+ assertEquals(3, opCreate)
+ assertEquals(4, indexA)
+ assertEquals(2, indexC)
+ assertEquals(4, typeMessage)
+ assertEquals(2, typeBlah)
+ }
+
+ runner.setProperty(PutElasticsearchRecord.INDEX_OP, "index")
+ runner.setProperty(PutElasticsearchRecord.INDEX_OP_RECORD_PATH, "/op")
+ runner.setProperty(PutElasticsearchRecord.TYPE, "blah")
+ runner.setProperty(PutElasticsearchRecord.TYPE_RECORD_PATH, "/type")
+ runner.setProperty(PutElasticsearchRecord.INDEX, "bulk_c")
+ runner.setProperty(PutElasticsearchRecord.INDEX_RECORD_PATH, "/index")
+ runner.setProperty(PutElasticsearchRecord.ID_RECORD_PATH, "/id")
+ runner.setProperty(PutElasticsearchRecord.RETAIN_ID_FIELD, "false")
+ runner.removeProperty(PutElasticsearchRecord.AT_TIMESTAMP_RECORD_PATH)
+ runner.enqueue(flowFileContents, [
+ "schema.name": "recordPathTest"
+ ])
+ runner.run()
+ runner.assertTransferCount(PutElasticsearchRecord.REL_SUCCESS, 1)
+ runner.assertTransferCount(PutElasticsearchRecord.REL_FAILURE, 0)
+ runner.assertTransferCount(PutElasticsearchRecord.REL_RETRY, 0)
+ runner.assertTransferCount(PutElasticsearchRecord.REL_FAILED_RECORDS, 0)
+ runner.assertTransferCount(PutElasticsearchRecord.REL_SUCCESSFUL_RECORDS, 0)
+ runner.assertTransferCount(PutElasticsearchRecord.REL_ERROR_RESPONSES, 0)
}
@Test
@@ -823,4 +958,11 @@ class PutElasticsearchRecordTest extends AbstractPutElasticsearchTest<PutElastic
}).count()
)
}
+
+ @Test
+ void testInvalidBulkHeaderProperty() {
+ runner.assertValid()
+ runner.setProperty(AbstractPutElasticsearch.BULK_HEADER_PREFIX + "routing", "not-record-path")
+ runner.assertNotValid()
+ }
}
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy
index 2d3b715ce8..5ab82471cc 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/TestElasticsearchClientService.groovy
@@ -23,15 +23,12 @@ import org.apache.nifi.controller.AbstractControllerService
import org.apache.nifi.controller.ConfigurationContext
import org.apache.nifi.elasticsearch.DeleteOperationResponse
import org.apache.nifi.elasticsearch.ElasticSearchClientService
-import org.apache.nifi.elasticsearch.ElasticsearchException
import org.apache.nifi.elasticsearch.IndexOperationRequest
import org.apache.nifi.elasticsearch.IndexOperationResponse
import org.apache.nifi.elasticsearch.SearchResponse
import org.apache.nifi.elasticsearch.UpdateOperationResponse
import org.apache.nifi.logging.ComponentLog
import org.apache.nifi.processors.elasticsearch.mock.MockElasticsearchException
-import org.elasticsearch.client.Response
-import org.elasticsearch.client.ResponseException
class TestElasticsearchClientService extends AbstractControllerService implements ElasticSearchClientService {
private boolean returnAggs
@@ -114,6 +111,11 @@ class TestElasticsearchClientService extends AbstractControllerService implement
return true
}
+ @Override
+ boolean documentExists(String index, String type, String id, Map<String, String> requestParameters) {
+ return true
+ }
+
@Override
Map<String, Object> get(String index, String type, String id, Map<String, String> requestParameters) {
common(throwErrorInGet || throwNotFoundInGet, requestParameters)
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy
index 9498b8da5a..902226824c 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-restapi-processors/src/test/groovy/org/apache/nifi/processors/elasticsearch/mock/AbstractMockElasticsearchClient.groovy
@@ -82,6 +82,11 @@ class AbstractMockElasticsearchClient extends AbstractControllerService implemen
return true
}
+ @Override
+ boolean documentExists(String index, String type, String id, Map<String, String> requestParameters) {
+ return true
+ }
+
@Override
Map<String, Object> get(String index, String type, String id, Map<String, String> requestParameters) {
return null
diff --git a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-test-utils/src/main/java/org/apache/nifi/elasticsearch/integration/AbstractElasticsearchITBase.java b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-test-utils/src/main/java/org/apache/nifi/elasticsearch/integration/AbstractElasticsearchITBase.java
index 48610e8261..bf9afcd371 100644
--- a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-test-utils/src/main/java/org/apache/nifi/elasticsearch/integration/AbstractElasticsearchITBase.java
+++ b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-test-utils/src/main/java/org/apache/nifi/elasticsearch/integration/AbstractElasticsearchITBase.java
@@ -18,18 +18,14 @@ package org.apache.nifi.elasticsearch.integration;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.compress.utils.IOUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.nio.entity.NStringEntity;
-import org.apache.nifi.elasticsearch.MapBuilder;
import org.apache.nifi.util.TestRunner;
import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;
import org.junit.jupiter.api.BeforeAll;
@@ -39,16 +35,12 @@ import org.testcontainers.utility.DockerImageName;
import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
-import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
-import java.util.Map;
import static org.apache.http.auth.AuthScope.ANY;
@@ -88,7 +80,7 @@ public abstract class AbstractElasticsearchITBase {
protected static String type;
- private static RestClient testDataManagementClient;
+ static RestClient testDataManagementClient;
protected static void stopTestcontainer() {
if (ENABLE_TEST_CONTAINERS) {
@@ -173,32 +165,6 @@ public abstract class AbstractElasticsearchITBase {
return MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(o);
}
- protected Pair<String, String> createApiKeyForIndex(String index) throws IOException {
- final String body = prettyJson(new MapBuilder()
- .of("name", "test-api-key")
- .of("role_descriptors", new MapBuilder()
- .of("test-role", new MapBuilder()
- .of("cluster", Collections.singletonList("all"))
- .of("index", Collections.singletonList(new MapBuilder()
- .of("names", Collections.singletonList(index))
- .of("privileges", Collections.singletonList("all"))
- .build()))
- .build())
- .build())
- .build());
- final String endpoint = String.format("%s/%s", elasticsearchHost, "_security/api_key");
- final Request request = new Request("POST", endpoint);
- final HttpEntity jsonBody = new NStringEntity(body, ContentType.APPLICATION_JSON);
- request.setEntity(jsonBody);
-
- final Response response = testDataManagementClient.performRequest(request);
- final InputStream inputStream = response.getEntity().getContent();
- final byte[] result = IOUtils.toByteArray(inputStream);
- inputStream.close();
- final Map<String, String> ret = MAPPER.readValue(new String(result, StandardCharsets.UTF_8), Map.class);
- return Pair.of(ret.get("id"), ret.get("api_key"));
- }
-
private static List<SetupAction> readSetupActions(final String scriptPath) throws IOException {
final List<SetupAction> actions = new ArrayList<>();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(scriptPath))))) {