You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by ni...@apache.org on 2022/05/23 08:51:23 UTC
[pulsar] branch master updated: [feature][connector] ElasticSearch Sink: option to output canonical key fields (JSON and Avro) (#15426)
This is an automated email from the ASF dual-hosted git repository.
nicoloboschi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new a4488fd2e5a [feature][connector] ElasticSearch Sink: option to output canonical key fields (JSON and Avro) (#15426)
a4488fd2e5a is described below
commit a4488fd2e5a07e1aa25ccc78ee4af9b782137424
Author: Nicolò Boschi <bo...@gmail.com>
AuthorDate: Mon May 23 10:51:12 2022 +0200
[feature][connector] ElasticSearch Sink: option to output canonical key fields (JSON and Avro) (#15426)
* [feat][elasticsearch-sink] Option to output canonical JSON
* [feat][elasticsearch-sink] Option to output canonical JSON
* style
* fix test
* style
* correct doc
* Update site2/docs/io-elasticsearch-sink.md
Co-authored-by: momo-jun <60...@users.noreply.github.com>
Co-authored-by: momo-jun <60...@users.noreply.github.com>
---
.../io/elasticsearch/ElasticSearchConfig.java | 7 ++
.../pulsar/io/elasticsearch/ElasticSearchSink.java | 44 +++++--
.../pulsar/io/elasticsearch/JsonConverter.java | 2 +-
.../elasticsearch/ElasticSearchExtractTests.java | 140 +++++++++++++++++++++
site2/docs/io-elasticsearch-sink.md | 1 +
5 files changed, 186 insertions(+), 8 deletions(-)
diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java
index 77bf89cf8f9..396e892fc12 100644
--- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java
+++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java
@@ -274,6 +274,13 @@ public class ElasticSearchConfig implements Serializable {
)
private CompatibilityMode compatibilityMode = CompatibilityMode.AUTO;
+ @FieldDoc(
+ defaultValue = "false",
+ help = "If canonicalKeyFields is true and record key schema is JSON or AVRO, the serialized object will "
+ + "not consider the properties order."
+ )
+ private boolean canonicalKeyFields = false;
+
public enum MalformedDocAction {
IGNORE,
WARN,
diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java
index dc82ee60e65..9f703a59ecb 100644
--- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java
+++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java
@@ -21,6 +21,10 @@ package org.apache.pulsar.io.elasticsearch;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import java.nio.charset.StandardCharsets;
@@ -29,6 +33,7 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.client.api.Message;
@@ -56,7 +61,8 @@ public class ElasticSearchSink implements Sink<GenericObject> {
private ElasticSearchConfig elasticSearchConfig;
private ElasticSearchClient elasticsearchClient;
- private ObjectMapper objectMapper = new ObjectMapper();
+ private final ObjectMapper objectMapper = new ObjectMapper();
+ private ObjectMapper sortedObjectMapper;
private List<String> primaryFields = null;
@Override
@@ -67,6 +73,19 @@ public class ElasticSearchSink implements Sink<GenericObject> {
if (!Strings.isNullOrEmpty(elasticSearchConfig.getPrimaryFields())) {
primaryFields = Arrays.asList(elasticSearchConfig.getPrimaryFields().split(","));
}
+ if (elasticSearchConfig.isCanonicalKeyFields()) {
+ sortedObjectMapper = JsonMapper
+ .builder()
+ .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)
+ .nodeFactory(new JsonNodeFactory() {
+ @Override
+ public ObjectNode objectNode() {
+ return new ObjectNode(this, new TreeMap<String, JsonNode>());
+ }
+
+ })
+ .build();
+ }
}
@Override
@@ -238,9 +257,11 @@ public class ElasticSearchSink implements Sink<GenericObject> {
}
}
+
/**
* Convert a JsonNode to an Elasticsearch id.
*/
+
public String stringifyKey(JsonNode jsonNode) throws JsonProcessingException {
List<String> fields = new ArrayList<>();
jsonNode.fieldNames().forEachRemaining(fields::add);
@@ -248,15 +269,24 @@ public class ElasticSearchSink implements Sink<GenericObject> {
}
public String stringifyKey(JsonNode jsonNode, List<String> fields) throws JsonProcessingException {
+ JsonNode toConvert;
if (fields.size() == 1) {
- JsonNode singleNode = jsonNode.get(fields.get(0));
- String id = objectMapper.writeValueAsString(singleNode);
- return (id.startsWith("\"") && id.endsWith("\""))
- ? id.substring(1, id.length() - 1) // remove double quotes
- : id;
+ toConvert = jsonNode.get(fields.get(0));
} else {
- return JsonConverter.toJsonArray(jsonNode, fields).toString();
+ toConvert = JsonConverter.toJsonArray(jsonNode, fields);
}
+
+ String serializedId;
+ if (elasticSearchConfig.isCanonicalKeyFields()) {
+ final Object obj = sortedObjectMapper.treeToValue(toConvert, Object.class);
+ serializedId = sortedObjectMapper.writeValueAsString(obj);
+ } else {
+ serializedId = objectMapper.writeValueAsString(toConvert);
+ }
+
+ return (serializedId.startsWith("\"") && serializedId.endsWith("\""))
+ ? serializedId.substring(1, serializedId.length() - 1) // remove double quotes
+ : serializedId;
}
public String stringifyValue(Schema<?> schema, Object val) throws JsonProcessingException {
diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java
index 8c1c51c9deb..a7d15914044 100644
--- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java
+++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java
@@ -212,7 +212,7 @@ public class JsonConverter {
public static ArrayNode toJsonArray(JsonNode jsonNode, List<String> fields) {
ArrayNode arrayNode = jsonNodeFactory.arrayNode();
- Iterator<String> it = jsonNode.fieldNames();
+ Iterator<String> it = jsonNode.fieldNames();
while (it.hasNext()) {
String fieldName = it.next();
if (fields.contains(fieldName)) {
diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
index 518b72aa3bd..f99316eb940 100644
--- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
+++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.io.elasticsearch;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNull;
+
import com.google.common.collect.ImmutableMap;
import java.util.Optional;
import org.apache.commons.lang3.tuple.Pair;
@@ -270,4 +271,143 @@ public class ElasticSearchExtractTests {
assertEquals(pair3.getLeft(), "[\"1\",1]");
assertNull(pair3.getRight());
}
+
+ @Test(dataProvider = "schemaType")
+ public void testSortKeysSingle(SchemaType schemaType) throws Exception {
+ RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
+
+ RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1");
+ udtSchemaBuilder.field("b_inside_inner").type(SchemaType.STRING).optional().defaultValue(null);
+ udtSchemaBuilder.field("a_inside_inner").type(SchemaType.STRING).optional().defaultValue(null);
+ GenericSchema<GenericRecord> udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType));
+
+ keySchemaBuilder.field("singleKey", udtGenericSchema).type(schemaType).optional().defaultValue(null);
+ GenericSchema<GenericRecord> keySchema = Schema.generic(keySchemaBuilder.build(schemaType));
+
+ final GenericRecord innerRecord = udtGenericSchema.newRecordBuilder()
+ .set("b_inside_inner", "0b_value_from_inner")
+ .set("a_inside_inner", "a_value_from_inner")
+ .build();
+
+ GenericRecord keyGenericRecord = keySchema.newRecordBuilder()
+ .set("singleKey", innerRecord)
+ .build();
+
+ Record<GenericObject> genericObjectRecord = getKeyValueGenericObject(schemaType, keySchema, keyGenericRecord);
+
+
+ ElasticSearchSink elasticSearchSink = new ElasticSearchSink();
+ elasticSearchSink.open(ImmutableMap.of(
+ "elasticSearchUrl", "http://localhost:9200",
+ "compatibilityMode", "ELASTICSEARCH",
+ "schemaEnable", "true",
+ "canonicalKeyFields", "false",
+ "keyIgnore", "false"), null);
+ Pair<String, String> pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
+ assertEquals(pair.getKey(), "{\"b_inside_inner\":\"0b_value_from_inner\",\"a_inside_inner\":\"a_value_from_inner\"}");
+
+ elasticSearchSink = new ElasticSearchSink();
+ elasticSearchSink.open(ImmutableMap.of(
+ "elasticSearchUrl", "http://localhost:9200",
+ "compatibilityMode", "ELASTICSEARCH",
+ "schemaEnable", "true",
+ "canonicalKeyFields", "true",
+ "keyIgnore", "false"), null);
+ pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
+ assertEquals(pair.getKey(), "{\"a_inside_inner\":\"a_value_from_inner\",\"b_inside_inner\":\"0b_value_from_inner\"}");
+
+ }
+
+ @Test(dataProvider = "schemaType")
+ public void testSortKeysMulti(SchemaType schemaType) throws Exception {
+ RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
+ keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
+ keySchemaBuilder.field("b").type(SchemaType.STRING).optional().defaultValue(null);
+ keySchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null);
+
+ RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1");
+ udtSchemaBuilder.field("b_inside_inner").type(SchemaType.STRING).optional().defaultValue(null);
+ udtSchemaBuilder.field("a_inside_inner").type(SchemaType.STRING).optional().defaultValue(null);
+ GenericSchema<GenericRecord> udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType));
+
+ keySchemaBuilder.field("inner", udtGenericSchema).type(schemaType).optional().defaultValue(null);
+ GenericSchema<GenericRecord> keySchema = Schema.generic(keySchemaBuilder.build(schemaType));
+
+ final GenericRecord innerRecord = udtGenericSchema.newRecordBuilder()
+ .set("b_inside_inner", "0b_value_from_inner")
+ .set("a_inside_inner", "a_value_from_inner")
+ .build();
+
+ GenericRecord keyGenericRecord = keySchema.newRecordBuilder()
+ .set("c", "c_key")
+ .set("b", "0b_key")
+ .set("a", "a_key")
+ .set("inner", innerRecord)
+ .build();
+
+ Record<GenericObject> genericObjectRecord = getKeyValueGenericObject(schemaType, keySchema, keyGenericRecord);
+
+
+ ElasticSearchSink elasticSearchSink = new ElasticSearchSink();
+ elasticSearchSink.open(ImmutableMap.of(
+ "elasticSearchUrl", "http://localhost:9200",
+ "compatibilityMode", "ELASTICSEARCH",
+ "schemaEnable", "true",
+ "canonicalKeyFields", "false",
+ "keyIgnore", "false"), null);
+ Pair<String, String> pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
+ assertEquals(pair.getKey(), "[\"a_key\",\"0b_key\",\"c_key\",{\"b_inside_inner\":\"0b_value_from_inner\",\"a_inside_inner\":\"a_value_from_inner\"}]");
+
+ elasticSearchSink = new ElasticSearchSink();
+ elasticSearchSink.open(ImmutableMap.of(
+ "elasticSearchUrl", "http://localhost:9200",
+ "compatibilityMode", "ELASTICSEARCH",
+ "schemaEnable", "true",
+ "canonicalKeyFields", "true",
+ "keyIgnore", "false"), null);
+ pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
+ assertEquals(pair.getKey(), "[\"a_key\",\"0b_key\",\"c_key\",{\"a_inside_inner\":\"a_value_from_inner\",\"b_inside_inner\":\"0b_value_from_inner\"}]");
+ }
+
+ private Record<GenericObject> getKeyValueGenericObject(SchemaType schemaType, GenericSchema<GenericRecord> keySchema, GenericRecord keyGenericRecord) {
+ RecordSchemaBuilder valueSchemaBuilder = SchemaBuilder.record("value");
+ valueSchemaBuilder.field("value").type(SchemaType.STRING);
+ GenericSchema<GenericRecord> valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType));
+
+ GenericRecord valueGenericRecord = valueSchema.newRecordBuilder()
+ .set("value", "value")
+ .build();
+
+ Schema<KeyValue<GenericRecord, GenericRecord>> keyValueSchema =
+ Schema.KeyValue(keySchema, valueSchema, KeyValueEncodingType.INLINE);
+ KeyValue<GenericRecord, GenericRecord> keyValue = new KeyValue<>(keyGenericRecord, valueGenericRecord);
+ GenericObject genericObject = new GenericObject() {
+ @Override
+ public SchemaType getSchemaType() {
+ return SchemaType.KEY_VALUE;
+ }
+
+ @Override
+ public Object getNativeObject() {
+ return keyValue;
+ }
+ };
+ Record<GenericObject> genericObjectRecord = new Record<GenericObject>() {
+ @Override
+ public Optional<String> getTopicName() {
+ return Optional.of("data-ks1.table1");
+ }
+
+ @Override
+ public Schema getSchema() {
+ return keyValueSchema;
+ }
+
+ @Override
+ public GenericObject getValue() {
+ return genericObject;
+ }
+ };
+ return genericObjectRecord;
+ }
}
\ No newline at end of file
diff --git a/site2/docs/io-elasticsearch-sink.md b/site2/docs/io-elasticsearch-sink.md
index ada9ce71916..26ea81f6026 100644
--- a/site2/docs/io-elasticsearch-sink.md
+++ b/site2/docs/io-elasticsearch-sink.md
@@ -86,6 +86,7 @@ The configuration of the Elasticsearch sink connector has the following properti
| `compatibilityMode` | enum (AUTO,ELASTICSEARCH,ELASTICSEARCH_7,OPENSEARCH) | AUTO | | Specify compatibility mode with the ElasticSearch cluster. `AUTO` value will try to auto detect the correct compatibility mode to use. Use `ELASTICSEARCH_7` if the target cluster is running ElasticSearch 7 or prior. Use `ELASTICSEARCH` if the target cluster is running ElasticSearch 8 or higher. Use `OPENSEARCH` if the target cluster is running OpenSearch. |
| `token` | String| false | " " (empty string)|The token used by the connector to connect to the ElasticSearch cluster. Only one between basic/token/apiKey authentication mode must be configured. |
| `apiKey` | String| false | " " (empty string)|The apiKey used by the connector to connect to the ElasticSearch cluster. Only one between basic/token/apiKey authentication mode must be configured. |
+| `canonicalKeyFields` | Boolean | false | false | Whether to sort the key fields for JSON and Avro or not. If it is set to `true` and the record key schema is `JSON` or `AVRO`, the serialized object does not consider the order of properties. |
### Definition of ElasticSearchSslConfig structure: