You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by ni...@apache.org on 2022/05/23 08:51:23 UTC

[pulsar] branch master updated: [feature][connector] ElasticSearch Sink: option to output canonical key fields (JSON and Avro) (#15426)

This is an automated email from the ASF dual-hosted git repository.

nicoloboschi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new a4488fd2e5a [feature][connector] ElasticSearch Sink: option to output canonical key fields (JSON and Avro) (#15426)
a4488fd2e5a is described below

commit a4488fd2e5a07e1aa25ccc78ee4af9b782137424
Author: Nicolò Boschi <bo...@gmail.com>
AuthorDate: Mon May 23 10:51:12 2022 +0200

    [feature][connector] ElasticSearch Sink: option to output canonical key fields (JSON and Avro) (#15426)
    
    * [feat][elasticsearch-sink] Option to output canonical JSON
    
    * [feat][elasticsearch-sink] Option to output canonical JSON
    
    * style
    
    * fix test
    
    * style
    
    * correct doc
    
    * Update site2/docs/io-elasticsearch-sink.md
    
    Co-authored-by: momo-jun <60...@users.noreply.github.com>
    
    Co-authored-by: momo-jun <60...@users.noreply.github.com>
---
 .../io/elasticsearch/ElasticSearchConfig.java      |   7 ++
 .../pulsar/io/elasticsearch/ElasticSearchSink.java |  44 +++++--
 .../pulsar/io/elasticsearch/JsonConverter.java     |   2 +-
 .../elasticsearch/ElasticSearchExtractTests.java   | 140 +++++++++++++++++++++
 site2/docs/io-elasticsearch-sink.md                |   1 +
 5 files changed, 186 insertions(+), 8 deletions(-)

diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java
index 77bf89cf8f9..396e892fc12 100644
--- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java
+++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchConfig.java
@@ -274,6 +274,13 @@ public class ElasticSearchConfig implements Serializable {
     )
     private CompatibilityMode compatibilityMode = CompatibilityMode.AUTO;
 
+    @FieldDoc(
+            defaultValue = "false",
+            help = "If canonicalKeyFields is true and record key schema is JSON or AVRO, the serialized object will "
+                    + "not consider the properties order."
+    )
+    private boolean canonicalKeyFields = false;
+
     public enum MalformedDocAction {
         IGNORE,
         WARN,
diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java
index dc82ee60e65..9f703a59ecb 100644
--- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java
+++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchSink.java
@@ -21,6 +21,10 @@ package org.apache.pulsar.io.elasticsearch;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import java.nio.charset.StandardCharsets;
@@ -29,6 +33,7 @@ import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pulsar.client.api.Message;
@@ -56,7 +61,8 @@ public class ElasticSearchSink implements Sink<GenericObject> {
 
     private ElasticSearchConfig elasticSearchConfig;
     private ElasticSearchClient elasticsearchClient;
-    private ObjectMapper objectMapper = new ObjectMapper();
+    private final ObjectMapper objectMapper = new ObjectMapper();
+    private ObjectMapper sortedObjectMapper;
     private List<String> primaryFields = null;
 
     @Override
@@ -67,6 +73,19 @@ public class ElasticSearchSink implements Sink<GenericObject> {
         if (!Strings.isNullOrEmpty(elasticSearchConfig.getPrimaryFields())) {
             primaryFields = Arrays.asList(elasticSearchConfig.getPrimaryFields().split(","));
         }
+        if (elasticSearchConfig.isCanonicalKeyFields()) {
+            sortedObjectMapper = JsonMapper
+                    .builder()
+                            .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)
+                    .nodeFactory(new JsonNodeFactory() {
+                        @Override
+                        public ObjectNode objectNode() {
+                            return new ObjectNode(this, new TreeMap<String, JsonNode>());
+                        }
+
+                    })
+                    .build();
+        }
     }
 
     @Override
@@ -238,9 +257,11 @@ public class ElasticSearchSink implements Sink<GenericObject> {
         }
     }
 
+
     /**
      * Convert a JsonNode to an Elasticsearch id.
      */
+
     public String stringifyKey(JsonNode jsonNode) throws JsonProcessingException {
         List<String> fields = new ArrayList<>();
         jsonNode.fieldNames().forEachRemaining(fields::add);
@@ -248,15 +269,24 @@ public class ElasticSearchSink implements Sink<GenericObject> {
     }
 
     public String stringifyKey(JsonNode jsonNode, List<String> fields) throws JsonProcessingException {
+        JsonNode toConvert;
         if (fields.size() == 1) {
-            JsonNode singleNode = jsonNode.get(fields.get(0));
-            String id = objectMapper.writeValueAsString(singleNode);
-            return (id.startsWith("\"") && id.endsWith("\""))
-                    ? id.substring(1, id.length() - 1)  // remove double quotes
-                    : id;
+            toConvert = jsonNode.get(fields.get(0));
         } else {
-            return JsonConverter.toJsonArray(jsonNode, fields).toString();
+            toConvert = JsonConverter.toJsonArray(jsonNode, fields);
         }
+
+        String serializedId;
+        if (elasticSearchConfig.isCanonicalKeyFields()) {
+            final Object obj = sortedObjectMapper.treeToValue(toConvert, Object.class);
+            serializedId = sortedObjectMapper.writeValueAsString(obj);
+        } else {
+            serializedId = objectMapper.writeValueAsString(toConvert);
+        }
+
+        return (serializedId.startsWith("\"") && serializedId.endsWith("\""))
+                ? serializedId.substring(1, serializedId.length() - 1)  // remove double quotes
+                : serializedId;
     }
 
     public String stringifyValue(Schema<?> schema, Object val) throws JsonProcessingException {
diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java
index 8c1c51c9deb..a7d15914044 100644
--- a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java
+++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/JsonConverter.java
@@ -212,7 +212,7 @@ public class JsonConverter {
 
     public static ArrayNode toJsonArray(JsonNode jsonNode, List<String> fields) {
         ArrayNode arrayNode = jsonNodeFactory.arrayNode();
-        Iterator<String>  it = jsonNode.fieldNames();
+        Iterator<String> it = jsonNode.fieldNames();
         while (it.hasNext()) {
             String fieldName = it.next();
             if (fields.contains(fieldName)) {
diff --git a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
index 518b72aa3bd..f99316eb940 100644
--- a/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
+++ b/pulsar-io/elastic-search/src/test/java/org/apache/pulsar/io/elasticsearch/ElasticSearchExtractTests.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.io.elasticsearch;
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNull;
+
 import com.google.common.collect.ImmutableMap;
 import java.util.Optional;
 import org.apache.commons.lang3.tuple.Pair;
@@ -270,4 +271,143 @@ public class ElasticSearchExtractTests {
         assertEquals(pair3.getLeft(), "[\"1\",1]");
         assertNull(pair3.getRight());
     }
+
+    @Test(dataProvider = "schemaType")
+    public void testSortKeysSingle(SchemaType schemaType) throws Exception {
+        RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
+
+        RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1");
+        udtSchemaBuilder.field("b_inside_inner").type(SchemaType.STRING).optional().defaultValue(null);
+        udtSchemaBuilder.field("a_inside_inner").type(SchemaType.STRING).optional().defaultValue(null);
+        GenericSchema<GenericRecord> udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType));
+
+        keySchemaBuilder.field("singleKey", udtGenericSchema).type(schemaType).optional().defaultValue(null);
+        GenericSchema<GenericRecord> keySchema = Schema.generic(keySchemaBuilder.build(schemaType));
+
+        final GenericRecord innerRecord = udtGenericSchema.newRecordBuilder()
+                .set("b_inside_inner", "0b_value_from_inner")
+                .set("a_inside_inner", "a_value_from_inner")
+                .build();
+
+        GenericRecord keyGenericRecord = keySchema.newRecordBuilder()
+                .set("singleKey", innerRecord)
+                .build();
+
+        Record<GenericObject> genericObjectRecord = getKeyValueGenericObject(schemaType, keySchema, keyGenericRecord);
+
+
+        ElasticSearchSink elasticSearchSink = new ElasticSearchSink();
+        elasticSearchSink.open(ImmutableMap.of(
+                "elasticSearchUrl", "http://localhost:9200",
+                "compatibilityMode", "ELASTICSEARCH",
+                "schemaEnable", "true",
+                "canonicalKeyFields", "false",
+                "keyIgnore", "false"), null);
+        Pair<String, String> pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
+        assertEquals(pair.getKey(), "{\"b_inside_inner\":\"0b_value_from_inner\",\"a_inside_inner\":\"a_value_from_inner\"}");
+
+        elasticSearchSink = new ElasticSearchSink();
+        elasticSearchSink.open(ImmutableMap.of(
+                "elasticSearchUrl", "http://localhost:9200",
+                "compatibilityMode", "ELASTICSEARCH",
+                "schemaEnable", "true",
+                "canonicalKeyFields", "true",
+                "keyIgnore", "false"), null);
+        pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
+        assertEquals(pair.getKey(), "{\"a_inside_inner\":\"a_value_from_inner\",\"b_inside_inner\":\"0b_value_from_inner\"}");
+
+    }
+
+    @Test(dataProvider = "schemaType")
+    public void testSortKeysMulti(SchemaType schemaType) throws Exception {
+        RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
+        keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
+        keySchemaBuilder.field("b").type(SchemaType.STRING).optional().defaultValue(null);
+        keySchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null);
+
+        RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1");
+        udtSchemaBuilder.field("b_inside_inner").type(SchemaType.STRING).optional().defaultValue(null);
+        udtSchemaBuilder.field("a_inside_inner").type(SchemaType.STRING).optional().defaultValue(null);
+        GenericSchema<GenericRecord> udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType));
+
+        keySchemaBuilder.field("inner", udtGenericSchema).type(schemaType).optional().defaultValue(null);
+        GenericSchema<GenericRecord> keySchema = Schema.generic(keySchemaBuilder.build(schemaType));
+
+        final GenericRecord innerRecord = udtGenericSchema.newRecordBuilder()
+                .set("b_inside_inner", "0b_value_from_inner")
+                .set("a_inside_inner", "a_value_from_inner")
+                .build();
+
+        GenericRecord keyGenericRecord = keySchema.newRecordBuilder()
+                .set("c", "c_key")
+                .set("b", "0b_key")
+                .set("a", "a_key")
+                .set("inner", innerRecord)
+                .build();
+
+        Record<GenericObject> genericObjectRecord = getKeyValueGenericObject(schemaType, keySchema, keyGenericRecord);
+
+
+        ElasticSearchSink elasticSearchSink = new ElasticSearchSink();
+        elasticSearchSink.open(ImmutableMap.of(
+                "elasticSearchUrl", "http://localhost:9200",
+                "compatibilityMode", "ELASTICSEARCH",
+                "schemaEnable", "true",
+                "canonicalKeyFields", "false",
+                "keyIgnore", "false"), null);
+        Pair<String, String> pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
+        assertEquals(pair.getKey(), "[\"a_key\",\"0b_key\",\"c_key\",{\"b_inside_inner\":\"0b_value_from_inner\",\"a_inside_inner\":\"a_value_from_inner\"}]");
+
+        elasticSearchSink = new ElasticSearchSink();
+        elasticSearchSink.open(ImmutableMap.of(
+                "elasticSearchUrl", "http://localhost:9200",
+                "compatibilityMode", "ELASTICSEARCH",
+                "schemaEnable", "true",
+                "canonicalKeyFields", "true",
+                "keyIgnore", "false"), null);
+        pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
+        assertEquals(pair.getKey(), "[\"a_key\",\"0b_key\",\"c_key\",{\"a_inside_inner\":\"a_value_from_inner\",\"b_inside_inner\":\"0b_value_from_inner\"}]");
+    }
+
+    private Record<GenericObject> getKeyValueGenericObject(SchemaType schemaType, GenericSchema<GenericRecord> keySchema, GenericRecord keyGenericRecord) {
+        RecordSchemaBuilder valueSchemaBuilder = SchemaBuilder.record("value");
+        valueSchemaBuilder.field("value").type(SchemaType.STRING);
+        GenericSchema<GenericRecord> valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType));
+
+        GenericRecord valueGenericRecord = valueSchema.newRecordBuilder()
+                .set("value", "value")
+                .build();
+
+        Schema<KeyValue<GenericRecord, GenericRecord>> keyValueSchema =
+                Schema.KeyValue(keySchema, valueSchema, KeyValueEncodingType.INLINE);
+        KeyValue<GenericRecord, GenericRecord> keyValue = new KeyValue<>(keyGenericRecord, valueGenericRecord);
+        GenericObject genericObject = new GenericObject() {
+            @Override
+            public SchemaType getSchemaType() {
+                return SchemaType.KEY_VALUE;
+            }
+
+            @Override
+            public Object getNativeObject() {
+                return keyValue;
+            }
+        };
+        Record<GenericObject> genericObjectRecord = new Record<GenericObject>() {
+            @Override
+            public Optional<String> getTopicName() {
+                return Optional.of("data-ks1.table1");
+            }
+
+            @Override
+            public Schema  getSchema() {
+                return keyValueSchema;
+            }
+
+            @Override
+            public GenericObject getValue() {
+                return genericObject;
+            }
+        };
+        return genericObjectRecord;
+    }
 }
\ No newline at end of file
diff --git a/site2/docs/io-elasticsearch-sink.md b/site2/docs/io-elasticsearch-sink.md
index ada9ce71916..26ea81f6026 100644
--- a/site2/docs/io-elasticsearch-sink.md
+++ b/site2/docs/io-elasticsearch-sink.md
@@ -86,6 +86,7 @@ The configuration of the Elasticsearch sink connector has the following properti
 | `compatibilityMode` | enum (AUTO,ELASTICSEARCH,ELASTICSEARCH_7,OPENSEARCH) | AUTO |  | Specify compatibility mode with the ElasticSearch cluster. `AUTO` value will try to auto detect the correct compatibility mode to use. Use `ELASTICSEARCH_7` if the target cluster is running ElasticSearch 7 or prior. Use `ELASTICSEARCH` if the target cluster is running ElasticSearch 8 or higher. Use `OPENSEARCH` if the target cluster is running OpenSearch. |
 | `token` | String| false | " " (empty string)|The token used by the connector to connect to the ElasticSearch cluster. Only one between basic/token/apiKey authentication mode must be configured. |
 | `apiKey` | String| false | " " (empty string)|The apiKey used by the connector to connect to the ElasticSearch cluster. Only one between basic/token/apiKey authentication mode must be configured. |
+| `canonicalKeyFields` | Boolean | false | false | Whether to sort the key fields for JSON and Avro or not. If it is set to `true` and the record key schema is `JSON` or `AVRO`, the serialized object does not consider the order of properties. |
 
 ### Definition of ElasticSearchSslConfig structure: