You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ar...@apache.org on 2020/07/20 06:44:31 UTC

[metamodel] 02/05: Introduced a workaround for the fact that Elasticsearch returns data based on indexed documents with dots in the field names in an inconsistent manner. I.e the mapping it returns for such documents indicate those kind of fields are treated as nested objects. But if you get the source map of such a document from Elasticsearch, it still contains the original fieldnames (with dots in them).

This is an automated email from the ASF dual-hosted git repository.

arjansh pushed a commit to annotated tag MetaModel-5.3.3-HI1
in repository https://gitbox.apache.org/repos/asf/metamodel.git

commit 25f665b12abd1379b7f3f8fe258f043a4ac1e2d3
Author: Arjan Seijkens <a....@quadient.com>
AuthorDate: Mon Jul 6 13:10:43 2020 +0200

    Introduced a workaround for the fact that Elasticsearch returns data based on indexed documents with dots in the field names in an inconsistent manner. I.e the mapping it returns for such documents indicate those kind of fields are treated as nested objects. But if you get the source map of such a document from Elasticsearch, it still contains the original fieldnames (with dots in them).
---
 CHANGES.md                                         |  1 +
 .../elasticsearch/common/ElasticSearchUtils.java   | 39 ++++++++++++++
 .../rest/ElasticSearchRestNestedDataIT.java        | 63 +++++++++++-----------
 3 files changed, 70 insertions(+), 33 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 41fb5bd..67fcf4c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,6 @@
 ### Apache MetaModel [WIP]
 
+ * [METAMODEL-1228] - Better handling of fieldnames with dots in Elasticsearch
  * [METAMODEL-1227] - Better handling of nested objects in Elasticsearch data
  * [METAMODEL-1224] - Ensured compatibility with newer versions of PostgreSQL
 
diff --git a/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java b/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
index 6ab4fa6..57ca7a1 100644
--- a/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
+++ b/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
@@ -290,6 +290,23 @@ public class ElasticSearchUtils {
                         } else {
                             values[i] = valueToDate;
                         }
+                    } else if (column.getType() == ColumnType.MAP && value == null) {
+                        // Because of a bug in Elasticsearch, when field names contain dots, it's possible that the
+                        // mapping of the index described a column to be of the type "MAP", while it's based on a number
+                        // of fields contains dots in their name. In this case we may have to work around that
+                        // inconsistency by creating column names with dots ourselves, based on the schema.
+                        final Map<String, Object> valueMap = new HashMap<>();
+
+                        sourceMap
+                                .keySet()
+                                .stream()
+                                .filter(fieldName -> fieldName.startsWith(column.getName() + "."))
+                                .forEach(fieldName -> evaluateField(sourceMap, valueMap, fieldName, fieldName
+                                        .substring(fieldName.indexOf('.') + 1)));
+
+                        if (!valueMap.isEmpty()) {
+                            values[i] = valueMap;
+                        }
                     } else {
                         values[i] = value;
                     }
@@ -299,4 +316,26 @@ public class ElasticSearchUtils {
 
         return new DefaultRow(header, values);
     }
+
+    private static void evaluateField(final Map<String, Object> sourceMap, final Map<String, Object> valueMap,
+            final String sourceFieldName, final String subFieldName) {
+        if (subFieldName.contains(".")) {
+            @SuppressWarnings("unchecked")
+            final Map<String, Object> nestedValueMap = (Map<String, Object>) valueMap
+                    .computeIfAbsent(subFieldName.substring(0, subFieldName.indexOf('.')), key -> createNestedValueMap(
+                            valueMap, key));
+
+            evaluateField(sourceMap, nestedValueMap, sourceFieldName, subFieldName
+                    .substring(subFieldName.indexOf('.') + 1));
+        } else {
+            valueMap.put(subFieldName, sourceMap.get(sourceFieldName));
+        }
+    }
+
+    private static Object createNestedValueMap(final Map<String, Object> valueMap, final String nestedFieldName) {
+        final Map<String, Object> nestedValueMap = new HashMap<>();
+        valueMap.put(nestedFieldName, nestedValueMap);
+
+        return nestedValueMap;
+    }
 }
diff --git a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
index 1265d61..25b0622 100644
--- a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
+++ b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
@@ -69,9 +69,13 @@ public class ElasticSearchRestNestedDataIT {
 
     @Test
     public void testNestedData() throws Exception {
+        final Map<String, Object> address = new HashMap<>();
+        address.put("street", "Main street 1");
+        address.put("city", "Newville");
+
         final Map<String, Object> user = new HashMap<>();
         user.put("fullname", "John Doe");
-        user.put("address", "Main street 1, Newville");
+        user.put("address", address);
 
         final Map<String, Object> userMessage = new LinkedHashMap<>();
         userMessage.put("user", user);
@@ -82,42 +86,15 @@ public class ElasticSearchRestNestedDataIT {
 
         client.index(indexRequest, RequestOptions.DEFAULT);
 
-        final Table table = dataContext.getDefaultSchema().getTableByName(DEFAULT_TABLE_NAME);
-
-        assertThat(table.getColumnNames(), containsInAnyOrder("_id", "message", "user"));
-
-        assertEquals(ColumnType.STRING, table.getColumnByName("message").getType());
-        assertEquals(ColumnType.MAP, table.getColumnByName("user").getType());
-
-        dataContext.refreshSchemas();
-        
-        try (final DataSet dataSet = dataContext
-                .query()
-                .from(DEFAULT_TABLE_NAME)
-                .select("user")
-                .and("message")
-                .execute()) {
-            assertEquals(ElasticSearchRestDataSet.class, dataSet.getClass());
-
-            assertTrue(dataSet.next());
-            final Row row = dataSet.getRow();
-            assertEquals("This is what I have to say.", row.getValue(table.getColumnByName("message")));
-            
-            final Object userValue = row.getValue(table.getColumnByName("user"));
-            assertTrue(userValue instanceof Map);
-
-            @SuppressWarnings("rawtypes")
-            final Map userValueMap = (Map) userValue;
-            assertEquals("John Doe", userValueMap.get("fullname"));
-            assertEquals("Main street 1, Newville", userValueMap.get("address"));
-        }
+        validateSchemaAndResults();
     }
 
     @Test
     public void testIndexOfDocumentWithDots() throws Exception {
         final String document =
                 "{ \"user.fullname\": \"John Doe\", "
-                + "\"user.address\": \"Main street 1, Newville\", "
+                + "\"user.address.street\": \"Main street 1\", "
+                + "\"user.address.city\": \"Newville\", "
                 + "\"message\": \"This is what I have to say.\" }";
 
         final IndexRequest indexRequest = new IndexRequest(INDEX_NAME).id("1");
@@ -125,12 +102,19 @@ public class ElasticSearchRestNestedDataIT {
 
         client.index(indexRequest, RequestOptions.DEFAULT);
 
+        validateSchemaAndResults();
+    }
+
+    private void validateSchemaAndResults() {
         final Table table = dataContext.getDefaultSchema().getTableByName(DEFAULT_TABLE_NAME);
 
         assertThat(table.getColumnNames(), containsInAnyOrder("_id", "message", "user"));
 
-        dataContext.refreshSchemas();
+        assertEquals(ColumnType.STRING, table.getColumnByName("message").getType());
+        assertEquals(ColumnType.MAP, table.getColumnByName("user").getType());
 
+        dataContext.refreshSchemas();
+        
         try (final DataSet dataSet = dataContext
                 .query()
                 .from(DEFAULT_TABLE_NAME)
@@ -142,8 +126,21 @@ public class ElasticSearchRestNestedDataIT {
             assertTrue(dataSet.next());
             final Row row = dataSet.getRow();
             assertEquals("This is what I have to say.", row.getValue(table.getColumnByName("message")));
+            
+            final Object userValue = row.getValue(table.getColumnByName("user"));
+            assertTrue(userValue instanceof Map);
 
-            assertNotNull(row.getValue(table.getColumnByName("user")));
+            @SuppressWarnings("rawtypes")
+            final Map userValueMap = (Map) userValue;
+            assertEquals("John Doe", userValueMap.get("fullname"));
+
+            final Object addressValue = userValueMap.get("address");
+            assertTrue(addressValue instanceof Map);
+
+            @SuppressWarnings("rawtypes")
+            final Map addressValueMap = (Map) addressValue;
+            assertEquals("Main street 1", addressValueMap.get("street"));
+            assertEquals("Newville", addressValueMap.get("city"));
         }
     }
 }