You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ar...@apache.org on 2020/07/20 06:43:57 UTC
[metamodel] branch master updated (c3797d3 -> 5dfb573)
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git.
from c3797d3 Merge remote-tracking branch 'tkingsbury42/master'
new e60bf6b Added integration tests that prove MetaModel doesn't handle nested data in Elasticsearch in a correct manner.
new bf7a78e Pushed changes to better indicate the problem.
new bb4ae06 Fixed resolution of type of nested objects for Elasticsearch.
new 27fbbb4 Updated CHANGES.md
new dc997ec Cosmetic change to force rebuild on Travis.
new 874a163 Merge branch 'master' into bug/nested-data-in-elasticsearch
new 5dfb573 Merge remote-tracking branch 'arjansh/bug/nested-data-in-elasticsearch'
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.md | 1 +
.../common/ElasticSearchMetaDataParser.java | 8 +-
.../rest/ElasticSearchRestNestedDataIT.java | 113 +++++++++++++++++++++
3 files changed, 121 insertions(+), 1 deletion(-)
create mode 100644 elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
[metamodel] 05/07: Cosmetic change to force rebuild on Travis.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit dc997ec3c809e1661693c73ca14f8f5bd2ad0f34
Author: Arjan Seijkens <a....@quadient.com>
AuthorDate: Wed Jul 15 10:16:22 2020 +0200
Cosmetic change to force rebuild on Travis.
---
.../metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
index 890cda7..0035679 100644
--- a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
+++ b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
@@ -46,7 +46,6 @@ public class ElasticSearchRestNestedDataIT {
private static final String INDEX_NAME = "nesteddata";
private static RestHighLevelClient client;
-
private static UpdateableDataContext dataContext;
@Before
[metamodel] 07/07: Merge remote-tracking branch
'arjansh/bug/nested-data-in-elasticsearch'
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit 5dfb5731731caed6b8ef17a9719370893ea28e10
Merge: c3797d3 874a163
Author: Arjan Seijkens <a....@quadient.com>
AuthorDate: Mon Jul 20 08:26:22 2020 +0200
Merge remote-tracking branch 'arjansh/bug/nested-data-in-elasticsearch'
CHANGES.md | 1 +
.../common/ElasticSearchMetaDataParser.java | 8 +-
.../rest/ElasticSearchRestNestedDataIT.java | 113 +++++++++++++++++++++
3 files changed, 121 insertions(+), 1 deletion(-)
[metamodel] 03/07: Fixed resolution of type of nested objects for
Elasticsearch.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit bb4ae066218617c961bd2980069c30c7095fbc6e
Author: Arjan Seijkens <a....@quadient.com>
AuthorDate: Fri Jul 3 13:52:08 2020 +0200
Fixed resolution of type of nested objects for Elasticsearch.
---
.../common/ElasticSearchMetaDataParser.java | 8 +++-
.../rest/ElasticSearchRestNestedDataIT.java | 46 ++++------------------
2 files changed, 15 insertions(+), 39 deletions(-)
diff --git a/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchMetaDataParser.java b/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchMetaDataParser.java
index 32f07ff..6979c54 100644
--- a/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchMetaDataParser.java
+++ b/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchMetaDataParser.java
@@ -62,7 +62,13 @@ public class ElasticSearchMetaDataParser {
final String metaDataFieldType = getMetaDataFieldTypeFromMetaDataField(fieldMetadata);
if (metaDataFieldType == null) {
- return ColumnType.STRING;
+ // When we're dealing with a field with nested values, it doesn't have a "type", but it has "properties". In
+ // that case we should handle it as a field with the type "Map".
+ if (fieldMetadata.get("properties") != null) {
+ return ColumnType.MAP;
+ } else {
+ return ColumnType.STRING;
+ }
}
return ElasticSearchUtils.getColumnTypeFromElasticSearchType(metaDataFieldType);
diff --git a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
index fc5ca69..890cda7 100644
--- a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
+++ b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
@@ -38,7 +38,6 @@ import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
-import org.elasticsearch.common.xcontent.XContentType;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -86,45 +85,11 @@ public class ElasticSearchRestNestedDataIT {
assertThat(table.getColumnNames(), containsInAnyOrder("_id", "message", "user"));
- dataContext.refreshSchemas();
-
- try (final DataSet dataSet = dataContext
- .query()
- .from(DEFAULT_TABLE_NAME)
- .select("user")
- .and("message")
- .execute()) {
- assertEquals(ElasticSearchRestDataSet.class, dataSet.getClass());
-
- assertTrue(dataSet.next());
- final Row row = dataSet.getRow();
- assertEquals("This is what I have to say.", row.getValue(table.getColumnByName("message")));
-
- assertTrue(row.getValue(table.getColumnByName("user")) instanceof Map);
- }
-
assertEquals(ColumnType.STRING, table.getColumnByName("message").getType());
assertEquals(ColumnType.MAP, table.getColumnByName("user").getType());
- }
-
- @Test
- public void testIndexOfDocumentWithDots() throws Exception {
- final String document =
- "{ \"user.fullname\": \"John Doe\", "
- + "\"user.address\": \"Main street 1, Newville\", "
- + "\"message\": \"This is what I have to say.\" }";
-
- final IndexRequest indexRequest = new IndexRequest(INDEX_NAME).id("1");
- indexRequest.source(document, XContentType.JSON);
-
- client.index(indexRequest, RequestOptions.DEFAULT);
-
- final Table table = dataContext.getDefaultSchema().getTableByName(DEFAULT_TABLE_NAME);
-
- assertThat(table.getColumnNames(), containsInAnyOrder("_id", "message", "user"));
dataContext.refreshSchemas();
-
+
try (final DataSet dataSet = dataContext
.query()
.from(DEFAULT_TABLE_NAME)
@@ -136,9 +101,14 @@ public class ElasticSearchRestNestedDataIT {
assertTrue(dataSet.next());
final Row row = dataSet.getRow();
assertEquals("This is what I have to say.", row.getValue(table.getColumnByName("message")));
+
+ final Object userValue = row.getValue(table.getColumnByName("user"));
+ assertTrue(userValue instanceof Map);
- assertNotNull(row.getValue(table.getColumnByName("user")));
+ @SuppressWarnings("rawtypes")
+ final Map userValueMap = (Map) userValue;
+ assertEquals("John Doe", userValueMap.get("fullname"));
+ assertEquals("Main street 1, Newville", userValueMap.get("address"));
}
}
-
}
[metamodel] 06/07: Merge branch 'master' into
bug/nested-data-in-elasticsearch
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit 874a1639a840c6c2cf74d8bc82b60c1b9edbd88d
Merge: dc997ec c3797d3
Author: Arjan Seijkens <a....@quadient.com>
AuthorDate: Thu Jul 16 11:16:07 2020 +0200
Merge branch 'master' into bug/nested-data-in-elasticsearch
.../java/org/apache/metamodel/schema/Column.java | 8 ++
.../apache/metamodel/schema/ImmutableColumn.java | 49 +++++++++++-
.../org/apache/metamodel/schema/MutableColumn.java | 25 +++++-
.../apache/metamodel/jdbc/JdbcMetadataLoader.java | 88 ++++++++++++----------
4 files changed, 124 insertions(+), 46 deletions(-)
[metamodel] 04/07: Updated CHANGES.md
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit 27fbbb42717092de331048d1262dd5eb8ec3ee6f
Author: Arjan Seijkens <a....@quadient.com>
AuthorDate: Fri Jul 3 13:57:23 2020 +0200
Updated CHANGES.md
---
CHANGES.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGES.md b/CHANGES.md
index d843e20..41fb5bd 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,6 @@
### Apache MetaModel [WIP]
+ * [METAMODEL-1227] - Better handling of nested objects in Elasticsearch data
* [METAMODEL-1224] - Ensured compatibility with newer versions of PostgreSQL
### Apache MetaModel 5.3.2
[metamodel] 01/07: Added integration tests that prove MetaModel
doesn't handle nested data in Elasticsearch in a correct manner.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit e60bf6b5e1b83106323db2ee578cb4f1baf2c430
Author: Arjan Seijkens <a....@quadient.com>
AuthorDate: Fri Jul 3 11:04:00 2020 +0200
Added integration tests that prove MetaModel doesn't handle nested data in Elasticsearch in a correct manner.
---
.../rest/ElasticSearchRestNestedDataIT.java | 127 +++++++++++++++++++++
1 file changed, 127 insertions(+)
diff --git a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
new file mode 100644
index 0000000..0a20676
--- /dev/null
+++ b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.elasticsearch.rest;
+
+import static org.apache.metamodel.elasticsearch.rest.ElasticSearchRestDataContext.DEFAULT_TABLE_NAME;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.http.HttpHost;
+import org.apache.metamodel.UpdateableDataContext;
+import org.apache.metamodel.data.DataSet;
+import org.apache.metamodel.data.Row;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.Table;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ElasticSearchRestNestedDataIT {
+ private static final String INDEX_NAME = "nesteddata";
+
+ private static RestHighLevelClient client;
+
+ private static UpdateableDataContext dataContext;
+
+ @Before
+ public void setUp() throws Exception {
+ final String dockerHostAddress = ElasticSearchRestDataContextIT.determineHostName();
+
+ client = ElasticSearchRestUtil
+ .createClient(new HttpHost(dockerHostAddress, ElasticSearchRestDataContextIT.DEFAULT_REST_CLIENT_PORT),
+ null, null);
+ client.indices().create(new CreateIndexRequest(INDEX_NAME), RequestOptions.DEFAULT);
+
+ dataContext = new ElasticSearchRestDataContext(client, INDEX_NAME);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ client.indices().delete(new DeleteIndexRequest(INDEX_NAME), RequestOptions.DEFAULT);
+ }
+
+ @Test
+ public void testNestedData() throws Exception {
+ final Map<String, Object> user = new HashMap<>();
+ user.put("fullname", "John Doe");
+ user.put("address", "Main street 1, Newville");
+
+ final Map<String, Object> userMessage = new LinkedHashMap<>();
+ userMessage.put("user", user);
+ userMessage.put("message", "This is what I have to say.");
+
+ final IndexRequest indexRequest = new IndexRequest(INDEX_NAME).id("1");
+ indexRequest.source(userMessage);
+
+ client.index(indexRequest, RequestOptions.DEFAULT);
+
+ final Table table = dataContext.getDefaultSchema().getTableByName(DEFAULT_TABLE_NAME);
+
+ assertThat(table.getColumnNames(), containsInAnyOrder("_id", "message", "user"));
+
+ assertEquals(ColumnType.MAP, table.getColumnByName("user").getType());
+ assertEquals(ColumnType.STRING, table.getColumnByName("message").getType());
+ }
+
+ @Test
+ public void testIndexOfDocumentWithDots() throws Exception {
+ final String document =
+ "{ \"user.fullname\": \"John Doe\", "
+ + "\"user.address\": \"Main street 1, Newville\", "
+ + "\"message\": \"This is what I have to say.\" }";
+
+ final IndexRequest indexRequest = new IndexRequest(INDEX_NAME).id("1");
+ indexRequest.source(document, XContentType.JSON);
+
+ client.index(indexRequest, RequestOptions.DEFAULT);
+
+ final Table table = dataContext.getDefaultSchema().getTableByName(DEFAULT_TABLE_NAME);
+
+ assertThat(table.getColumnNames(), containsInAnyOrder("_id", "message", "user"));
+
+ dataContext.refreshSchemas();
+
+ try (final DataSet dataSet = dataContext
+ .query()
+ .from(DEFAULT_TABLE_NAME)
+ .select("user")
+ .and("message")
+ .execute()) {
+ assertEquals(ElasticSearchRestDataSet.class, dataSet.getClass());
+
+ assertTrue(dataSet.next());
+ final Row row = dataSet.getRow();
+ assertEquals("This is what I have to say.", row.getValue(table.getColumnByName("message")));
+
+ assertNotNull(row.getValue(table.getColumnByName("user")));
+ }
+ }
+
+}
[metamodel] 02/07: Pushed changes to better indicate the problem.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit bf7a78e283bb7842cb86a0abe293452f942cb111
Author: Arjan Seijkens <a....@quadient.com>
AuthorDate: Fri Jul 3 12:25:12 2020 +0200
Pushed changes to better indicate the problem.
---
.../rest/ElasticSearchRestNestedDataIT.java | 19 ++++++++++++++++++-
1 file changed, 18 insertions(+), 1 deletion(-)
diff --git a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
index 0a20676..fc5ca69 100644
--- a/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
+++ b/elasticsearch/rest/src/test/java/org/apache/metamodel/elasticsearch/rest/ElasticSearchRestNestedDataIT.java
@@ -86,8 +86,25 @@ public class ElasticSearchRestNestedDataIT {
assertThat(table.getColumnNames(), containsInAnyOrder("_id", "message", "user"));
- assertEquals(ColumnType.MAP, table.getColumnByName("user").getType());
+ dataContext.refreshSchemas();
+
+ try (final DataSet dataSet = dataContext
+ .query()
+ .from(DEFAULT_TABLE_NAME)
+ .select("user")
+ .and("message")
+ .execute()) {
+ assertEquals(ElasticSearchRestDataSet.class, dataSet.getClass());
+
+ assertTrue(dataSet.next());
+ final Row row = dataSet.getRow();
+ assertEquals("This is what I have to say.", row.getValue(table.getColumnByName("message")));
+
+ assertTrue(row.getValue(table.getColumnByName("user")) instanceof Map);
+ }
+
assertEquals(ColumnType.STRING, table.getColumnByName("message").getType());
+ assertEquals(ColumnType.MAP, table.getColumnByName("user").getType());
}
@Test