You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ar...@apache.org on 2021/02/17 08:11:21 UTC
[metamodel] branch master updated (55e5995 -> 0cbf4ae)
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git.
from 55e5995 Reorganize imports and remove extra whitespace
new 8386549 MM-1233: Elasticsearch uses 'match' query for text based columns conditions in WHERE clause.
new a17e2d6 MM-1233: termQuery for IN operator.
new 142c4ae MM-1233: Refactoring, DIFFERENT_FROM operator.
new 2f31f6e METAMODEL-1233: JUnit tests were extended.
new 0cbf4ae MM-1233: CHANGES.md update.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.md | 1 +
.../elasticsearch/common/ElasticSearchUtils.java | 112 ++++++++++++---------
.../common/ElasticSearchUtilsTest.java | 74 +++++++++++++-
3 files changed, 139 insertions(+), 48 deletions(-)
[metamodel] 05/05: MM-1233: CHANGES.md update.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit 0cbf4aeab2b5c65c70775ee7057a73b634b6e047
Author: jakub <j....@quadient.com>
AuthorDate: Wed Feb 17 08:11:05 2021 +0100
MM-1233: CHANGES.md update.
---
CHANGES.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGES.md b/CHANGES.md
index b23f663..770f856 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,6 @@
### Apache MetaModel [WIP]
+ * [METAMODEL-1233] - Elasticsearch: "term" query is used for text search.
* [METAMODEL-1229] - FixedWidth works with UrlResource/HttpInputStream.
* [METAMODEL-1228] - Better handling of fieldnames with dots in Elasticsearch
* [METAMODEL-1227] - Better handling of nested objects in Elasticsearch data
[metamodel] 02/05: MM-1233: termQuery for IN operator.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit a17e2d684f448153aea92a8857cb0e259f3d7e5d
Author: jakub <j....@quadient.com>
AuthorDate: Mon Feb 15 08:46:38 2021 +0100
MM-1233: termQuery for IN operator.
---
.../apache/metamodel/elasticsearch/common/ElasticSearchUtils.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java b/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
index b2c1f27..5458174 100644
--- a/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
+++ b/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
@@ -216,7 +216,11 @@ public class ElasticSearchUtils {
}
} else if (OperatorType.IN.equals(operator)) {
final List<?> operands = CollectionUtils.toList(operand);
- itemQueryBuilder = QueryBuilders.termsQuery(fieldName, operands);
+ if (column.getType().isLiteral()) {
+ itemQueryBuilder = QueryBuilders.termQuery(fieldName, operand);
+ } else {
+ itemQueryBuilder = QueryBuilders.termsQuery(fieldName, operands);
+ }
} else {
// not (yet) support operator types
return null;
[metamodel] 01/05: MM-1233: Elasticsearch uses 'match' query for
text based columns conditions in WHERE clause.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit 8386549ee9580d6f14232041e60bc2ff476ae1dd
Author: jakub <j....@quadient.com>
AuthorDate: Fri Feb 12 16:02:12 2021 +0100
MM-1233: Elasticsearch uses 'match' query for text based columns conditions in WHERE clause.
---
.../elasticsearch/common/ElasticSearchUtils.java | 2 ++
.../common/ElasticSearchUtilsTest.java | 24 ++++++++++++++++++++--
2 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java b/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
index f5c70e8..b2c1f27 100644
--- a/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
+++ b/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
@@ -202,6 +202,8 @@ public class ElasticSearchUtils {
if (OperatorType.EQUALS_TO.equals(operator)) {
if (operand == null) {
itemQueryBuilder = getMissingQuery(fieldName);
+ } else if (column.getType().isLiteral()) {
+ itemQueryBuilder = QueryBuilders.matchQuery(fieldName, operand);
} else {
itemQueryBuilder = QueryBuilders.termQuery(fieldName, operand);
}
diff --git a/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java b/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java
index 9fb7e03..8228316 100644
--- a/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java
+++ b/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java
@@ -18,15 +18,24 @@
*/
package org.apache.metamodel.elasticsearch.common;
-import junit.framework.TestCase;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.metamodel.data.DataSetHeader;
import org.apache.metamodel.data.Row;
import org.apache.metamodel.data.SimpleDataSetHeader;
+import org.apache.metamodel.query.FilterItem;
+import org.apache.metamodel.query.OperatorType;
import org.apache.metamodel.query.SelectItem;
import org.apache.metamodel.schema.ColumnType;
import org.apache.metamodel.schema.MutableColumn;
+import org.elasticsearch.index.query.QueryBuilder;
-import java.util.*;
+import junit.framework.TestCase;
public class ElasticSearchUtilsTest extends TestCase {
@@ -60,4 +69,15 @@ public class ElasticSearchUtilsTest extends TestCase {
assertTrue(stringValue instanceof String);
assertTrue(dateValue instanceof Date);
}
+
+ /**
+ * For text-based conditions a 'match' query is recommended (instead of 'term' query).
+ */
+ public void testMatchQueryIsCreatedForTextFilter() {
+ final SelectItem selectItem = new SelectItem(new MutableColumn("column_name", ColumnType.STRING));
+ final FilterItem filterItem = new FilterItem(selectItem, OperatorType.EQUALS_TO, "text-value");
+ final QueryBuilder queryBuilder =
+ ElasticSearchUtils.createQueryBuilderForSimpleWhere(Collections.singletonList(filterItem), null);
+ assertEquals("match", queryBuilder.getName());
+ }
}
[metamodel] 04/05: METAMODEL-1233: JUnit tests were extended.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit 2f31f6ec923a860aa66b29165ecbc5dfc4c84d4f
Author: jakub <j....@quadient.com>
AuthorDate: Tue Feb 16 15:08:40 2021 +0100
METAMODEL-1233: JUnit tests were extended.
---
.../common/ElasticSearchUtilsTest.java | 25 ++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java b/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java
index 4d18b27..923ae9e 100644
--- a/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java
+++ b/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java
@@ -33,6 +33,8 @@ import org.apache.metamodel.query.OperatorType;
import org.apache.metamodel.query.SelectItem;
import org.apache.metamodel.schema.ColumnType;
import org.apache.metamodel.schema.MutableColumn;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilder;
import junit.framework.TestCase;
@@ -78,6 +80,7 @@ public class ElasticSearchUtilsTest extends TestCase {
final FilterItem filterItem = new FilterItem(selectItem, OperatorType.EQUALS_TO, "text-value");
final QueryBuilder queryBuilder =
ElasticSearchUtils.createQueryBuilderForSimpleWhere(Collections.singletonList(filterItem), null);
+ assertNotNull(queryBuilder);
assertEquals("match", queryBuilder.getName());
}
@@ -85,24 +88,46 @@ public class ElasticSearchUtilsTest extends TestCase {
* For text-based conditions a 'match' query is recommended (instead of 'term' query).
* In case of 'DIFFERENT_FROM', we need a 'bool' query with 'must not' and 'match' query.
*/
+ @SuppressWarnings("unchecked")
public void testBoolQueryIsCreatedForTextDifferentFrom() {
final SelectItem selectItem = new SelectItem(new MutableColumn("column_name", ColumnType.STRING));
final FilterItem filterItem = new FilterItem(selectItem, OperatorType.DIFFERENT_FROM, "text-value");
final QueryBuilder queryBuilder =
ElasticSearchUtils.createQueryBuilderForSimpleWhere(Collections.singletonList(filterItem), null);
+ assertNotNull(queryBuilder);
assertEquals("bool", queryBuilder.getName());
+ final Map<String, Object> queryMap =
+ XContentHelper.convertToMap(XContentType.JSON.xContent(), queryBuilder.toString(), false);
+ final Map<String, Object> boolMap = (Map<String, Object>) queryMap.get("bool");
+ assertNotNull(boolMap);
+ assertTrue(boolMap.containsKey("must_not"));
+ final List<Object> mustNotList = (List<Object>) boolMap.get("must_not");
+ final Map<String, Object> mustNotMap = (Map<String, Object>) mustNotList.get(0);
+ assertTrue(mustNotMap.containsKey("match"));
}
/**
* For text-based conditions a 'match' query is recommended (instead of 'term' query).
* To simulate 'IN' operator, we need a 'bool' query with multiple 'match' queries combined with 'OR'.
*/
+ @SuppressWarnings("unchecked")
public void testBoolQueryIsCreatedForTextIn() {
final SelectItem selectItem = new SelectItem(new MutableColumn("column_name", ColumnType.STRING));
final FilterItem filterItem =
new FilterItem(selectItem, OperatorType.IN, Arrays.asList("text-value-a", "text-value-b"));
final QueryBuilder queryBuilder =
ElasticSearchUtils.createQueryBuilderForSimpleWhere(Collections.singletonList(filterItem), null);
+ assertNotNull(queryBuilder);
assertEquals("bool", queryBuilder.getName());
+ final Map<String, Object> queryMap =
+ XContentHelper.convertToMap(XContentType.JSON.xContent(), queryBuilder.toString(), false);
+ final Map<String, Object> matchMap = (Map<String, Object>) queryMap.get("bool");
+ assertNotNull(matchMap);
+ assertTrue(matchMap.containsKey("should"));
+ final List<Object> shouldList = (List<Object>) matchMap.get("should");
+ assertNotNull(shouldList);
+ assertEquals(2, shouldList.size());
+ assertTrue(((Map<String, Object>)shouldList.get(0)).containsKey("match"));
+ assertTrue(((Map<String, Object>)shouldList.get(1)).containsKey("match"));
}
}
[metamodel] 03/05: MM-1233: Refactoring, DIFFERENT_FROM operator.
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arjansh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit 142c4ae09f55af87fd15a66af78c887a663e4252
Author: jakub <j....@quadient.com>
AuthorDate: Mon Feb 15 13:43:20 2021 +0100
MM-1233: Refactoring, DIFFERENT_FROM operator.
---
.../elasticsearch/common/ElasticSearchUtils.java | 118 ++++++++++++---------
.../common/ElasticSearchUtilsTest.java | 27 ++++-
2 files changed, 92 insertions(+), 53 deletions(-)
diff --git a/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java b/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
index 5458174..d051c32 100644
--- a/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
+++ b/elasticsearch/common/src/main/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtils.java
@@ -170,63 +170,18 @@ public class ElasticSearchUtils {
* @return a {@link QueryBuilder} if one was produced, or null if the items
* could not be pushed down to an ElasticSearch query
*/
- public static QueryBuilder createQueryBuilderForSimpleWhere(List<FilterItem> whereItems,
- LogicalOperator logicalOperator) {
+ public static QueryBuilder createQueryBuilderForSimpleWhere(final List<FilterItem> whereItems,
+ final LogicalOperator logicalOperator) {
if (whereItems.isEmpty()) {
return QueryBuilders.matchAllQuery();
}
- List<QueryBuilder> children = new ArrayList<>(whereItems.size());
- for (FilterItem item : whereItems) {
- final QueryBuilder itemQueryBuilder;
-
- if (item.isCompoundFilter()) {
- final List<FilterItem> childItems = Arrays.asList(item.getChildItems());
- itemQueryBuilder = createQueryBuilderForSimpleWhere(childItems, item.getLogicalOperator());
- if (itemQueryBuilder == null) {
- // something was not supported, so we have to forfeit here
- // too.
- return null;
- }
- } else {
- final Column column = item.getSelectItem().getColumn();
- if (column == null) {
- // unsupported type of where item - must have a column
- // reference
- return null;
- }
- final String fieldName = column.getName();
- final Object operand = item.getOperand();
- final OperatorType operator = item.getOperator();
-
- if (OperatorType.EQUALS_TO.equals(operator)) {
- if (operand == null) {
- itemQueryBuilder = getMissingQuery(fieldName);
- } else if (column.getType().isLiteral()) {
- itemQueryBuilder = QueryBuilders.matchQuery(fieldName, operand);
- } else {
- itemQueryBuilder = QueryBuilders.termQuery(fieldName, operand);
- }
- } else if (OperatorType.DIFFERENT_FROM.equals(operator)) {
- if (operand == null) {
- itemQueryBuilder = getExistsQuery(fieldName);
- } else {
- itemQueryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(fieldName,
- operand));
- }
- } else if (OperatorType.IN.equals(operator)) {
- final List<?> operands = CollectionUtils.toList(operand);
- if (column.getType().isLiteral()) {
- itemQueryBuilder = QueryBuilders.termQuery(fieldName, operand);
- } else {
- itemQueryBuilder = QueryBuilders.termsQuery(fieldName, operands);
- }
- } else {
- // not (yet) support operator types
- return null;
- }
+ final List<QueryBuilder> children = new ArrayList<>(whereItems.size());
+ for (final FilterItem item : whereItems) {
+ final QueryBuilder itemQueryBuilder = createFilterItemQueryBuilder(item);
+ if (itemQueryBuilder == null) {
+ return null;
}
-
children.add(itemQueryBuilder);
}
@@ -249,6 +204,65 @@ public class ElasticSearchUtils {
return result;
}
+ private static QueryBuilder createFilterItemQueryBuilder(final FilterItem filterItem) {
+ final QueryBuilder itemQueryBuilder;
+ if (filterItem.isCompoundFilter()) {
+ final List<FilterItem> childItems = Arrays.asList(filterItem.getChildItems());
+ itemQueryBuilder = createQueryBuilderForSimpleWhere(childItems, filterItem.getLogicalOperator());
+ } else {
+ final Column column = filterItem.getSelectItem().getColumn();
+ if (column == null) {
+ // unsupported type of where item - must have a column reference
+ return null;
+ }
+ itemQueryBuilder = createQueryBuilderForOperator(filterItem, column);
+ }
+
+ return itemQueryBuilder;
+ }
+
+ private static QueryBuilder createQueryBuilderForOperator(final FilterItem filterItem, final Column column) {
+ if (OperatorType.EQUALS_TO.equals(filterItem.getOperator())) {
+ if (filterItem.getOperand() == null) {
+ return getMissingQuery(column.getName());
+ } else {
+ return matchOrTermQuery(column, filterItem.getOperand());
+ }
+ } else if (OperatorType.DIFFERENT_FROM.equals(filterItem.getOperator())) {
+ if (filterItem.getOperand() == null) {
+ return getExistsQuery(column.getName());
+ } else {
+ return QueryBuilders.boolQuery().mustNot(matchOrTermQuery(column, filterItem.getOperand()));
+ }
+ } else if (OperatorType.IN.equals(filterItem.getOperator())) {
+ final List<?> operands = CollectionUtils.toList(filterItem.getOperand());
+ if (column.getType().isLiteral()) {
+ return createMultipleValuesQueryBuilder(column.getName(), operands);
+ } else {
+ return QueryBuilders.termsQuery(column.getName(), operands);
+ }
+ } else {
+ // not (yet) supported operator types
+ return null;
+ }
+ }
+
+ private static QueryBuilder matchOrTermQuery(final Column column, final Object operand) {
+ if (column.getType().isLiteral()) {
+ return QueryBuilders.matchQuery(column.getName(), operand);
+ } else {
+ return QueryBuilders.termQuery(column.getName(), operand);
+ }
+ }
+
+ private static QueryBuilder createMultipleValuesQueryBuilder(final String columnName, final List<?> operands) {
+ final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ for (final Object value : operands) {
+ boolQueryBuilder.should(QueryBuilders.matchQuery(columnName, value.toString()));
+ }
+ return boolQueryBuilder;
+ }
+
public static ColumnType getColumnTypeFromElasticSearchType(final String metaDataFieldType) {
final ColumnType columnType;
if (metaDataFieldType.startsWith("date")) {
diff --git a/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java b/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java
index 8228316..4d18b27 100644
--- a/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java
+++ b/elasticsearch/common/src/test/java/org/apache/metamodel/elasticsearch/common/ElasticSearchUtilsTest.java
@@ -73,11 +73,36 @@ public class ElasticSearchUtilsTest extends TestCase {
/**
* For text-based conditions a 'match' query is recommended (instead of 'term' query).
*/
- public void testMatchQueryIsCreatedForTextFilter() {
+ public void testMatchQueryIsCreatedForTextEqualTo() {
final SelectItem selectItem = new SelectItem(new MutableColumn("column_name", ColumnType.STRING));
final FilterItem filterItem = new FilterItem(selectItem, OperatorType.EQUALS_TO, "text-value");
final QueryBuilder queryBuilder =
ElasticSearchUtils.createQueryBuilderForSimpleWhere(Collections.singletonList(filterItem), null);
assertEquals("match", queryBuilder.getName());
}
+
+ /**
+ * For text-based conditions a 'match' query is recommended (instead of 'term' query).
+ * In case of 'DIFFERENT_FROM', we need a 'bool' query with 'must not' and 'match' query.
+ */
+ public void testBoolQueryIsCreatedForTextDifferentFrom() {
+ final SelectItem selectItem = new SelectItem(new MutableColumn("column_name", ColumnType.STRING));
+ final FilterItem filterItem = new FilterItem(selectItem, OperatorType.DIFFERENT_FROM, "text-value");
+ final QueryBuilder queryBuilder =
+ ElasticSearchUtils.createQueryBuilderForSimpleWhere(Collections.singletonList(filterItem), null);
+ assertEquals("bool", queryBuilder.getName());
+ }
+
+ /**
+ * For text-based conditions a 'match' query is recommended (instead of 'term' query).
+ * To simulate 'IN' operator, we need a 'bool' query with multiple 'match' queries combined with 'OR'.
+ */
+ public void testBoolQueryIsCreatedForTextIn() {
+ final SelectItem selectItem = new SelectItem(new MutableColumn("column_name", ColumnType.STRING));
+ final FilterItem filterItem =
+ new FilterItem(selectItem, OperatorType.IN, Arrays.asList("text-value-a", "text-value-b"));
+ final QueryBuilder queryBuilder =
+ ElasticSearchUtils.createQueryBuilderForSimpleWhere(Collections.singletonList(filterItem), null);
+ assertEquals("bool", queryBuilder.getName());
+ }
}