You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by mm...@apache.org on 2018/07/20 17:41:59 UTC
[46/53] [abbrv] calcite git commit: [CALCITE-2331] Evaluation of
predicate "(A or B) and C" fails for Elasticsearch adapter (Andrei Sereda)
http://git-wip-us.apache.org/repos/asf/calcite/blob/6344afc4/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/EmbeddedElasticsearchPolicy.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/EmbeddedElasticsearchPolicy.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/EmbeddedElasticsearchPolicy.java
index 9a2f69e..5cdd82f 100644
--- a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/EmbeddedElasticsearchPolicy.java
+++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/EmbeddedElasticsearchPolicy.java
@@ -18,14 +18,24 @@ package org.apache.calcite.adapter.elasticsearch;
import org.apache.calcite.util.Closer;
+import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.transport.TransportAddress;
import org.junit.rules.ExternalResource;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
import java.util.Objects;
/**
@@ -83,7 +93,71 @@ class EmbeddedElasticsearchPolicy extends ExternalResource {
}
/**
- * Exposes Jackson API to be used for low-level ES.
+ * Creates index in elastic search given mapping.
+ *
+ * @param index index of the index
+ * @param mapping field and field type mapping
+ * @throws IOException if there is an error
+ */
+ void createIndex(String index, Map<String, String> mapping) throws IOException {
+ Objects.requireNonNull(index, "index");
+ Objects.requireNonNull(mapping, "mapping");
+
+ ObjectNode json = mapper().createObjectNode();
+ for (Map.Entry<String, String> entry: mapping.entrySet()) {
+ json.set(entry.getKey(), json.objectNode().put("type", entry.getValue()));
+ }
+
+ json = (ObjectNode) json.objectNode().set("properties", json);
+ json = (ObjectNode) json.objectNode().set(index, json);
+ json = (ObjectNode) json.objectNode().set("mappings", json);
+
+ // create index and mapping
+ final HttpEntity entity = new StringEntity(mapper().writeValueAsString(json),
+ ContentType.APPLICATION_JSON);
+ restClient().performRequest("PUT", "/" + index, Collections.emptyMap(), entity);
+ }
+
+ void insertDocument(String index, ObjectNode document) throws IOException {
+ Objects.requireNonNull(index, "index");
+ Objects.requireNonNull(document, "document");
+ String uri = String.format(Locale.ROOT,
+ "/%s/%s/?refresh", index, index);
+ StringEntity entity = new StringEntity(mapper().writeValueAsString(document),
+ ContentType.APPLICATION_JSON);
+
+ restClient().performRequest("POST", uri,
+ Collections.emptyMap(),
+ entity);
+ }
+
+ void insertBulk(String index, List<ObjectNode> documents) throws IOException {
+ Objects.requireNonNull(index, "index");
+ Objects.requireNonNull(documents, "documents");
+
+ if (documents.isEmpty()) {
+ // nothing to process
+ return;
+ }
+
+ List<String> bulk = new ArrayList<>(documents.size() * 2);
+ for (ObjectNode doc: documents) {
+ bulk.add("{\"index\": {} }"); // index/type will be derived from _bulk URI
+ bulk.add(mapper().writeValueAsString(doc));
+ }
+
+ final StringEntity entity = new StringEntity(String.join("\n", bulk) + "\n",
+ ContentType.APPLICATION_JSON);
+
+ final String uri = String.format(Locale.ROOT, "/%s/%s/_bulk?refresh", index, index);
+
+ restClient().performRequest("POST", uri,
+ Collections.emptyMap(),
+ entity);
+ }
+
+ /**
+ * Exposes Jackson API to be used to parse search results.
* @return existing instance of ObjectMapper
*/
ObjectMapper mapper() {
http://git-wip-us.apache.org/repos/asf/calcite/blob/6344afc4/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ProjectionTest.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ProjectionTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ProjectionTest.java
new file mode 100644
index 0000000..9830036
--- /dev/null
+++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/ProjectionTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.elasticsearch;
+
+import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.impl.ViewTable;
+import org.apache.calcite.schema.impl.ViewTableMacro;
+import org.apache.calcite.test.CalciteAssert;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.ImmutableMap;
+
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * Checks renaming of fields (also upper, lower cases) during projections
+ */
+public class ProjectionTest {
+
+ @ClassRule
+ public static final EmbeddedElasticsearchPolicy NODE = EmbeddedElasticsearchPolicy.create();
+
+ private static final String NAME = "docs";
+
+ @BeforeClass
+ public static void setupInstance() throws Exception {
+
+ final Map<String, String> mappings = ImmutableMap.of("A", "keyword",
+ "b", "keyword", "cCC", "keyword", "DDd", "keyword");
+
+ NODE.createIndex(NAME, mappings);
+
+ String doc = "{'A': 'aa', 'b': 'bb', 'cCC': 'cc', 'DDd': 'dd'}".replace('\'', '"');
+ NODE.insertDocument(NAME, (ObjectNode) NODE.mapper().readTree(doc));
+ }
+
+ private CalciteAssert.ConnectionFactory newConnectionFactory() {
+ return new CalciteAssert.ConnectionFactory() {
+ @Override public Connection createConnection() throws SQLException {
+ final Connection connection = DriverManager.getConnection("jdbc:calcite:");
+ final SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema();
+
+ root.add("elastic", new ElasticsearchSchema(NODE.restClient(), NODE.mapper(), NAME));
+
+ // add calcite view programmatically
+ final String viewSql = String.format(Locale.ROOT,
+ "select cast(_MAP['A'] AS varchar(2)) AS \"a\", "
+ + " cast(_MAP['b'] AS varchar(2)) AS \"b\", "
+ + " cast(_MAP['cCC'] AS varchar(2)) AS \"c\", "
+ + " cast(_MAP['DDd'] AS varchar(2)) AS \"d\" "
+ + " from \"elastic\".\"%s\"", NAME);
+
+ ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
+ Collections.singletonList("elastic"), Arrays.asList("elastic", "view"), false);
+ root.add("VIEW", macro);
+
+ return connection;
+ }
+ };
+ }
+
+ @Test
+ public void projection() {
+ CalciteAssert.that()
+ .with(newConnectionFactory())
+ .query("select * from view")
+ .returns("a=aa; b=bb; c=cc; d=dd\n");
+ }
+}
+
+// End ProjectionTest.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/6344afc4/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/QueryBuildersTest.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/QueryBuildersTest.java b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/QueryBuildersTest.java
new file mode 100644
index 0000000..7f11715
--- /dev/null
+++ b/elasticsearch/src/test/java/org/apache/calcite/adapter/elasticsearch/QueryBuildersTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.elasticsearch;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.StringWriter;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Check that internal queries are correctly converted to ES search query (as JSON)
+ */
+public class QueryBuildersTest {
+
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ @Test
+ public void term() throws Exception {
+ assertEquals("{\"term\":{\"foo\":\"bar\"}}",
+ toJson(QueryBuilders.termQuery("foo", "bar")));
+ assertEquals("{\"term\":{\"foo\":true}}",
+ toJson(QueryBuilders.termQuery("foo", true)));
+ assertEquals("{\"term\":{\"foo\":false}}",
+ toJson(QueryBuilders.termQuery("foo", false)));
+ assertEquals("{\"term\":{\"foo\":123}}",
+ toJson(QueryBuilders.termQuery("foo", (long) 123)));
+ assertEquals("{\"term\":{\"foo\":41}}",
+ toJson(QueryBuilders.termQuery("foo", (short) 41)));
+ assertEquals("{\"term\":{\"foo\":42.42}}",
+ toJson(QueryBuilders.termQuery("foo", 42.42D)));
+ assertEquals("{\"term\":{\"foo\":1.1}}",
+ toJson(QueryBuilders.termQuery("foo", 1.1F)));
+ }
+
+ @Test
+ public void boolQuery() throws Exception {
+ QueryBuilders.QueryBuilder q1 = QueryBuilders.boolQuery()
+ .must(QueryBuilders.termQuery("foo", "bar"));
+
+ assertEquals("{\"bool\":{\"must\":{\"term\":{\"foo\":\"bar\"}}}}",
+ toJson(q1));
+
+ QueryBuilders.QueryBuilder q2 = QueryBuilders.boolQuery()
+ .must(QueryBuilders.termQuery("f1", "v1")).must(QueryBuilders.termQuery("f2", "v2"));
+
+ assertEquals("{\"bool\":{\"must\":[{\"term\":{\"f1\":\"v1\"}},{\"term\":{\"f2\":\"v2\"}}]}}",
+ toJson(q2));
+
+ QueryBuilders.QueryBuilder q3 = QueryBuilders.boolQuery()
+ .mustNot(QueryBuilders.termQuery("f1", "v1"));
+
+ assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"f1\":\"v1\"}}}}",
+ toJson(q3));
+
+ }
+
+ @Test
+ public void exists() throws Exception {
+ assertEquals("{\"exists\":{\"field\":\"foo\"}}",
+ toJson(QueryBuilders.existsQuery("foo")));
+ }
+
+ @Test
+ public void range() throws Exception {
+ assertEquals("{\"range\":{\"f\":{\"lt\":0}}}",
+ toJson(QueryBuilders.rangeQuery("f").lt(0)));
+ assertEquals("{\"range\":{\"f\":{\"gt\":0}}}",
+ toJson(QueryBuilders.rangeQuery("f").gt(0)));
+ assertEquals("{\"range\":{\"f\":{\"gte\":0}}}",
+ toJson(QueryBuilders.rangeQuery("f").gte(0)));
+ assertEquals("{\"range\":{\"f\":{\"lte\":0}}}",
+ toJson(QueryBuilders.rangeQuery("f").lte(0)));
+ assertEquals("{\"range\":{\"f\":{\"gt\":1,\"lt\":2}}}",
+ toJson(QueryBuilders.rangeQuery("f").gt(1).lt(2)));
+ assertEquals("{\"range\":{\"f\":{\"gt\":11,\"lt\":0}}}",
+ toJson(QueryBuilders.rangeQuery("f").lt(0).gt(11)));
+ assertEquals("{\"range\":{\"f\":{\"gt\":1,\"lte\":2}}}",
+ toJson(QueryBuilders.rangeQuery("f").gt(1).lte(2)));
+ assertEquals("{\"range\":{\"f\":{\"gte\":1,\"lte\":\"zz\"}}}",
+ toJson(QueryBuilders.rangeQuery("f").gte(1).lte("zz")));
+ assertEquals("{\"range\":{\"f\":{\"gte\":1}}}",
+ toJson(QueryBuilders.rangeQuery("f").gte(1)));
+ assertEquals("{\"range\":{\"f\":{\"gte\":\"zz\"}}}",
+ toJson(QueryBuilders.rangeQuery("f").gte("zz")));
+ assertEquals("{\"range\":{\"f\":{\"gt\":\"a\",\"lt\":\"z\"}}}",
+ toJson(QueryBuilders.rangeQuery("f").gt("a").lt("z")));
+ assertEquals("{\"range\":{\"f\":{\"gte\":3}}}",
+ toJson(QueryBuilders.rangeQuery("f").gt(1).gt(2).gte(3)));
+ assertEquals("{\"range\":{\"f\":{\"lte\":3}}}",
+ toJson(QueryBuilders.rangeQuery("f").lt(1).lt(2).lte(3)));
+ }
+
+ private String toJson(QueryBuilders.QueryBuilder builder) throws IOException {
+ StringWriter writer = new StringWriter();
+ JsonGenerator gen = mapper.getFactory().createGenerator(writer);
+ builder.writeJson(gen);
+ gen.flush();
+ gen.close();
+ return writer.toString();
+ }
+}
+
+// End QueryBuildersTest.java
http://git-wip-us.apache.org/repos/asf/calcite/blob/6344afc4/elasticsearch/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java
----------------------------------------------------------------------
diff --git a/elasticsearch/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java b/elasticsearch/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java
index 68fc073..823d4f1 100644
--- a/elasticsearch/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java
+++ b/elasticsearch/src/test/java/org/apache/calcite/test/ElasticsearchChecker.java
@@ -17,6 +17,7 @@
package org.apache.calcite.test;
import java.util.List;
+import java.util.Objects;
import java.util.function.Consumer;
/**
@@ -26,17 +27,19 @@ public class ElasticsearchChecker {
private ElasticsearchChecker() {}
+
/** Returns a function that checks that a particular Elasticsearch pipeline is
* generated to implement a query.
* @param strings expected expressions
* @return validation function
*/
public static Consumer<List> elasticsearchChecker(final String... strings) {
+ Objects.requireNonNull(strings, "strings");
return actual -> {
Object[] actualArray = actual == null || actual.isEmpty() ? null
- : ((List) actual.get(0)).toArray();
+ : ((List) actual.get(0)).toArray();
CalciteAssert.assertArrayEqual("expected Elasticsearch query not found", strings,
- actualArray);
+ actualArray);
};
}
}