You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@calcite.apache.org by Julian Hyde <jh...@apache.org> on 2018/06/22 21:11:56 UTC
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Looks like this change broke “mvn site” (perhaps also “mvn javadoc:test-javadoc”).
[ERROR] Failed to execute goal org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate report:
[ERROR] Exit code: 1 - /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29: error: package org.elasticsearch.node.internal does not exist
[ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
[ERROR] ^
[ERROR]
> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
>
> [CALCITE-2347] running ElasticSearch in embedded mode for unit tests of ES adapter (Andrei Sereda)
>
> After discussion on dev-list Integration tests (for ES) have been removed. They're now
> superseded by unit tests (which execute queries against a real elastic instance)
>
> Added local file (zips-mini.json) which contains a small subset of original zips.json
> (allows to bootstrap tests faster)
>
> Created separate ES JUnit rule which can be re-used across different tests.
>
> Both v2 and v5 of ES adapters are supported.
>
> Close apache/calcite#716
>
>
> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
> Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
> Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
> Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
>
> Branch: refs/heads/master
> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
> Parents: 37944bb
> Author: Andrei Sereda <an...@nospam.com>
> Authored: Thu May 31 18:19:10 2018 -0400
> Committer: Michael Mior <mm...@uwaterloo.ca>
> Committed: Thu Jun 21 06:38:50 2018 -0400
>
> ----------------------------------------------------------------------
> .../AbstractElasticsearchTable.java | 12 +
> .../elasticsearch/ElasticsearchProject.java | 61 ++-
> elasticsearch2/pom.xml | 6 +
> .../Elasticsearch2Enumerator.java | 12 +-
> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
> .../ElasticSearch2AdapterTest.java | 395 ++++++++++++++++++
> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> .../calcite/test/Elasticsearch2AdapterIT.java | 270 -------------
> .../resources/elasticsearch-zips-model.json | 50 ---
> .../src/test/resources/zips-mini.json | 149 +++++++
> elasticsearch5/pom.xml | 31 ++
> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
> .../ElasticSearch5AdapterTest.java | 399 +++++++++++++++++++
> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> .../calcite/test/Elasticsearch5AdapterIT.java | 270 -------------
> .../resources/elasticsearch-zips-model.json | 50 ---
> elasticsearch5/src/test/resources/log4j2.xml | 16 +
> .../src/test/resources/zips-mini.json | 149 +++++++
> pom.xml | 20 +-
> 25 files changed, 1866 insertions(+), 670 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> ----------------------------------------------------------------------
> diff --git a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> index 0980469..8cc5933 100644
> --- a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> +++ b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> @@ -75,6 +75,18 @@ public abstract class AbstractElasticsearchTable extends AbstractQueryableTable
> relOptTable, this, null);
> }
>
> + /**
> + * In ES 5.x scripted fields start with {@code params._source.foo} while in ES2.x
> + * {@code _source.foo}. Helper method to build correct query based on runtime version of elastic.
> + *
> + * @see <a href="https://github.com/elastic/elasticsearch/issues/20068">_source variable</a>
> + * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html">Scripted Fields</a>
> + */
> + protected String scriptedFieldPrefix() {
> + // this is default pattern starting 5.x
> + return "params._source";
> + }
> +
> /** Executes a "find" operation on the underlying type.
> *
> * <p>For example,
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> ----------------------------------------------------------------------
> diff --git a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> index b42abd7..961c8b0 100644
> --- a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> +++ b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> @@ -27,11 +27,15 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery;
> import org.apache.calcite.rel.type.RelDataType;
> import org.apache.calcite.rex.RexNode;
> import org.apache.calcite.util.Pair;
> -import org.apache.calcite.util.Util;
> +
> +import com.google.common.base.Function;
> +import com.google.common.collect.Lists;
>
> import java.util.ArrayList;
> import java.util.List;
>
> +import javax.annotation.Nullable;
> +
> /**
> * Implementation of {@link org.apache.calcite.rel.core.Project}
> * relational expression in Elasticsearch.
> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends Project implements ElasticsearchRel {
> implementor.visitChild(0, getInput());
>
> final List<String> inFields =
> - ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> + ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> final ElasticsearchRules.RexToElasticsearchTranslator translator =
> - new ElasticsearchRules.RexToElasticsearchTranslator(
> - (JavaTypeFactory) getCluster().getTypeFactory(), inFields);
> + new ElasticsearchRules.RexToElasticsearchTranslator(
> + (JavaTypeFactory) getCluster().getTypeFactory(), inFields);
>
> - final List<String> findItems = new ArrayList<>();
> - final List<String> scriptFieldItems = new ArrayList<>();
> + final List<String> fields = new ArrayList<>();
> + final List<String> scriptFields = new ArrayList<>();
> for (Pair<RexNode, String> pair: getNamedProjects()) {
> final String name = pair.right;
> final String expr = pair.left.accept(translator);
>
> if (expr.equals("\"" + name + "\"")) {
> - findItems.add(ElasticsearchRules.quote(name));
> + fields.add(name);
> } else if (expr.matches("\"literal\":.+")) {
> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> - + ":{\"script\": "
> - + expr.split(":")[1] + "}");
> + scriptFields.add(ElasticsearchRules.quote(name)
> + + ":{\"script\": "
> + + expr.split(":")[1] + "}");
> } else {
> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> - + ":{\"script\":\"params._source."
> - + expr.replaceAll("\"", "") + "\"}");
> + scriptFields.add(ElasticsearchRules.quote(name)
> + + ":{\"script\":"
> + // _source (ES2) vs params._source (ES5)
> + + "\"" + implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> + + expr.replaceAll("\"", "") + "\"}");
> + }
> + }
> +
> + StringBuilder query = new StringBuilder();
> + if (scriptFields.isEmpty()) {
> + List<String> newList = Lists.transform(fields, new Function<String, String>() {
> + @Nullable
> + @Override public String apply(@Nullable String input) {
> + return ElasticsearchRules.quote(input);
> + }
> + });
> +
> + final String findString = String.join(", ", newList);
> + query.append("\"_source\" : [").append(findString).append("]");
> + } else {
> + // if scripted fields are present, ES ignores _source attribute
> + for (String field: fields) {
> + scriptFields.add(ElasticsearchRules.quote(field) + ":{\"script\": "
> + // _source (ES2) vs params._source (ES5)
> + + "\"" + implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> + + field + "\"}");
> }
> + query.append("\"script_fields\": {" + String.join(", ", scriptFields) + "}");
> }
> - final String findString = Util.toString(findItems, "", ", ", "");
> - final String scriptFieldString = "\"script_fields\": {"
> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
> - final String fieldString = "\"_source\" : [" + findString + "]"
> - + ", " + scriptFieldString;
>
> for (String opfield : implementor.list) {
> if (opfield.startsWith("\"_source\"")) {
> implementor.list.remove(opfield);
> }
> }
> - implementor.add(fieldString);
> + implementor.add(query.toString());
> }
> }
>
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
> index f24622c..6fbee03 100644
> --- a/elasticsearch2/pom.xml
> +++ b/elasticsearch2/pom.xml
> @@ -73,6 +73,12 @@ limitations under the License.
> <version>${elasticsearch-java-driver.version}</version>
> </dependency>
> <dependency>
> + <!-- Lang groovy dependency is needed for testing with embedded ES (scripted fields like loc[0]) -->
> + <groupId>org.elasticsearch.module</groupId>
> + <artifactId>lang-groovy</artifactId>
> + <scope>test</scope>
> + </dependency>
> + <dependency>
> <groupId>com.carrotsearch</groupId>
> <artifactId>hppc</artifactId>
> <version>${hppc.version}</version>
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> index 84370ab..c3d2ac0 100644
> --- a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> +++ b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
> import java.util.Date;
> import java.util.Iterator;
> import java.util.List;
> +import java.util.Locale;
> import java.util.Map;
>
> /**
> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator implements Enumerator<Object> {
> private static Function1<SearchHit, Object[]> listGetter(
> final List<Map.Entry<String, Class>> fields) {
> return new Function1<SearchHit, Object[]>() {
> - public Object[] apply(SearchHit searchHitFields) {
> + public Object[] apply(SearchHit hit) {
> Object[] objects = new Object[fields.size()];
> for (int i = 0; i < fields.size(); i++) {
> final Map.Entry<String, Class> field = fields.get(i);
> final String name = field.getKey();
> - if (searchHitFields.fields().isEmpty()) {
> - objects[i] = convert(searchHitFields.getSource().get(name), field.getValue());
> + if (hit.fields().isEmpty()) {
> + objects[i] = convert(hit.getSource().get(name), field.getValue());
> + } else if (hit.fields().containsKey(name)) {
> + objects[i] = convert(hit.field(name).getValue(), field.getValue());
> } else {
> - objects[i] = convert(searchHitFields.field(name).getValue(), field.getValue());
> + throw new IllegalStateException(
> + String.format(Locale.getDefault(), "No result for %s", field));
> }
> }
> return objects;
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> index 668402b..46e3fc5 100644
> --- a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> +++ b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> @@ -22,6 +22,9 @@ import org.apache.calcite.schema.impl.AbstractSchema;
>
> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
>
> +import com.google.common.annotations.VisibleForTesting;
> +import com.google.common.base.Preconditions;
> +
> import com.google.common.collect.ImmutableList;
> import com.google.common.collect.ImmutableMap;
>
> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends AbstractSchema
> }
> }
>
> + /**
> + * Allows schema to be instantiated from existing elastic search client.
> + * This constructor is used in tests.
> + */
> + @VisibleForTesting
> + Elasticsearch2Schema(Client client, String index) {
> + this.client = Preconditions.checkNotNull(client, "client");
> + this.index = Preconditions.checkNotNull(index, "index");
> + }
> +
> @Override protected Map<String, Table> getTableMap() {
> final ImmutableMap.Builder<String, Table> builder = ImmutableMap.builder();
>
> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends AbstractSchema
>
> final List<DiscoveryNode> nodes = ImmutableList.copyOf(transportClient.connectedNodes());
> if (nodes.isEmpty()) {
> - throw new RuntimeException("Cannot connect to any elasticsearch nodes");
> + throw new IllegalStateException("Cannot connect to any elasticsearch node: "
> + + transportNodes);
> }
>
> client = transportClient;
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> index 636aa5f..2928835 100644
> --- a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> +++ b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends AbstractElasticsearchTable {
> this.client = client;
> }
>
> + /**
> + * ES version 2.x. To access document attributes ES2 uses {@code _source.foo} syntax.
> + */
> + @Override protected String scriptedFieldPrefix() {
> + return "_source";
> + }
> +
> @Override protected Enumerable<Object> find(String index, List<String> ops,
> - List<Map.Entry<String, Class>> fields) {
> + List<Map.Entry<String, Class>> fields) {
> final String dbName = index;
>
> final String queryString = "{" + Util.toString(ops, "", ", ", "") + "}";
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> new file mode 100644
> index 0000000..287e094
> --- /dev/null
> +++ b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> @@ -0,0 +1,395 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to you under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.calcite.adapter.elasticsearch2;
> +
> +import org.apache.calcite.jdbc.CalciteConnection;
> +import org.apache.calcite.schema.SchemaPlus;
> +import org.apache.calcite.schema.impl.ViewTable;
> +import org.apache.calcite.schema.impl.ViewTableMacro;
> +import org.apache.calcite.test.CalciteAssert;
> +import org.apache.calcite.test.ElasticChecker;
> +
> +import com.google.common.io.LineProcessor;
> +import com.google.common.io.Resources;
> +
> +import org.elasticsearch.action.bulk.BulkItemResponse;
> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
> +import org.elasticsearch.action.bulk.BulkResponse;
> +import org.elasticsearch.common.xcontent.XContentBuilder;
> +import org.elasticsearch.common.xcontent.XContentFactory;
> +
> +import org.junit.BeforeClass;
> +import org.junit.ClassRule;
> +import org.junit.Test;
> +
> +import java.io.IOException;
> +import java.nio.charset.StandardCharsets;
> +import java.sql.Connection;
> +import java.sql.DriverManager;
> +import java.sql.SQLException;
> +import java.util.Arrays;
> +import java.util.Collections;
> +import java.util.Locale;
> +
> +/**
> + * Set of tests for ES adapter. Uses real instance via {@link EmbeddedElasticRule}. Document
> + * source is local {@code zips-mini.json} file (located in the classpath).
> + */
> +public class ElasticSearch2AdapterTest {
> +
> + @ClassRule //init once for all tests
> + public static final EmbeddedElasticRule NODE = EmbeddedElasticRule.create();
> +
> + private static final String ZIPS = "zips";
> +
> + /**
> + * Used to create {@code zips} index and insert some data
> + */
> + @BeforeClass
> + public static void setupInstance() throws Exception {
> + // define mapping so fields are searchable (term query)
> + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
> + .startObject("properties")
> + .startObject("city").field("type", "string")
> + .field("index", "not_analyzed").endObject()
> + .startObject("state").field("type", "string")
> + .field("index", "not_analyzed").endObject()
> + .startObject("pop").field("type", "long").endObject()
> + .endObject()
> + .endObject();
> +
> + // create index
> + NODE.client().admin().indices()
> + .prepareCreate(ZIPS)
> + .addMapping(ZIPS, mapping)
> + .get();
> +
> + BulkRequestBuilder bulk = NODE.client().prepareBulk().setRefresh(true);
> +
> + // load records from file
> + Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
> + @Override public boolean processLine(String line) throws IOException {
> + line = line.replaceAll("_id", "id"); // _id is a reserved attribute in ES
> + bulk.add(NODE.client().prepareIndex(ZIPS, ZIPS).setSource(line));
> + return true;
> + }
> +
> + @Override public Void getResult() {
> + return null;
> + }
> + });
> +
> + if (bulk.numberOfActions() == 0) {
> + throw new IllegalStateException("No records to be indexed");
> + }
> +
> + BulkResponse response = bulk.execute().get();
> +
> + if (response.hasFailures()) {
> + throw new IllegalStateException(
> + String.format(Locale.getDefault(), "Failed to populate %s:\n%s", NODE.httpAddress(),
> + Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
> + .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
> + }
> +
> + }
> +
> + private CalciteAssert.ConnectionFactory newConnectionFactory() {
> + return new CalciteAssert.ConnectionFactory() {
> + @Override public Connection createConnection() throws SQLException {
> + final Connection connection = DriverManager.getConnection("jdbc:calcite:");
> + final SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema();
> +
> + root.add("elastic", new Elasticsearch2Schema(NODE.client(), ZIPS));
> +
> + // add calcite view programmatically
> + final String viewSql = "select cast(_MAP['city'] AS varchar(20)) AS \"city\", "
> + + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n"
> + + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n"
> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
> + + " cast(_MAP['state'] AS varchar(2)) AS \"state\", "
> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
> + + "from \"elastic\".\"zips\"";
> +
> + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
> + Collections.singletonList("elastic"), Arrays.asList("elastic", "view"), false);
> + root.add("ZIPS", macro);
> +
> + return connection;
> + }
> + };
> + }
> +
> + private CalciteAssert.AssertThat calciteAssert() {
> + return CalciteAssert.that()
> + .with(newConnectionFactory());
> + }
> +
> + /**
> + * Tests using calcite view
> + */
> + @Test
> + public void view() throws Exception {
> + calciteAssert()
> + .query("select * from zips where \"city\" = 'BROOKLYN'")
> + .returns("city=BROOKLYN; longitude=-73.956985; latitude=40.646694; "
> + + "pop=111396; state=NY; id=11226\n")
> + .returnsCount(1);
> + }
> +
> + @Test
> + public void emptyResult() {
> + CalciteAssert.that()
> + .with(newConnectionFactory())
> + .query("select * from zips limit 0")
> + .returnsCount(0);
> +
> + CalciteAssert.that()
> + .with(newConnectionFactory())
> + .query("select * from \"elastic\".\"zips\" where _MAP['Foo'] = '_MISSING_'")
> + .returnsCount(0);
> + }
> +
> + @Test
> + public void basic() throws Exception {
> + CalciteAssert.that()
> + .with(newConnectionFactory())
> + .query("select * from \"elastic\".\"zips\" where _MAP['city'] = 'BROOKLYN'")
> + .returnsCount(1);
> +
> + CalciteAssert.that()
> + .with(newConnectionFactory())
> + .query("select * from \"elastic\".\"zips\" where"
> + + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')")
> + .returnsCount(2);
> +
> + // lower-case
> + CalciteAssert.that()
> + .with(newConnectionFactory())
> + .query("select * from \"elastic\".\"zips\" where "
> + + "_MAP['city'] in ('brooklyn', 'Brooklyn', 'BROOK') ")
> + .returnsCount(0);
> +
> + // missing field
> + CalciteAssert.that()
> + .with(newConnectionFactory())
> + .query("select * from \"elastic\".\"zips\" where _MAP['CITY'] = 'BROOKLYN'")
> + .returnsCount(0);
> +
> + // limit works
> + CalciteAssert.that()
> + .with(newConnectionFactory())
> + .query("select * from \"elastic\".\"zips\" limit 42")
> + .returnsCount(42);
> +
> + }
> +
> + @Test public void testSort() {
> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> + + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
> +
> + calciteAssert()
> + .query("select * from zips order by \"state\"")
> + .returnsCount(10)
> + .explainContains(explain);
> + }
> +
> + @Test public void testSortLimit() {
> + final String sql = "select \"state\", \"id\" from zips\n"
> + + "order by \"state\", \"id\" offset 2 rows fetch next 3 rows only";
> + calciteAssert()
> + .query(sql)
> + .returnsUnordered("state=AK; id=99801",
> + "state=AL; id=35215",
> + "state=AL; id=35401")
> + .queryContains(
> + ElasticChecker.elasticsearchChecker(
> + "\"_source\" : [\"state\", \"id\"]",
> + "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]",
> + "\"from\": 2",
> + "\"size\": 3"));
> + }
> +
> +
> +
> + @Test public void testOffsetLimit() {
> + final String sql = "select \"state\", \"id\" from zips\n"
> + + "offset 2 fetch next 3 rows only";
> + calciteAssert()
> + .query(sql)
> + .runs()
> + .queryContains(
> + ElasticChecker.elasticsearchChecker(
> + "\"from\": 2",
> + "\"size\": 3",
> + "\"_source\" : [\"state\", \"id\"]"));
> + }
> +
> + @Test public void testLimit() {
> + final String sql = "select \"state\", \"id\" from zips\n"
> + + "fetch next 3 rows only";
> +
> + calciteAssert()
> + .query(sql)
> + .runs()
> + .queryContains(
> + ElasticChecker.elasticsearchChecker(
> + "\"size\": 3",
> + "\"_source\" : [\"state\", \"id\"]"));
> + }
> +
> + @Test public void testFilterSort() {
> + final String sql = "select * from zips\n"
> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
> + + "order by \"state\", \"id\"";
> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> + + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC], dir1=[ASC])\n"
> + + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> + + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", '70000'))])\n"
> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
> + calciteAssert()
> + .query(sql)
> + .returnsOrdered("city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; "
> + + "pop=96074; state=CA; id=90011",
> + "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; "
> + + "pop=99568; state=CA; id=90201",
> + "city=NORWALK; longitude=-118.081767; latitude=33.90564; "
> + + "pop=94188; state=CA; id=90650")
> + .queryContains(
> + ElasticChecker.elasticsearchChecker("\"query\" : "
> + + "{\"constant_score\":{\"filter\":{\"bool\":"
> + + "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
> + + "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
> + "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> + + "\"city\":{\"script\": \"_source.city\"}, "
> + + "\"pop\":{\"script\": \"_source.pop\"}, "
> + + "\"state\":{\"script\": \"_source.state\"}, "
> + + "\"id\":{\"script\": \"_source.id\"}}",
> + "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]"))
> + .explainContains(explain);
> + }
> +
> + @Test public void testFilterSortDesc() {
> + final String sql = "select * from zips\n"
> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
> + + "order by \"state\" desc, \"pop\"";
> + calciteAssert()
> + .query(sql)
> + .limit(4)
> + .returnsOrdered(
> + "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011",
> + "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201");
> + }
> +
> + @Test public void testFilterRedundant() {
> + final String sql = "select * from zips\n"
> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and \"state\" = 'OK'";
> + calciteAssert()
> + .query(sql)
> + .runs()
> + .queryContains(
> + ElasticChecker.elasticsearchChecker(""
> + + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":"
> + + "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
> + "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> + + "\"city\":{\"script\": \"_source.city\"}, "
> + + "\"pop\":{\"script\": \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
> + + "\"id\":{\"script\": \"_source.id\"}}"
> + ));
> + }
> +
> + @Test public void testInPlan() {
> + final String[] searches = {
> + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
> + + "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
> + + "{\"pop\":99568}}]}}]}}}}",
> + "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> + + "\"city\":{\"script\": \"_source.city\"}, "
> + + "\"pop\":{\"script\": \"_source.pop\"}, "
> + + "\"state\":{\"script\": \"_source.state\"}, "
> + + "\"id\":{\"script\": \"_source.id\"}}"
> + };
> +
> + calciteAssert()
> + .query("select * from zips where \"pop\" in (96074, 99568)")
> + .returnsUnordered(
> + "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201",
> + "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011"
> + )
> + .queryContains(ElasticChecker.elasticsearchChecker(searches));
> + }
> +
> + @Test public void testZips() {
> + calciteAssert()
> + .query("select \"state\", \"city\" from zips")
> + .returnsCount(10);
> + }
> +
> + @Test public void testProject() {
> + final String sql = "select \"state\", \"city\", 0 as \"zero\"\n"
> + + "from zips\n"
> + + "order by \"state\", \"city\"";
> +
> + calciteAssert()
> + .query(sql)
> + .limit(2)
> + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0",
> + "state=AK; city=FAIRBANKS; zero=0")
> + .queryContains(
> + ElasticChecker.elasticsearchChecker("\"script_fields\": "
> + + "{\"zero\":{\"script\": \"0\"}, "
> + + "\"state\":{\"script\": \"_source.state\"}, "
> + + "\"city\":{\"script\": \"_source.city\"}}",
> + "\"sort\": [ {\"state\": \"asc\"}, {\"city\": \"asc\"}]"));
> + }
> +
> + @Test public void testFilter() {
> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> + + " ElasticsearchProject(state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA')])\n"
> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
> + calciteAssert()
> + .query("select \"state\", \"city\" from zips where \"state\" = 'CA'")
> + .limit(3)
> + .returnsUnordered("state=CA; city=BELL GARDENS",
> + "state=CA; city=LOS ANGELES",
> + "state=CA; city=NORWALK")
> + .explainContains(explain);
> + }
> +
> + @Test public void testFilterReversed() {
> + calciteAssert()
> + .query("select \"state\", \"city\" from zips where 'WI' < \"state\" order by \"city\"")
> + .limit(2)
> + .returnsUnordered("state=WV; city=BECKLEY",
> + "state=WY; city=CHEYENNE");
> + calciteAssert()
> + .query("select \"state\", \"city\" from zips where \"state\" > 'WI' order by \"city\"")
> + .limit(2)
> + .returnsUnordered("state=WV; city=BECKLEY",
> + "state=WY; city=CHEYENNE");
> + }
> +
> +}
> +
> +// End ElasticSearch2AdapterTest.java
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> new file mode 100644
> index 0000000..4474add
> --- /dev/null
> +++ b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> @@ -0,0 +1,147 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to you under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.calcite.adapter.elasticsearch2;
> +
> +import com.google.common.base.Preconditions;
> +import com.google.common.io.Files;
> +
> +import org.elasticsearch.Version;
> +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
> +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
> +import org.elasticsearch.client.Client;
> +import org.elasticsearch.common.settings.Settings;
> +import org.elasticsearch.common.transport.TransportAddress;
> +import org.elasticsearch.node.Node;
> +import org.elasticsearch.node.internal.InternalSettingsPreparer;
> +import org.elasticsearch.plugins.Plugin;
> +import org.elasticsearch.script.groovy.GroovyPlugin;
> +
> +import java.io.File;
> +import java.util.Arrays;
> +import java.util.Collection;
> +import java.util.Collections;
> +
> +/**
> + * Represents a single elastic search node which can run embedded in a java application.
> + * Intended for unit and integration tests. Settings and plugins are crafted for Calcite.
> + */
> +class EmbeddedElasticNode implements AutoCloseable {
> +
> + private final LocalNode node;
> + private volatile boolean isStarted;
> +
> + private EmbeddedElasticNode(LocalNode node) {
> + this.node = Preconditions.checkNotNull(node, "node");
> + }
> +
> + /**
> + * Having separate class to expose (protected) constructor which allows to install
> + * different plugins. In our case it is {@code GroovyPlugin} for scripted fields like
> + * {@code loc[0]} or {@code loc[1]['foo']}.
> + */
> + private static class LocalNode extends Node {
> + private LocalNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
> + super(InternalSettingsPreparer.prepareEnvironment(settings, null),
> + Version.CURRENT,
> + classpathPlugins);
> + }
> + }
> +
> + /**
> + * Creates an instance with existing settings
> + */
> + private static EmbeddedElasticNode create(Settings settings) {
> + // ensure GroovyPlugin is installed or otherwise scripted fields would not work
> + LocalNode node = new LocalNode(settings, Collections.singleton(GroovyPlugin.class));
> + return new EmbeddedElasticNode(node);
> + }
> +
> + /**
> + * Creates elastic node as single member of a cluster. Node will not be started
> + * unless {@link #start()} is explicitly called.
> + */
> + public static EmbeddedElasticNode create() {
> + File data = Files.createTempDir();
> + data.deleteOnExit();
> + File home = Files.createTempDir();
> + home.deleteOnExit();
> +
> + Settings settings = Settings.builder()
> + .put("node.name", "fake-elastic")
> + .put("path.home", home.getAbsolutePath())
> + .put("path.data", data.getAbsolutePath())
> + .put("script.inline", true) // requires GroovyPlugin
> + .put("script.indexed", true) // requires GroovyPlugin
> + .put("cluster.routing.allocation.disk.threshold_enabled", false)
> + .put("node.local", true)
> + .put("node.data", true)
> + .put("network.host", "localhost")
> + .build();
> +
> + return create(settings);
> + }
> +
> + /**
> + * Starts current node
> + */
> + public void start() {
> + Preconditions.checkState(!isStarted, "already started");
> + node.start();
> + this.isStarted = true;
> + }
> +
> + /**
> + * Returns current address to connect to with HTTP client.
> + */
> + public TransportAddress httpAddress() {
> + Preconditions.checkState(isStarted, "node is not started");
> +
> + NodesInfoResponse response = client().admin().cluster().prepareNodesInfo()
> + .execute().actionGet();
> + if (response.getNodes().length != 1) {
> + throw new IllegalStateException("Expected single node but got "
> + + response.getNodes().length);
> + }
> + NodeInfo node = response.getNodes()[0];
> + return node.getHttp().address().boundAddresses()[0];
> + }
> +
> + /**
> + * Exposes elastic
> + * <a href="https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html">transport client</a>
> + *
> + * (use of HTTP client is preferred).
> + */
> + public Client client() {
> + Preconditions.checkState(isStarted, "node is not started");
> + return node.client();
> + }
> +
> + @Override public void close() throws Exception {
> + node.close();
> + // cleanup data dirs
> + File data = new File(node.settings().get("path.data"));
> + File home = new File(node.settings().get("path.home"));
> + for (File file: Arrays.asList(data, home)) {
> + if (file.exists()) {
> + file.delete();
> + }
> + }
> + }
> +}
> +
> +// End EmbeddedElasticNode.java
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> new file mode 100644
> index 0000000..a633078
> --- /dev/null
> +++ b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> @@ -0,0 +1,97 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to you under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.calcite.adapter.elasticsearch2;
> +
> +import com.google.common.base.Preconditions;
> +
> +import org.elasticsearch.client.Client;
> +import org.elasticsearch.common.transport.TransportAddress;
> +import org.junit.rules.ExternalResource;
> +
> +/**
> + * Used to initialize a single elastic node. For performance reasons (node startup costs),
> + * same instance is usually shared across multiple tests.
> + *
> + * This rule should be used as follows:
> + * <pre>
> + * {@code
> + *
> + * public class MyTest {
> + * @literal @ClassRule
> + * public static final ElasticSearchRule RULE = ElasticSearchRule.create();
> + *
> + * @literal @BeforeClass
> + * public void setup() {
> + * // ... populate instance
> + * }
> + *
> + * @literal @Test
> + * public void myTest() {
> + * TransportAddress address = RULE.httpAddress();
> + * // ....
> + * }
> + * }
> + * }
> + * </pre>
> + *
> + * @see ExternalResource
> + */
> +class EmbeddedElasticRule extends ExternalResource {
> +
> + private final EmbeddedElasticNode node;
> +
> + private EmbeddedElasticRule(EmbeddedElasticNode resource) {
> + this.node = Preconditions.checkNotNull(resource, "resource");
> + }
> +
> + @Override protected void before() throws Throwable {
> + node.start();
> + }
> +
> + @Override protected void after() {
> + try {
> + node.close();
> + } catch (Exception e) {
> + throw new RuntimeException(e);
> + }
> + }
> +
> + /**
> + * Factory method to create this rule.
> + */
> + public static EmbeddedElasticRule create() {
> + return new EmbeddedElasticRule(EmbeddedElasticNode.create());
> + }
> +
> + /**
> + * Exposes current ES transport client.
> + */
> + Client client() {
> + return node.client();
> + }
> +
> + /**
> + * HTTP address for rest clients (can be ES native or any other).
> + */
> + TransportAddress httpAddress() {
> + return node.httpAddress();
> + }
> +
> +
> +}
> +
> +// End EmbeddedElasticRule.java
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> new file mode 100644
> index 0000000..21fc491
> --- /dev/null
> +++ b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> @@ -0,0 +1,49 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements. See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to you under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.calcite.test;
> +
> +import com.google.common.base.Function;
> +
> +import java.util.List;
> +
> +import javax.annotation.Nullable;
> +
> +/**
> + * Internal util methods for ElasticSearch tests
> + */
> +public class ElasticChecker {
> +
> + private ElasticChecker() {}
> +
> +
> + /** Returns a function that checks that a particular Elasticsearch pipeline is
> + * generated to implement a query. */
> + public static Function<List, Void> elasticsearchChecker(final String... strings) {
> + return new Function<List, Void>() {
> + @Nullable
> + @Override public Void apply(@Nullable List actual) {
> + Object[] actualArray = actual == null || actual.isEmpty() ? null
> + : ((List) actual.get(0)).toArray();
> + CalciteAssert.assertArrayEqual("expected Elasticsearch query not found", strings,
> + actualArray);
> + return null;
> + }
> + };
> + }
> +}
> +
> +// End ElasticChecker.java
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java b/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> deleted file mode 100644
> index 4e0c2b6..0000000
> --- a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> +++ /dev/null
> @@ -1,270 +0,0 @@
> -/*
> - * Licensed to the Apache Software Foundation (ASF) under one or more
> - * contributor license agreements. See the NOTICE file distributed with
> - * this work for additional information regarding copyright ownership.
> - * The ASF licenses this file to you under the Apache License, Version 2.0
> - * (the "License"); you may not use this file except in compliance with
> - * the License. You may obtain a copy of the License at
> - *
> - * http://www.apache.org/licenses/LICENSE-2.0
> - *
> - * Unless required by applicable law or agreed to in writing, software
> - * distributed under the License is distributed on an "AS IS" BASIS,
> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> - * See the License for the specific language governing permissions and
> - * limitations under the License.
> - */
> -package org.apache.calcite.test;
> -
> -import org.apache.calcite.util.Util;
> -
> -import com.google.common.base.Function;
> -import com.google.common.collect.ImmutableMap;
> -
> -import org.junit.Test;
> -
> -import java.util.List;
> -import javax.annotation.Nullable;
> -
> -/**
> - * Tests for the {@code org.apache.calcite.adapter.elasticsearch2} package.
> - *
> - * <p>Before calling this test, you need to populate Elasticsearch, as follows:
> - *
> - * <blockquote><code>
> - * git clone https://github.com/vlsi/calcite-test-dataset<br>
> - * cd calcite-test-dataset<br>
> - * mvn install
> - * </code></blockquote>
> - *
> - * <p>This will create a virtual machine with Elasticsearch and the "zips" test
> - * dataset.
> - */
> -public class Elasticsearch2AdapterIT {
> - /**
> - * Whether to run Elasticsearch tests. Enabled by default, however test is only
> - * included if "it" profile is activated ({@code -Pit}). To disable,
> - * specify {@code -Dcalcite.test.elasticsearch=false} on the Java command line.
> - */
> - private static final boolean ENABLED = Util.getBooleanProperty("calcite.test.elasticsearch",
> - true);
> -
> - /** Connection factory based on the "zips-es" model. */
> - private static final ImmutableMap<String, String> ZIPS = ImmutableMap.of("model",
> - Elasticsearch2AdapterIT.class.getResource("/elasticsearch-zips-model.json").getPath());
> -
> - /** Whether to run this test. */
> - private boolean enabled() {
> - return ENABLED;
> - }
> -
> - /** Returns a function that checks that a particular Elasticsearch pipeline is
> - * generated to implement a query. */
> - private static Function<List, Void> elasticsearchChecker(final String... strings) {
> - return new Function<List, Void>() {
> - @Nullable
> - @Override public Void apply(@Nullable List actual) {
> - Object[] actualArray = actual == null || actual.isEmpty() ? null
> - : ((List) actual.get(0)).toArray();
> - CalciteAssert.assertArrayEqual("expected Elasticsearch query not found", strings,
> - actualArray);
> - return null;
> - }
> - };
> - }
> -
> - @Test public void testSort() {
> - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> - + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> - + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> - + " ElasticsearchTableScan(table=[[elasticsearch_raw, zips]])";
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query("select * from zips order by \"state\"")
> - .returnsCount(10)
> - .explainContains(explain);
> - }
> -
> - @Test public void testSortLimit() {
> - final String sql = "select \"state\", \"id\" from zips\n"
> - + "order by \"state\", \"id\" offset 2 rows fetch next 3 rows only";
> - CalciteAssert.that()
> - .with(ZIPS)
> - .query(sql)
> - .returnsUnordered("state=AK; id=99503",
> - "state=AK; id=99504",
> - "state=AK; id=99505")
> - .queryContains(
> - elasticsearchChecker(
> - "\"fields\" : [\"state\", \"id\"], \"script_fields\": {}",
> - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]",
> - "\"from\": 2",
> - "\"size\": 3"));
> - }
> -
> - @Test public void testOffsetLimit() {
> - final String sql = "select \"state\", \"id\" from zips\n"
> - + "offset 2 fetch next 3 rows only";
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query(sql)
> - .runs()
> - .queryContains(
> - elasticsearchChecker(
> - "\"from\": 2",
> - "\"size\": 3",
> - "\"fields\" : [\"state\", \"id\"], \"script_fields\": {}"));
> - }
> -
> - @Test public void testLimit() {
> - final String sql = "select \"state\", \"id\" from zips\n"
> - + "fetch next 3 rows only";
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query(sql)
> - .runs()
> - .queryContains(
> - elasticsearchChecker(
> - "\"size\": 3",
> - "\"fields\" : [\"state\", \"id\"], \"script_fields\": {}"));
> - }
> -
> - @Test public void testFilterSort() {
> - final String sql = "select * from zips\n"
> - + "where \"city\" = 'SPRINGFIELD' and \"id\" >= '70000'\n"
> - + "order by \"state\", \"id\"";
> - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> - + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC], dir1=[ASC])\n"
> - + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> - + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'SPRINGFIELD'), >=(CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", '70000'))])\n"
> - + " ElasticsearchTableScan(table=[[elasticsearch_raw, zips]])";
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query(sql)
> - .returnsOrdered(
> - "city=SPRINGFIELD; longitude=-92.54567; latitude=35.274879; pop=752; state=AR; id=72157",
> - "city=SPRINGFIELD; longitude=-102.617322; latitude=37.406727; pop=1992; state=CO; id=81073",
> - "city=SPRINGFIELD; longitude=-90.577479; latitude=30.415738; pop=5597; state=LA; id=70462",
> - "city=SPRINGFIELD; longitude=-123.015259; latitude=44.06106; pop=32384; state=OR; id=97477",
> - "city=SPRINGFIELD; longitude=-122.917108; latitude=44.056056; pop=27521; state=OR; id=97478")
> - .queryContains(
> - elasticsearchChecker("\"query\" : {\"constant_score\":{\"filter\":{\"bool\":"
> - + "{\"must\":[{\"term\":{\"city\":\"springfield\"}},{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
> - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"], \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, \"latitude\":{\"script\":\"_source.loc[1]\"}}",
> - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]"))
> - .explainContains(explain);
> - }
> -
> - @Test public void testFilterSortDesc() {
> - final String sql = "select * from zips\n"
> - + "where \"pop\" BETWEEN 20000 AND 20100\n"
> - + "order by \"state\" desc, \"pop\"";
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query(sql)
> - .limit(4)
> - .returnsOrdered(
> - "city=SHERIDAN; longitude=-106.964795; latitude=44.78486; pop=20025; state=WY; id=82801",
> - "city=MOUNTLAKE TERRAC; longitude=-122.304036; latitude=47.793061; pop=20059; state=WA; id=98043",
> - "city=FALMOUTH; longitude=-77.404537; latitude=38.314557; pop=20039; state=VA; id=22405",
> - "city=FORT WORTH; longitude=-97.318409; latitude=32.725551; pop=20012; state=TX; id=76104");
> - }
> -
> - @Test public void testFilterRedundant() {
> - final String sql = "select * from zips\n"
> - + "where \"state\" > 'CA' and \"state\" < 'AZ' and \"state\" = 'OK'";
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query(sql)
> - .runs()
> - .queryContains(
> - elasticsearchChecker(""
> - + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":"
> - + "{\"must\":[{\"term\":{\"state\":\"ok\"}}]}}}}",
> - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"], \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, \"latitude\":{\"script\":\"_source.loc[1]\"}}"));
> - }
> -
> - @Test public void testInPlan() {
> - final String[] searches = {
> - "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
> - + "[{\"bool\":{\"must\":[{\"term\":{\"pop\":20012}}]}},{\"bool\":{\"must\":[{\"term\":"
> - + "{\"pop\":15590}}]}}]}}}}",
> - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"], \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, \"latitude\":{\"script\":\"_source.loc[1]\"}}"
> - };
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query("select * from zips where \"pop\" in (20012, 15590)")
> - .returnsUnordered(
> - "city=COVINA; longitude=-117.884285; latitude=34.08596; pop=15590; state=CA; id=91723",
> - "city=ARLINGTON; longitude=-97.091987; latitude=32.654752; pop=15590; state=TX; id=76018",
> - "city=CROFTON; longitude=-76.680166; latitude=39.011163; pop=15590; state=MD; id=21114",
> - "city=FORT WORTH; longitude=-97.318409; latitude=32.725551; pop=20012; state=TX; id=76104",
> - "city=DINUBA; longitude=-119.39087; latitude=36.534931; pop=20012; state=CA; id=93618")
> - .queryContains(elasticsearchChecker(searches));
> - }
> -
> - @Test public void testZips() {
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query("select \"state\", \"city\" from zips")
> - .returnsCount(10);
> - }
> -
> - @Test public void testProject() {
> - final String sql = "select \"state\", \"city\", 0 as \"zero\"\n"
> - + "from zips\n"
> - + "order by \"state\", \"city\"";
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query(sql)
> - .limit(2)
> - .returnsUnordered("state=AK; city=ELMENDORF AFB; zero=0",
> - "state=AK; city=EIELSON AFB; zero=0")
> - .queryContains(
> - elasticsearchChecker("\"sort\": [ {\"state\": \"asc\"}, {\"city\": \"asc\"}]",
> - "\"fields\" : [\"state\", \"city\"], \"script_fields\": {\"zero\":{\"script\": \"0\"}}"));
> - }
> -
> - @Test public void testFilter() {
> - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> - + " ElasticsearchProject(state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> - + " ElasticsearchFilter(condition=[=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA')])\n"
> - + " ElasticsearchTableScan(table=[[elasticsearch_raw, zips]])";
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query("select \"state\", \"city\" from zips where \"state\" = 'CA'")
> - .limit(2)
> - .returnsUnordered("state=CA; city=LOS ANGELES",
> - "state=CA; city=LOS ANGELES")
> - .explainContains(explain);
> - }
> -
> - @Test public void testFilterReversed() {
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query("select \"state\", \"city\" from zips where 'WI' < \"state\"")
> - .limit(2)
> - .returnsUnordered("state=WV; city=WELCH",
> - "state=WV; city=HANOVER");
> - CalciteAssert.that()
> - .enable(enabled())
> - .with(ZIPS)
> - .query("select \"state\", \"city\" from zips where \"state\" > 'WI'")
> - .limit(2)
> - .returnsUnordered("state=WV; city=WELCH",
> - "state=WV; city=HANOVER");
> - }
> -}
> -
> -// End Elasticsearch2AdapterIT.java
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/resources/elasticsearch-zips-model.json
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/src/test/resources/elasticsearch-zips-model.json b/elasticsearch2/src/test/resources/elasticsearch-zips-model.json
> deleted file mode 100644
> index 6c00a9c..0000000
> --- a/elasticsearch2/src/test/resources/elasticsearch-zips-model.json
> +++ /dev/null
> @@ -1,50 +0,0 @@
> -/*
> - * Licensed to the Apache Software Foundation (ASF) under one or more
> - * contributor license agreements. See the NOTICE file distributed with
> - * this work for additional information regarding copyright ownership.
> - * The ASF licenses this file to you under the Apache License, Version 2.0
> - * (the License); you may not use this file except in compliance with
> - * the License. You may obtain a copy of the License at
> - *
> - * http://www.apache.org/licenses/LICENSE-2.0
> - *
> - * Unless required by applicable law or agreed to in writing, software
> - * distributed under the License is distributed on an AS IS BASIS,
> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> - * See the License for the specific language governing permissions and
> - * limitations under the License.
> - */
> -{
> - "version": "1.0",
> - "defaultSchema": "elasticsearch",
> - "schemas": [
> - {
> - "type": "custom",
> - "name": "elasticsearch_raw",
> - "factory": "org.apache.calcite.adapter.elasticsearch2.Elasticsearch2SchemaFactory",
> - "operand": {
> - "coordinates": "{'127.0.0.1': 9300}",
> - "userConfig": "{'bulk.flush.max.actions': 10, 'bulk.flush.max.size.mb': 1}",
> - "index": "usa"
> - }
> - },
> - {
> - "name": "elasticsearch",
> - "tables": [
> - {
> - "name": "ZIPS",
> - "type": "view",
> - "sql": [
> - "select cast(_MAP['city'] AS varchar(20)) AS \"city\",\n",
> - " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n",
> - " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n",
> - " cast(_MAP['pop'] AS integer) AS \"pop\",\n",
> - " cast(_MAP['state'] AS varchar(2)) AS \"state\",\n",
> - " cast(_MAP['id'] AS varchar(5)) AS \"id\"\n",
> - "from \"elasticsearch_raw\".\"zips\""
> - ]
> - }
> - ]
> - }
> - ]
> -}
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/resources/zips-mini.json
> ----------------------------------------------------------------------
> diff --git a/elasticsearch2/src/test/resources/zips-mini.json b/elasticsearch2/src/test/resources/zips-mini.json
> new file mode 100644
> index 0000000..d70eadc
> --- /dev/null
> +++ b/elasticsearch2/src/test/resources/zips-mini.json
> @@ -0,0 +1,149 @@
> +{ "_id" : "01701", "city" : "FRAMINGHAM", "loc" : [ -71.42548600000001, 42.300665 ], "pop" : 65046, "state" : "MA" }
> +{ "_id" : "02154", "city" : "NORTH WALTHAM", "loc" : [ -71.236497, 42.382492 ], "pop" : 57871, "state" : "MA" }
> +{ "_id" : "02401", "city" : "BROCKTON", "loc" : [ -71.03434799999999, 42.081571 ], "pop" : 59498, "state" : "MA" }
> +{ "_id" : "02840", "city" : "MIDDLETOWN", "loc" : [ -71.30347999999999, 41.504502 ], "pop" : 47687, "state" : "RI" }
> +{ "_id" : "02860", "city" : "PAWTUCKET", "loc" : [ -71.39071300000001, 41.872873 ], "pop" : 45442, "state" : "RI" }
> +{ "_id" : "02895", "city" : "NORTH SMITHFIELD", "loc" : [ -71.513683, 41.99948 ], "pop" : 53733, "state" : "RI" }
> +{ "_id" : "03060", "city" : "NASHUA", "loc" : [ -71.466684, 42.756395 ], "pop" : 41438, "state" : "NH" }
> +{ "_id" : "03103", "city" : "MANCHESTER", "loc" : [ -71.449325, 42.965563 ], "pop" : 36613, "state" : "NH" }
> +{ "_id" : "03301", "city" : "CONCORD", "loc" : [ -71.527734, 43.218525 ], "pop" : 34035, "state" : "NH" }
> +{ "_id" : "04240", "city" : "LEWISTON", "loc" : [ -70.191619, 44.098538 ], "pop" : 40173, "state" : "ME" }
> +{ "_id" : "04401", "city" : "BANGOR", "loc" : [ -68.791839, 44.824199 ], "pop" : 40434, "state" : "ME" }
> +{ "_id" : "05301", "city" : "BRATTLEBORO", "loc" : [ -72.593322, 42.857353 ], "pop" : 17522, "state" : "VT" }
> +{ "_id" : "05401", "city" : "BURLINGTON", "loc" : [ -73.219875, 44.484023 ], "pop" : 39127, "state" : "VT" }
> +{ "_id" : "05701", "city" : "RUTLAND", "loc" : [ -72.97077299999999, 43.614131 ], "pop" : 22576, "state" : "VT" }
> +{ "_id" : "06010", "city" : "BRISTOL", "loc" : [ -72.930193, 41.682293 ], "pop" : 60670, "state" : "CT" }
> +{ "_id" : "06450", "city" : "MERIDEN", "loc" : [ -72.799734, 41.533396 ], "pop" : 59441, "state" : "CT" }
> +{ "_id" : "06902", "city" : "STAMFORD", "loc" : [ -73.53742800000001, 41.052552 ], "pop" : 54605, "state" : "CT" }
> +{ "_id" : "07002", "city" : "BAYONNE", "loc" : [ -74.119169, 40.666399 ], "pop" : 61444, "state" : "NJ" }
> +{ "_id" : "07087", "city" : "WEEHAWKEN", "loc" : [ -74.030558, 40.768153 ], "pop" : 69646, "state" : "NJ" }
> +{ "_id" : "07111", "city" : "IRVINGTON", "loc" : [ -74.23127100000001, 40.7261 ], "pop" : 60986, "state" : "NJ" }
> +{ "_id" : "10021", "city" : "NEW YORK", "loc" : [ -73.958805, 40.768476 ], "pop" : 106564, "state" : "NY" }
> +{ "_id" : "11226", "city" : "BROOKLYN", "loc" : [ -73.956985, 40.646694 ], "pop" : 111396, "state" : "NY" }
> +{ "_id" : "11373", "city" : "JACKSON HEIGHTS", "loc" : [ -73.878551, 40.740388 ], "pop" : 88241, "state" : "NY" }
> +{ "_id" : "17042", "city" : "CLEONA", "loc" : [ -76.425895, 40.335912 ], "pop" : 61993, "state" : "PA" }
> +{ "_id" : "18042", "city" : "FORKS TOWNSHIP", "loc" : [ -75.23582, 40.6867 ], "pop" : 65784, "state" : "PA" }
> +{ "_id" : "19143", "city" : "PHILADELPHIA", "loc" : [ -75.228819, 39.944815 ], "pop" : 80454, "state" : "PA" }
> +{ "_id" : "19711", "city" : "NEWARK", "loc" : [ -75.737534, 39.701129 ], "pop" : 50573, "state" : "DE" }
> +{ "_id" : "19720", "city" : "MANOR", "loc" : [ -75.589938, 39.67703 ], "pop" : 46906, "state" : "DE" }
> +{ "_id" : "19901", "city" : "DOVER", "loc" : [ -75.535983, 39.156639 ], "pop" : 46005, "state" : "DE" }
> +{ "_id" : "20011", "city" : "WASHINGTON", "loc" : [ -77.020251, 38.951786 ], "pop" : 62924, "state" : "DC" }
> +{ "_id" : "20301", "city" : "PENTAGON", "loc" : [ -77.038196, 38.891019 ], "pop" : 21, "state" : "DC" }
> +{ "_id" : "21061", "city" : "GLEN BURNIE", "loc" : [ -76.61886199999999, 39.158968 ], "pop" : 75692, "state" : "MD" }
> +{ "_id" : "21207", "city" : "GWYNN OAK", "loc" : [ -76.734064, 39.329628 ], "pop" : 76002, "state" : "MD" }
> +{ "_id" : "21215", "city" : "BALTIMORE", "loc" : [ -76.67939699999999, 39.344572 ], "pop" : 74402, "state" : "MD" }
> +{ "_id" : "22901", "city" : "CHARLOTTESVILLE", "loc" : [ -78.490869, 38.054752 ], "pop" : 62708, "state" : "VA" }
> +{ "_id" : "23464", "city" : "VIRGINIA BEACH", "loc" : [ -76.175909, 36.797772 ], "pop" : 67276, "state" : "VA" }
> +{ "_id" : "23602", "city" : "NEWPORT NEWS", "loc" : [ -76.53212499999999, 37.131684 ], "pop" : 68525, "state" : "VA" }
> +{ "_id" : "25801", "city" : "BECKLEY", "loc" : [ -81.206084, 37.793214 ], "pop" : 45196, "state" : "WV" }
> +{ "_id" : "26003", "city" : "ELM GROVE", "loc" : [ -80.685126, 40.072736 ], "pop" : 49136, "state" : "WV" }
> +{ "_id" : "26505", "city" : "STAR CITY", "loc" : [ -79.95422499999999, 39.633858 ], "pop" : 70185, "state" : "WV" }
> +{ "_id" : "27292", "city" : "LEXINGTON", "loc" : [ -80.262049, 35.82306 ], "pop" : 69179, "state" : "NC" }
> +{ "_id" : "28677", "city" : "STATESVILLE", "loc" : [ -80.894009, 35.799022 ], "pop" : 52895, "state" : "NC" }
> +{ "_id" : "29150", "city" : "OSWEGO", "loc" : [ -80.32100800000001, 33.928199 ], "pop" : 46394, "state" : "SC" }
> +{ "_id" : "29501", "city" : "FLORENCE", "loc" : [ -79.772786, 34.18375 ], "pop" : 66990, "state" : "SC" }
> +{ "_id" : "29801", "city" : "AIKEN", "loc" : [ -81.71942900000001, 33.553024 ], "pop" : 51233, "state" : "SC" }
> +{ "_id" : "30032", "city" : "DECATUR", "loc" : [ -84.263165, 33.740825 ], "pop" : 56056, "state" : "GA" }
> +{ "_id" : "30906", "city" : "PEACH ORCHARD", "loc" : [ -82.038358, 33.402024 ], "pop" : 58646, "state" : "GA" }
> +{ "_id" : "32216", "city" : "JACKSONVILLE", "loc" : [ -81.547387, 30.293907 ], "pop" : 58867, "state" : "FL" }
> +{ "_id" : "33012", "city" : "HIALEAH", "loc" : [ -80.30589999999999, 25.865395 ], "pop" : 73194, "state" : "FL" }
> +{ "_id" : "33311", "city" : "FORT LAUDERDALE", "loc" : [ -80.172786, 26.142104 ], "pop" : 65378, "state" : "FL" }
> +{ "_id" : "35215", "city" : "CENTER POINT", "loc" : [ -86.693197, 33.635447 ], "pop" : 43862, "state" : "AL" }
> +{ "_id" : "35401", "city" : "TUSCALOOSA", "loc" : [ -87.56266599999999, 33.196891 ], "pop" : 42124, "state" : "AL" }
> +{ "_id" : "35901", "city" : "SOUTHSIDE", "loc" : [ -86.010279, 33.997248 ], "pop" : 44165, "state" : "AL" }
> +{ "_id" : "37042", "city" : "CLARKSVILLE", "loc" : [ -87.418621, 36.585315 ], "pop" : 43296, "state" : "TN" }
> +{ "_id" : "37211", "city" : "NASHVILLE", "loc" : [ -86.72403799999999, 36.072486 ], "pop" : 51478, "state" : "TN" }
> +{ "_id" : "38109", "city" : "MEMPHIS", "loc" : [ -90.073238, 35.042538 ], "pop" : 60508, "state" : "TN" }
> +{ "_id" : "39180", "city" : "VICKSBURG", "loc" : [ -90.85065, 32.325824 ], "pop" : 46968, "state" : "MS" }
> +{ "_id" : "39401", "city" : "HATTIESBURG", "loc" : [ -89.306471, 31.314553 ], "pop" : 41866, "state" : "MS" }
> +{ "_id" : "39440", "city" : "LAUREL", "loc" : [ -89.13115500000001, 31.705444 ], "pop" : 45040, "state" : "MS" }
> +{ "_id" : "40214", "city" : "LOUISVILLE", "loc" : [ -85.77802699999999, 38.159318 ], "pop" : 42198, "state" : "KY" }
> +{ "_id" : "40216", "city" : "SHIVELY", "loc" : [ -85.831771, 38.186138 ], "pop" : 41719, "state" : "KY" }
> +{ "_id" : "40601", "city" : "HATTON", "loc" : [ -84.88061, 38.192831 ], "pop" : 46563, "state" : "KY" }
> +{ "_id" : "44035", "city" : "ELYRIA", "loc" : [ -82.10508799999999, 41.372353 ], "pop" : 66674, "state" : "OH" }
> +{ "_id" : "44060", "city" : "MENTOR", "loc" : [ -81.342133, 41.689468 ], "pop" : 60109, "state" : "OH" }
> +{ "_id" : "44107", "city" : "EDGEWATER", "loc" : [ -81.79714300000001, 41.482654 ], "pop" : 59702, "state" : "OH" }
> +{ "_id" : "46360", "city" : "MICHIGAN CITY", "loc" : [ -86.869899, 41.698031 ], "pop" : 55392, "state" : "IN" }
> +{ "_id" : "47130", "city" : "JEFFERSONVILLE", "loc" : [ -85.735885, 38.307767 ], "pop" : 56543, "state" : "IN" }
> +{ "_id" : "47906", "city" : "WEST LAFAYETTE", "loc" : [ -86.923661, 40.444025 ], "pop" : 54702, "state" : "IN" }
> +{ "_id" : "48180", "city" : "TAYLOR", "loc" : [ -83.267269, 42.231738 ], "pop" : 70811, "state" : "MI" }
> +{ "_id" : "48185", "city" : "WESTLAND", "loc" : [ -83.374908, 42.318882 ], "pop" : 84712, "state" : "MI" }
> +{ "_id" : "48227", "city" : "DETROIT", "loc" : [ -83.193732, 42.388303 ], "pop" : 68390, "state" : "MI" }
> +{ "_id" : "50010", "city" : "AMES", "loc" : [ -93.639398, 42.029859 ], "pop" : 52105, "state" : "IA" }
> +{ "_id" : "50317", "city" : "PLEASANT HILL", "loc" : [ -93.549446, 41.612499 ], "pop" : 39883, "state" : "IA" }
> +{ "_id" : "52001", "city" : "DUBUQUE", "loc" : [ -90.68191400000001, 42.514977 ], "pop" : 41934, "state" : "IA" }
> +{ "_id" : "53209", "city" : "MILWAUKEE", "loc" : [ -87.947834, 43.118765 ], "pop" : 51008, "state" : "WI" }
> +{ "_id" : "54401", "city" : "WAUSAU", "loc" : [ -89.633955, 44.963433 ], "pop" : 51083, "state" : "WI" }
> +{ "_id" : "54901", "city" : "OSHKOSH", "loc" : [ -88.54363499999999, 44.021962 ], "pop" : 57187, "state" : "WI" }
> +{ "_id" : "55106", "city" : "SAINT PAUL", "loc" : [ -93.048817, 44.968384 ], "pop" : 47905, "state" : "MN" }
> +{ "_id" : "55112", "city" : "NEW BRIGHTON", "loc" : [ -93.199691, 45.074129 ], "pop" : 44128, "state" : "MN" }
> +{ "_id" : "55337", "city" : "BURNSVILLE", "loc" : [ -93.275283, 44.76086 ], "pop" : 51421, "state" : "MN" }
> +{ "_id" : "57103", "city" : "SIOUX FALLS", "loc" : [ -96.686415, 43.537386 ], "pop" : 32508, "state" : "SD" }
> +{ "_id" : "57401", "city" : "ABERDEEN", "loc" : [ -98.485642, 45.466109 ], "pop" : 28786, "state" : "SD" }
> +{ "_id" : "57701", "city" : "ROCKERVILLE", "loc" : [ -103.200259, 44.077041 ], "pop" : 45328, "state" : "SD" }
> +{ "_id" : "58103", "city" : "FARGO", "loc" : [ -96.812252, 46.856406 ], "pop" : 38483, "state" : "ND" }
> +{ "_id" : "58501", "city" : "BISMARCK", "loc" : [ -100.774755, 46.823448 ], "pop" : 36602, "state" : "ND" }
> +{ "_id" : "58701", "city" : "MINOT", "loc" : [ -101.298476, 48.22914 ], "pop" : 42195, "state" : "ND" }
> +{ "_id" : "59102", "city" : "BILLINGS", "loc" : [ -108.572662, 45.781265 ], "pop" : 40121, "state" : "MT" }
> +{ "_id" : "59601", "city" : "HELENA", "loc" : [ -112.021283, 46.613066 ], "pop" : 40102, "state" : "MT" }
> +{ "_id" : "59801", "city" : "MISSOULA", "loc" : [ -114.025207, 46.856274 ], "pop" : 33811, "state" : "MT" }
> +{ "_id" : "60623", "city" : "CHICAGO", "loc" : [ -87.7157, 41.849015 ], "pop" : 112047, "state" : "IL" }
> +{ "_id" : "60634", "city" : "NORRIDGE", "loc" : [ -87.796054, 41.945213 ], "pop" : 69160, "state" : "IL" }
> +{ "_id" : "60650", "city" : "CICERO", "loc" : [ -87.76008, 41.84776 ], "pop" : 67670, "state" : "IL" }
> +{ "_id" : "63031", "city" : "FLORISSANT", "loc" : [ -90.340097, 38.806865 ], "pop" : 52659, "state" : "MO" }
> +{ "_id" : "63116", "city" : "SAINT LOUIS", "loc" : [ -90.26254299999999, 38.581356 ], "pop" : 49014, "state" : "MO" }
> +{ "_id" : "63136", "city" : "JENNINGS", "loc" : [ -90.260189, 38.738878 ], "pop" : 54994, "state" : "MO" }
> +{ "_id" : "66502", "city" : "MANHATTAN", "loc" : [ -96.585776, 39.193757 ], "pop" : 50178, "state" : "KS" }
> +{ "_id" : "67212", "city" : "WICHITA", "loc" : [ -97.438344, 37.700683 ], "pop" : 41349, "state" : "KS" }
> +{ "_id" : "67401", "city" : "BAVARIA", "loc" : [ -97.60878700000001, 38.823802 ], "pop" : 45208, "state" : "KS" }
> +{ "_id" : "68104", "city" : "OMAHA", "loc" : [ -95.999888, 41.29186 ], "pop" : 35325, "state" : "NE" }
> +{ "_id" : "68502", "city" : "LINCOLN", "loc" : [ -96.693763, 40.789282 ], "pop" : 27576, "state" : "NE" }
> +{ "_id" : "68847", "city" : "KEARNEY", "loc" : [ -99.077883, 40.713608 ], "pop" : 28674, "state" : "NE" }
> +{ "_id" : "70072", "city" : "MARRERO", "loc" : [ -90.110462, 29.859756 ], "pop" : 58905, "state" : "LA" }
> +{ "_id" : "70117", "city" : "NEW ORLEANS", "loc" : [ -90.03124, 29.970298 ], "pop" : 56494, "state" : "LA" }
> +{ "_id" : "70560", "city" : "NEW IBERIA", "loc" : [ -91.819959, 30.001027 ], "pop" : 56105, "state" : "LA" }
> +{ "_id" : "72032", "city" : "CONWAY", "loc" : [ -92.423574, 35.084199 ], "pop" : 43236, "state" : "AR" }
> +{ "_id" : "72076", "city" : "GRAVEL RIDGE", "loc" : [ -92.13043500000001, 34.881985 ], "pop" : 37428, "state" : "AR" }
> +{ "_id" : "72401", "city" : "JONESBORO", "loc" : [ -90.69652600000001, 35.833016 ], "pop" : 53532, "state" : "AR" }
> +{ "_id" : "73034", "city" : "EDMOND", "loc" : [ -97.47983499999999, 35.666483 ], "pop" : 43814, "state" : "OK" }
> +{ "_id" : "73505", "city" : "LAWTON", "loc" : [ -98.455234, 34.617939 ], "pop" : 45542, "state" : "OK" }
> +{ "_id" : "74801", "city" : "SHAWNEE", "loc" : [ -96.931321, 35.34907 ], "pop" : 40076, "state" : "OK" }
> +{ "_id" : "78207", "city" : "SAN ANTONIO", "loc" : [ -98.52596699999999, 29.422855 ], "pop" : 58355, "state" : "TX" }
> +{ "_id" : "78521", "city" : "BROWNSVILLE", "loc" : [ -97.461236, 25.922103 ], "pop" : 79463, "state" : "TX" }
> +{ "_id" : "78572", "city" : "ALTON", "loc" : [ -98.342647, 26.24153 ], "pop" : 67604, "state" : "TX" }
> +{ "_id" : "80123", "city" : "BOW MAR", "loc" : [ -105.07766, 39.596854 ], "pop" : 59418, "state" : "CO" }
> +{ "_id" : "80221", "city" : "FEDERAL HEIGHTS", "loc" : [ -105.007985, 39.840562 ], "pop" : 54069, "state" : "CO" }
> +{ "_id" : "80631", "city" : "GARDEN CITY", "loc" : [ -104.704756, 40.413968 ], "pop" : 53905, "state" : "CO" }
> +{ "_id" : "82001", "city" : "CHEYENNE", "loc" : [ -104.796234, 41.143719 ], "pop" : 33107, "state" : "WY" }
> +{ "_id" : "82070", "city" : "LARAMIE", "loc" : [ -105.581146, 41.312907 ], "pop" : 29327, "state" : "WY" }
> +{ "_id" : "82716", "city" : "GILLETTE", "loc" : [ -105.497442, 44.282009 ], "pop" : 25968, "state" : "WY" }
> +{ "_id" : "83301", "city" : "TWIN FALLS", "loc" : [ -114.469265, 42.556495 ], "pop" : 34539, "state" : "ID" }
> +{ "_id" : "83704", "city" : "BOISE", "loc" : [ -116.295099, 43.633001 ], "pop" : 40912, "state" : "ID" }
> +{ "_id" : "83814", "city" : "COEUR D ALENE", "loc" : [ -116.784976, 47.692841 ], "pop" : 33589, "state" : "ID" }
> +{ "_id" : "84118", "city" : "KEARNS", "loc" : [ -111.98521, 40.652759 ], "pop" : 55999, "state" : "UT" }
> +{ "_id" : "84120", "city" : "WEST VALLEY CITY", "loc" : [ -112.009783, 40.68708 ], "pop" : 52854, "state" : "UT" }
> +{ "_id" : "84604", "city" : "PROVO", "loc" : [ -111.654906, 40.260681 ], "pop" : 43841, "state" : "UT" }
> +{ "_id" : "85023", "city" : "PHOENIX", "loc" : [ -112.111838, 33.632383 ], "pop" : 54668, "state" : "AZ" }
> +{ "_id" : "85204", "city" : "MESA", "loc" : [ -111.789554, 33.399168 ], "pop" : 55180, "state" : "AZ" }
> +{ "_id" : "85364", "city" : "YUMA", "loc" : [ -114.642362, 32.701507 ], "pop" : 57131, "state" : "AZ" }
> +{ "_id" : "87501", "city" : "POJOAQUE VALLEY", "loc" : [ -105.974818, 35.702472 ], "pop" : 51715, "state" : "NM" }
> +{ "_id" : "88001", "city" : "LAS CRUCES", "loc" : [ -106.746034, 32.321641 ], "pop" : 57502, "state" : "NM" }
> +{ "_id" : "88201", "city" : "ROSWELL", "loc" : [ -104.525857, 33.388504 ], "pop" : 53644, "state" : "NM" }
> +{ "_id" : "89031", "city" : "NORTH LAS VEGAS", "loc" : [ -115.124832, 36.206228 ], "pop" : 48113, "state" : "NV" }
> +{ "_id" : "89115", "city" : "LAS VEGAS", "loc" : [ -115.067062, 36.215818 ], "pop" : 51532, "state" : "NV" }
> +{ "_id" : "89502", "city" : "RENO", "loc" : [ -119.776395, 39.497239 ], "pop" : 38332, "state" : "NV" }
> +{ "_id" : "90011", "city" : "LOS ANGELES", "loc" : [ -118.258189, 34.007856 ], "pop" : 96074, "state" : "CA" }
> +{ "_id" : "90201", "city" : "BELL GARDENS", "loc" : [ -118.17205, 33.969177 ], "pop" : 99568, "state" : "CA" }
> +{ "_id" : "90650", "city" : "NORWALK", "loc" : [ -118.081767, 33.90564 ], "pop" : 94188, "state" : "CA" }
> +{ "_id" : "96734", "city" : "KAILUA", "loc" : [ -157.744781, 21.406262 ], "pop" : 53403, "state" : "HI" }
> +{ "_id" : "96744", "city" : "KANEOHE", "loc" : [ -157.811543, 21.422819 ], "pop" : 55236, "state" : "HI" }
> +{ "_id" : "96818", "city" : "HONOLULU", "loc" : [ -157.926925, 21.353173 ], "pop" : 62915, "state" : "HI" }
> +{ "_id" : "97005", "city" : "BEAVERTON", "loc" : [ -122.805395, 45.475035 ], "pop" : 46660, "state" : "OR" }
> +{ "_id" : "97206", "city" : "PORTLAND", "loc" : [ -122.59727, 45.483995 ], "pop" : 43134, "state" : "OR" }
> +{ "_id" : "97301", "city" : "SALEM", "loc" : [ -122.979692, 44.926039 ], "pop" : 48007, "state" : "OR" }
> +{ "_id" : "98031", "city" : "KENT", "loc" : [ -122.193184, 47.388004 ], "pop" : 50515, "state" : "WA" }
> +{ "_id" : "98059", "city" : "RENTON", "loc" : [ -122.151178, 47.467383 ], "pop" : 48197, "state" : "WA" }
> +{ "_id" : "98310", "city" : "BREMERTON", "loc" : [ -122.629913, 47.601916 ], "pop" : 49057, "state" : "WA" }
> +{ "_id" : "99504", "city" : "ANCHORAGE", "loc" : [ -149.74467, 61.203696 ], "pop" : 32383, "state" : "AK" }
> +{ "_id" : "99709", "city" : "FAIRBANKS", "loc" : [ -147.846917, 64.85437 ], "pop" : 23238, "state" : "AK" }
> +{ "_id" : "99801", "city" : "JUNEAU", "loc" : [ -134.529429, 58.362767 ], "pop" : 24947, "state" : "AK" }
> \ No newline at end of file
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch5/pom.xml
> ----------------------------------------------------------------------
> diff --git a/elasticsearch5/pom.xml b/elasticsearch5/pom.xml
> index 21d60f1..0fe9df9 100644
> --- a/elasticsearch5/pom.xml
> +++ b/elasticsearch5/pom.xml
> @@ -78,6 +78,35 @@ limitations under the License.
> <version>${elasticsearch5-java-driver.version}</version>
> </dependency>
> <dependency>
> + <groupId>org.elasticsearch.plugin</groupId>
> + <artifactId>transport-netty3-client</artifactId>
> + <version>${elasticsearch5-java-driver.version}</version>
> + <scope>test</scope>
> + </dependency>
> + <dependency>
> + <!-- Elastic search doesn't export painless script artifact to maven central.
> + Using 3rd party version (codelibs)
> + This JAR is used only in tests
> + https://discuss.elastic.co/t/5-0-0-using-painless-in-esintegtestcase/64447/13
> + -->
> + <groupId>org.codelibs.elasticsearch.module</groupId>
> + <artifactId>lang-painless</artifactId>
> + <version>${elasticsearch5-java-driver.version}</version>
> + <scope>test</scope>
> + </dependency>
> + <dependency>
> + <groupId>org.apache.logging.log4j</groupId>
> + <artifactId>log4j-api</artifactId>
> + <version>${log4j2.version}</version>
> + <scope>test</scope>
> + </dependency>
> + <dependency>
> + <groupId>org.apache.logging.log4j</groupId>
> + <artifactId>log4j-core</artifactId>
> + <version>${log4j2.version}</version>
> + <scope>test</scope>
> + </dependency>
> + <dependency>
> <groupId>com.carrotsearch</groupId>
> <artifactId>hppc</artifactId>
> <version>${hppc.version}</version>
> @@ -110,6 +139,8 @@ limitations under the License.
> <ignoredUnusedDeclaredDependency>org.apache.calcite.avatica:avatica</ignoredUnusedDeclaredDependency>
> <ignoredUnusedDeclaredDependency>org.slf4j:slf4j-api</ignoredUnusedDeclaredDependency>
> <ignoredUnusedDeclaredDependency>org.slf4j:slf4j-log4j12</ignoredUnusedDeclaredDependency>
> + <ignoredUnusedDeclaredDependency>org.apache.logging.log4j:log4j-api</ignoredUnusedDeclaredDependency>
> + <ignoredUnusedDeclaredDependency>org.apache.logging.log4j:log4j-core</ignoredUnusedDeclaredDependency>
> </ignoredUnusedDeclaredDependencies>
> </configuration>
> </execution>
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Schema.java
> ----------------------------------------------------------------------
> diff --git a/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Schema.java b/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Schema.java
> index 6e16f2f..ebb96c8 100644
> --- a/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Schema.java
> +++ b/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Schema.java
> @@ -22,6 +22,8 @@ import org.apache.calcite.schema.impl.AbstractSchema;
>
> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
>
> +import com.google.common.annotations.VisibleForTesting;
> +import com.google.common.base.Preconditions;
> import com.google.common.collect.ImmutableList;
> import com.google.common.collect.ImmutableMap;
>
> @@ -87,6 +89,17 @@ public class Elasticsearch5Schema extends AbstractSchema
> }
> }
>
> + /**
> + * Allows schema to be instantiated from existing elastic search client.
> + * This constructor is used in tests.
> + */
> + @VisibleForTesting
> + Elasticsearch5Schema(Client client, String index) {
> + this.client = Preconditions.checkNotNull(client, "client");
> + this.index = Preconditions.checkNotNull(index, "index");
> + }
> +
> +
> @Override protected Map<String, Table> getTableMap() {
> final ImmutableMap.Builder<String, Table> builder = ImmutableMap.builder();
>
> @@ -123,8 +136,10 @@ public class Elasticsearch5Schema extends AbstractSchema
>
> final List<DiscoveryNode> nodes =
> ImmutableList.copyOf(transportClient.connectedNodes());
> +
> if (nodes.isEmpty()) {
> - throw new RuntimeException("Cannot connect to any elasticsearch nodes");
> + throw new IllegalStateException("Cannot connect to any elasticsearch node from "
> + + transportNodes);
> }
>
> client = transportClient;
>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Table.java
> ----------------------------------------------------------------------
> diff --git a/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Table.java b/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Table.java
> index 175e3cc..66e3edf 100644
> --- a/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Table.java
> +++ b/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Table.java
> @@ -25,15 +25,19 @@ import org.apache.calcite.linq4j.function.Function1;
> import org.apache.calcite.util.Util;
>
> import org.elasticsearch.client.Client;
> +import org.elasticsearch.common.settings.Settings;
> import org.elasticsearch.common.xcontent.NamedXContentRegistry;
> import org.elasticsearch.common.xcontent.XContent;
> import org.elasticsearch.common.xcontent.XContentParser;
> import org.elasticsearch.common.xcontent.json.JsonXContent;
> import org.elasticsearch.index.query.QueryParseContext;
> import org.elasticsearch.search.SearchHit;
> +import org.elasticsearch.search.SearchModule;
> import org.elasticsearch.search.builder.SearchSourceBuilder;
>
> import java.io.IOException;
> +import java.io.UncheckedIOException;
> +import java.util.Collections;
> import java.util.Iterator;
> import java.util.List;
> import java.util.Map;
> @@ -61,13 +65,16 @@ public class Elasticsearch5Table extends AbstractElasticsearchTable {
> searchSourceBuilder = new SearchSourceBuilder();
> } else {
> String queryString = "{" + Util.toString(ops, "", ", ", "") + "}";
> - NamedXContentRegistry xContentRegistry = NamedXContentRegistry.EMPTY;
> XContent xContent = JsonXContent.jsonXContent;
> + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
> + NamedXContentRegistry xContentRegistry =
> + new NamedXContentRegistry(searchModule.getNamedXContents());
> +
> try (XContentParser parser = xContent.createParser(xContentRegistry, queryString)) {
> final QueryParseContext queryParseContext = new QueryParseContext(parser);
> searchSourceBuilder = SearchSourceBuilder.fromXContent(queryParseContext);
> } catch (IOException ex) {
> - throw new RuntimeException(ex);
> + throw new UncheckedIOException(ex);
> }
> }
> final Function1<SearchHit, Object> getter = Elasticsearch5Enumerator.getter(fields);
>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Volodymyr Vysotskyi <vo...@apache.org>.
Thanks for the help.
Yes, when calcite is built before running "mvn site" without "clean" it
succeeds.
Kind regards,
Volodymyr Vysotskyi
чт, 28 черв. 2018 о 23:25 Andrei Sereda <an...@sereda.cc> пише:
> You need to build and install core module first. CalciteAssert is part of
> core-test jar.
>
> $ cd core && mvn -DskipTests clean install
>
> Then run site task in mongodb.
>
> Let me know if it helps.
>
>
> On Thu, Jun 28, 2018 at 4:19 PM Volodymyr Vysotskyi <vo...@apache.org>
> wrote:
>
> > Error message is:
> >
> > [ERROR] Failed to execute goal
> > org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> > project calcite-mongodb: Error generating
> > maven-javadoc-plugin:3.0.1:test-javadoc report:
> > [ERROR] Exit code: 1 -
> >
> >
> /home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:22:
> > error: cannot find symbol
> > [ERROR] import org.apache.calcite.test.CalciteAssert;
> > [ERROR] ^
> > [ERROR] symbol: class CalciteAssert
> > [ERROR] location: package org.apache.calcite.test
> > [ERROR]
> >
> >
> /home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:140:
> > error: package CalciteAssert does not exist
> > [ERROR] private CalciteAssert.AssertThat assertModel(String model) {
> > [ERROR] ^
> > [ERROR]
> >
> >
> /home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:148:
> > error: package CalciteAssert does not exist
> > [ERROR] private CalciteAssert.AssertThat assertModel(URL url) {
> > [ERROR] ^
> > [ERROR]
> >
> >
> /home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:44:
> > error: reference not found
> > [ERROR] * Similar to {@link CalciteAssert#checkResultUnordered}, but
> > filters strings
> > [ERROR] ^
> > [ERROR]
> >
> >
> /home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:36:
> > error: reference not found
> > [ERROR] * Util class which needs to be in the same package as {@link
> > CalciteAssert}
> > [ERROR] ^
> >
> > Kind regards,
> > Volodymyr Vysotskyi
> >
> >
> > чт, 28 черв. 2018 о 23:07 Andrei Sereda <an...@sereda.cc> пише:
> >
> > > What exactly is the error message ?
> > >
> > > On Thu, Jun 28, 2018 at 4:01 PM Volodymyr Vysotskyi <
> > volodymyr@apache.org>
> > > wrote:
> > >
> > > > Just to clarify: in my previous letter, I meant that only "mvn
> > > > javadoc:javadoc
> > > > javadoc:test-javadoc" was fixed.
> > > > But "mvn clean -DskipTests site" fails on my PC as before.
> > > >
> > > > Kind regards,
> > > > Volodymyr Vysotskyi
> > > >
> > > >
> > > > чт, 28 черв. 2018 о 01:03 Andrei Sereda <an...@sereda.cc> пише:
> > > >
> > > > > Submitted PR-744 <https://github.com/apache/calcite/pull/744> to
> > > unify
> > > > > ES2
> > > > > and ES5.
> > > > > Javadoc Classpath collisions shouldn't happen anymore.
> > > > >
> > > > > On Mon, Jun 25, 2018 at 8:51 PM Michael Mior <mm...@apache.org>
> > wrote:
> > > > >
> > > > > > Thanks!
> > > > > > --
> > > > > > Michael Mior
> > > > > > mmior@apache.org
> > > > > >
> > > > > >
> > > > > >
> > > > > > Le lun. 25 juin 2018 à 20:17, Julian Hyde <jh...@apache.org> a
> > > écrit :
> > > > > >
> > > > > > > I’m reviewing and testing just the package-exclusion part now.
> > > > > Hopefully
> > > > > > > will commit shortly.
> > > > > > >
> > > > > > > > On Jun 25, 2018, at 2:11 PM, Andrei Sereda <andrei@sereda.cc
> >
> > > > wrote:
> > > > > > > >
> > > > > > > > PS. ES2+ES5 changes will be submitted in a couple of days.
> > > > > > > >
> > > > > > > > On Mon, Jun 25, 2018 at 4:56 PM Andrei Sereda
> <andrei@sereda.cc
> > >
> > > > > > wrote:
> > > > > > > >
> > > > > > > >> Temporary excluded elasticsearch2 package from javadoc. This
> > > > should
> > > > > > > >> unblock users.
> > > > > > > >> Tested with JDK 8,9 and 10 (maven 3.5.2 / 3.5.4) macOSX
> > > > > > > >>
> > > > > > > >> Please double-check PR-741 <
> > > > > > https://github.com/apache/calcite/pull/741>
> > > > > > > >>
> > > > > > > >> CALCITE-2376 (unify ES2 and ES5) has been agreed upon with
> > > > Christian
> > > > > > and
> > > > > > > >> Michael. Once implemented, it should automatically fix
> current
> > > > > > classpath
> > > > > > > >> issue in javadoc.
> > > > > > > >>
> > > > > > > >>
> > > > > > > >> On Mon, Jun 25, 2018 at 2:23 PM Andrei Sereda
> > <andrei@sereda.cc
> > > >
> > > > > > wrote:
> > > > > > > >>
> > > > > > > >>> OK. I'm working on it.
> > > > > > > >>>
> > > > > > > >>> On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <
> > jhyde@apache.org>
> > > > > > wrote:
> > > > > > > >>>
> > > > > > > >>>>> On Jun 25, 2018, at 11:11 AM, Andrei Sereda
> > <andrei@sereda.cc
> > > >
> > > > > > > wrote:
> > > > > > > >>>>>
> > > > > > > >>>>> Would you consider merging this partial javadoc fix (PR
> 741
> > > > > > > >>>>> <https://github.com/apache/calcite/pull/741>) as
> precursor
> > > for
> > > > > > > >>>> CALCITE-2376 ?
> > > > > > > >>>>> Former PR has also fixes javadoc issues for mongo
> adapter.
> > > > > > > >>>>
> > > > > > > >>>> No.
> > > > > > > >>>>
> > > > > > > >>>> I have 5 commits that I would like to merge but cannot
> > because
> > > > my
> > > > > > > builds
> > > > > > > >>>> fail. We need to fix the big problem before we do anything
> > > else.
> > > > > > > >>>>
> > > > > > > >>>>> I'm working on CALCITE-2376 which unifies ES2 and ES5
> > > adapters.
> > > > > > I'll
> > > > > > > >>>> make
> > > > > > > >>>>> sure it is passing both javadoc / site tasks in maven.
> > > > > > > >>>>
> > > > > > > >>>> Sounds very cool, but I’m not an ES expert. I think you
> > should
> > > > get
> > > > > > > >>>> buy-in from other people who have worked on the ES
> > adapter(s)
> > > > > before
> > > > > > > you go
> > > > > > > >>>> too far down that path.
> > > > > > > >>>>
> > > > > > > >>>> Also, it is a big solution to a small (but urgent)
> problem.
> > > > Let’s
> > > > > > fix
> > > > > > > >>>> that problem with a quick, small fix.
> > > > > > > >>>>
> > > > > > > >>>>> On a separate note, do you think it makes sense to add
> > > javadoc
> > > > > > > >>>> generation
> > > > > > > >>>>> to travis build ?
> > > > > > > >>>>
> > > > > > > >>>> I would love to, but IIRC, travis times out if you add
> > javadoc
> > > > > > > >>>> generation.
> > > > > > > >>>>
> > > > > > > >>>> Julian
> > > > > > > >>>>
> > > > > > > >>>>
> > > > > > >
> > > > > > >
> > > > > >
> > > > >
> > > >
> > >
> >
>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Andrei Sereda <an...@sereda.cc>.
You need to build and install core module first. CalciteAssert is part of
core-test jar.
$ cd core && mvn -DskipTests clean install
Then run site task in mongodb.
Let me know if it helps.
On Thu, Jun 28, 2018 at 4:19 PM Volodymyr Vysotskyi <vo...@apache.org>
wrote:
> Error message is:
>
> [ERROR] Failed to execute goal
> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> project calcite-mongodb: Error generating
> maven-javadoc-plugin:3.0.1:test-javadoc report:
> [ERROR] Exit code: 1 -
>
> /home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:22:
> error: cannot find symbol
> [ERROR] import org.apache.calcite.test.CalciteAssert;
> [ERROR] ^
> [ERROR] symbol: class CalciteAssert
> [ERROR] location: package org.apache.calcite.test
> [ERROR]
>
> /home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:140:
> error: package CalciteAssert does not exist
> [ERROR] private CalciteAssert.AssertThat assertModel(String model) {
> [ERROR] ^
> [ERROR]
>
> /home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:148:
> error: package CalciteAssert does not exist
> [ERROR] private CalciteAssert.AssertThat assertModel(URL url) {
> [ERROR] ^
> [ERROR]
>
> /home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:44:
> error: reference not found
> [ERROR] * Similar to {@link CalciteAssert#checkResultUnordered}, but
> filters strings
> [ERROR] ^
> [ERROR]
>
> /home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:36:
> error: reference not found
> [ERROR] * Util class which needs to be in the same package as {@link
> CalciteAssert}
> [ERROR] ^
>
> Kind regards,
> Volodymyr Vysotskyi
>
>
> чт, 28 черв. 2018 о 23:07 Andrei Sereda <an...@sereda.cc> пише:
>
> > What exactly is the error message ?
> >
> > On Thu, Jun 28, 2018 at 4:01 PM Volodymyr Vysotskyi <
> volodymyr@apache.org>
> > wrote:
> >
> > > Just to clarify: in my previous letter, I meant that only "mvn
> > > javadoc:javadoc
> > > javadoc:test-javadoc" was fixed.
> > > But "mvn clean -DskipTests site" fails on my PC as before.
> > >
> > > Kind regards,
> > > Volodymyr Vysotskyi
> > >
> > >
> > > чт, 28 черв. 2018 о 01:03 Andrei Sereda <an...@sereda.cc> пише:
> > >
> > > > Submitted PR-744 <https://github.com/apache/calcite/pull/744> to
> > unify
> > > > ES2
> > > > and ES5.
> > > > Javadoc Classpath collisions shouldn't happen anymore.
> > > >
> > > > On Mon, Jun 25, 2018 at 8:51 PM Michael Mior <mm...@apache.org>
> wrote:
> > > >
> > > > > Thanks!
> > > > > --
> > > > > Michael Mior
> > > > > mmior@apache.org
> > > > >
> > > > >
> > > > >
> > > > > Le lun. 25 juin 2018 à 20:17, Julian Hyde <jh...@apache.org> a
> > écrit :
> > > > >
> > > > > > I’m reviewing and testing just the package-exclusion part now.
> > > > Hopefully
> > > > > > will commit shortly.
> > > > > >
> > > > > > > On Jun 25, 2018, at 2:11 PM, Andrei Sereda <an...@sereda.cc>
> > > wrote:
> > > > > > >
> > > > > > > PS. ES2+ES5 changes will be submitted in a couple of days.
> > > > > > >
> > > > > > > On Mon, Jun 25, 2018 at 4:56 PM Andrei Sereda <andrei@sereda.cc
> >
> > > > > wrote:
> > > > > > >
> > > > > > >> Temporary excluded elasticsearch2 package from javadoc. This
> > > should
> > > > > > >> unblock users.
> > > > > > >> Tested with JDK 8,9 and 10 (maven 3.5.2 / 3.5.4) macOSX
> > > > > > >>
> > > > > > >> Please double-check PR-741 <
> > > > > https://github.com/apache/calcite/pull/741>
> > > > > > >>
> > > > > > >> CALCITE-2376 (unify ES2 and ES5) has been agreed upon with
> > > Christian
> > > > > and
> > > > > > >> Michael. Once implemented, it should automatically fix current
> > > > > classpath
> > > > > > >> issue in javadoc.
> > > > > > >>
> > > > > > >>
> > > > > > >> On Mon, Jun 25, 2018 at 2:23 PM Andrei Sereda
> <andrei@sereda.cc
> > >
> > > > > wrote:
> > > > > > >>
> > > > > > >>> OK. I'm working on it.
> > > > > > >>>
> > > > > > >>> On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <
> jhyde@apache.org>
> > > > > wrote:
> > > > > > >>>
> > > > > > >>>>> On Jun 25, 2018, at 11:11 AM, Andrei Sereda
> <andrei@sereda.cc
> > >
> > > > > > wrote:
> > > > > > >>>>>
> > > > > > >>>>> Would you consider merging this partial javadoc fix (PR 741
> > > > > > >>>>> <https://github.com/apache/calcite/pull/741>) as precursor
> > for
> > > > > > >>>> CALCITE-2376 ?
> > > > > > >>>>> Former PR has also fixes javadoc issues for mongo adapter.
> > > > > > >>>>
> > > > > > >>>> No.
> > > > > > >>>>
> > > > > > >>>> I have 5 commits that I would like to merge but cannot
> because
> > > my
> > > > > > builds
> > > > > > >>>> fail. We need to fix the big problem before we do anything
> > else.
> > > > > > >>>>
> > > > > > >>>>> I'm working on CALCITE-2376 which unifies ES2 and ES5
> > adapters.
> > > > > I'll
> > > > > > >>>> make
> > > > > > >>>>> sure it is passing both javadoc / site tasks in maven.
> > > > > > >>>>
> > > > > > >>>> Sounds very cool, but I’m not an ES expert. I think you
> should
> > > get
> > > > > > >>>> buy-in from other people who have worked on the ES
> adapter(s)
> > > > before
> > > > > > you go
> > > > > > >>>> too far down that path.
> > > > > > >>>>
> > > > > > >>>> Also, it is a big solution to a small (but urgent) problem.
> > > Let’s
> > > > > fix
> > > > > > >>>> that problem with a quick, small fix.
> > > > > > >>>>
> > > > > > >>>>> On a separate note, do you think it makes sense to add
> > javadoc
> > > > > > >>>> generation
> > > > > > >>>>> to travis build ?
> > > > > > >>>>
> > > > > > >>>> I would love to, but IIRC, travis times out if you add
> javadoc
> > > > > > >>>> generation.
> > > > > > >>>>
> > > > > > >>>> Julian
> > > > > > >>>>
> > > > > > >>>>
> > > > > >
> > > > > >
> > > > >
> > > >
> > >
> >
>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Volodymyr Vysotskyi <vo...@apache.org>.
Error message is:
[ERROR] Failed to execute goal
org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
project calcite-mongodb: Error generating
maven-javadoc-plugin:3.0.1:test-javadoc report:
[ERROR] Exit code: 1 -
/home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:22:
error: cannot find symbol
[ERROR] import org.apache.calcite.test.CalciteAssert;
[ERROR] ^
[ERROR] symbol: class CalciteAssert
[ERROR] location: package org.apache.calcite.test
[ERROR]
/home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:140:
error: package CalciteAssert does not exist
[ERROR] private CalciteAssert.AssertThat assertModel(String model) {
[ERROR] ^
[ERROR]
/home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:148:
error: package CalciteAssert does not exist
[ERROR] private CalciteAssert.AssertThat assertModel(URL url) {
[ERROR] ^
[ERROR]
/home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:44:
error: reference not found
[ERROR] * Similar to {@link CalciteAssert#checkResultUnordered}, but
filters strings
[ERROR] ^
[ERROR]
/home/user515050/work/batch_commits/calcite/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:36:
error: reference not found
[ERROR] * Util class which needs to be in the same package as {@link
CalciteAssert}
[ERROR] ^
Kind regards,
Volodymyr Vysotskyi
чт, 28 черв. 2018 о 23:07 Andrei Sereda <an...@sereda.cc> пише:
> What exactly is the error message ?
>
> On Thu, Jun 28, 2018 at 4:01 PM Volodymyr Vysotskyi <vo...@apache.org>
> wrote:
>
> > Just to clarify: in my previous letter, I meant that only "mvn
> > javadoc:javadoc
> > javadoc:test-javadoc" was fixed.
> > But "mvn clean -DskipTests site" fails on my PC as before.
> >
> > Kind regards,
> > Volodymyr Vysotskyi
> >
> >
> > чт, 28 черв. 2018 о 01:03 Andrei Sereda <an...@sereda.cc> пише:
> >
> > > Submitted PR-744 <https://github.com/apache/calcite/pull/744> to
> unify
> > > ES2
> > > and ES5.
> > > Javadoc Classpath collisions shouldn't happen anymore.
> > >
> > > On Mon, Jun 25, 2018 at 8:51 PM Michael Mior <mm...@apache.org> wrote:
> > >
> > > > Thanks!
> > > > --
> > > > Michael Mior
> > > > mmior@apache.org
> > > >
> > > >
> > > >
> > > > Le lun. 25 juin 2018 à 20:17, Julian Hyde <jh...@apache.org> a
> écrit :
> > > >
> > > > > I’m reviewing and testing just the package-exclusion part now.
> > > Hopefully
> > > > > will commit shortly.
> > > > >
> > > > > > On Jun 25, 2018, at 2:11 PM, Andrei Sereda <an...@sereda.cc>
> > wrote:
> > > > > >
> > > > > > PS. ES2+ES5 changes will be submitted in a couple of days.
> > > > > >
> > > > > > On Mon, Jun 25, 2018 at 4:56 PM Andrei Sereda <an...@sereda.cc>
> > > > wrote:
> > > > > >
> > > > > >> Temporary excluded elasticsearch2 package from javadoc. This
> > should
> > > > > >> unblock users.
> > > > > >> Tested with JDK 8,9 and 10 (maven 3.5.2 / 3.5.4) macOSX
> > > > > >>
> > > > > >> Please double-check PR-741 <
> > > > https://github.com/apache/calcite/pull/741>
> > > > > >>
> > > > > >> CALCITE-2376 (unify ES2 and ES5) has been agreed upon with
> > Christian
> > > > and
> > > > > >> Michael. Once implemented, it should automatically fix current
> > > > classpath
> > > > > >> issue in javadoc.
> > > > > >>
> > > > > >>
> > > > > >> On Mon, Jun 25, 2018 at 2:23 PM Andrei Sereda <andrei@sereda.cc
> >
> > > > wrote:
> > > > > >>
> > > > > >>> OK. I'm working on it.
> > > > > >>>
> > > > > >>> On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <jh...@apache.org>
> > > > wrote:
> > > > > >>>
> > > > > >>>>> On Jun 25, 2018, at 11:11 AM, Andrei Sereda <andrei@sereda.cc
> >
> > > > > wrote:
> > > > > >>>>>
> > > > > >>>>> Would you consider merging this partial javadoc fix (PR 741
> > > > > >>>>> <https://github.com/apache/calcite/pull/741>) as precursor
> for
> > > > > >>>> CALCITE-2376 ?
> > > > > >>>>> Former PR has also fixes javadoc issues for mongo adapter.
> > > > > >>>>
> > > > > >>>> No.
> > > > > >>>>
> > > > > >>>> I have 5 commits that I would like to merge but cannot because
> > my
> > > > > builds
> > > > > >>>> fail. We need to fix the big problem before we do anything
> else.
> > > > > >>>>
> > > > > >>>>> I'm working on CALCITE-2376 which unifies ES2 and ES5
> adapters.
> > > > I'll
> > > > > >>>> make
> > > > > >>>>> sure it is passing both javadoc / site tasks in maven.
> > > > > >>>>
> > > > > >>>> Sounds very cool, but I’m not an ES expert. I think you should
> > get
> > > > > >>>> buy-in from other people who have worked on the ES adapter(s)
> > > before
> > > > > you go
> > > > > >>>> too far down that path.
> > > > > >>>>
> > > > > >>>> Also, it is a big solution to a small (but urgent) problem.
> > Let’s
> > > > fix
> > > > > >>>> that problem with a quick, small fix.
> > > > > >>>>
> > > > > >>>>> On a separate note, do you think it makes sense to add
> javadoc
> > > > > >>>> generation
> > > > > >>>>> to travis build ?
> > > > > >>>>
> > > > > >>>> I would love to, but IIRC, travis times out if you add javadoc
> > > > > >>>> generation.
> > > > > >>>>
> > > > > >>>> Julian
> > > > > >>>>
> > > > > >>>>
> > > > >
> > > > >
> > > >
> > >
> >
>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Andrei Sereda <an...@sereda.cc>.
What exactly is the error message ?
On Thu, Jun 28, 2018 at 4:01 PM Volodymyr Vysotskyi <vo...@apache.org>
wrote:
> Just to clarify: in my previous letter, I meant that only "mvn
> javadoc:javadoc
> javadoc:test-javadoc" was fixed.
> But "mvn clean -DskipTests site" fails on my PC as before.
>
> Kind regards,
> Volodymyr Vysotskyi
>
>
> чт, 28 черв. 2018 о 01:03 Andrei Sereda <an...@sereda.cc> пише:
>
> > Submitted PR-744 <https://github.com/apache/calcite/pull/744> to unify
> > ES2
> > and ES5.
> > Javadoc Classpath collisions shouldn't happen anymore.
> >
> > On Mon, Jun 25, 2018 at 8:51 PM Michael Mior <mm...@apache.org> wrote:
> >
> > > Thanks!
> > > --
> > > Michael Mior
> > > mmior@apache.org
> > >
> > >
> > >
> > > Le lun. 25 juin 2018 à 20:17, Julian Hyde <jh...@apache.org> a écrit :
> > >
> > > > I’m reviewing and testing just the package-exclusion part now.
> > Hopefully
> > > > will commit shortly.
> > > >
> > > > > On Jun 25, 2018, at 2:11 PM, Andrei Sereda <an...@sereda.cc>
> wrote:
> > > > >
> > > > > PS. ES2+ES5 changes will be submitted in a couple of days.
> > > > >
> > > > > On Mon, Jun 25, 2018 at 4:56 PM Andrei Sereda <an...@sereda.cc>
> > > wrote:
> > > > >
> > > > >> Temporary excluded elasticsearch2 package from javadoc. This
> should
> > > > >> unblock users.
> > > > >> Tested with JDK 8,9 and 10 (maven 3.5.2 / 3.5.4) macOSX
> > > > >>
> > > > >> Please double-check PR-741 <
> > > https://github.com/apache/calcite/pull/741>
> > > > >>
> > > > >> CALCITE-2376 (unify ES2 and ES5) has been agreed upon with
> Christian
> > > and
> > > > >> Michael. Once implemented, it should automatically fix current
> > > classpath
> > > > >> issue in javadoc.
> > > > >>
> > > > >>
> > > > >> On Mon, Jun 25, 2018 at 2:23 PM Andrei Sereda <an...@sereda.cc>
> > > wrote:
> > > > >>
> > > > >>> OK. I'm working on it.
> > > > >>>
> > > > >>> On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <jh...@apache.org>
> > > wrote:
> > > > >>>
> > > > >>>>> On Jun 25, 2018, at 11:11 AM, Andrei Sereda <an...@sereda.cc>
> > > > wrote:
> > > > >>>>>
> > > > >>>>> Would you consider merging this partial javadoc fix (PR 741
> > > > >>>>> <https://github.com/apache/calcite/pull/741>) as precursor for
> > > > >>>> CALCITE-2376 ?
> > > > >>>>> Former PR has also fixes javadoc issues for mongo adapter.
> > > > >>>>
> > > > >>>> No.
> > > > >>>>
> > > > >>>> I have 5 commits that I would like to merge but cannot because
> my
> > > > builds
> > > > >>>> fail. We need to fix the big problem before we do anything else.
> > > > >>>>
> > > > >>>>> I'm working on CALCITE-2376 which unifies ES2 and ES5 adapters.
> > > I'll
> > > > >>>> make
> > > > >>>>> sure it is passing both javadoc / site tasks in maven.
> > > > >>>>
> > > > >>>> Sounds very cool, but I’m not an ES expert. I think you should
> get
> > > > >>>> buy-in from other people who have worked on the ES adapter(s)
> > before
> > > > you go
> > > > >>>> too far down that path.
> > > > >>>>
> > > > >>>> Also, it is a big solution to a small (but urgent) problem.
> Let’s
> > > fix
> > > > >>>> that problem with a quick, small fix.
> > > > >>>>
> > > > >>>>> On a separate note, do you think it makes sense to add javadoc
> > > > >>>> generation
> > > > >>>>> to travis build ?
> > > > >>>>
> > > > >>>> I would love to, but IIRC, travis times out if you add javadoc
> > > > >>>> generation.
> > > > >>>>
> > > > >>>> Julian
> > > > >>>>
> > > > >>>>
> > > >
> > > >
> > >
> >
>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Volodymyr Vysotskyi <vo...@apache.org>.
Just to clarify: in my previous letter, I meant that only "mvn javadoc:javadoc
javadoc:test-javadoc" was fixed.
But "mvn clean -DskipTests site" fails on my PC as before.
Kind regards,
Volodymyr Vysotskyi
чт, 28 черв. 2018 о 01:03 Andrei Sereda <an...@sereda.cc> пише:
> Submitted PR-744 <https://github.com/apache/calcite/pull/744> to unify
> ES2
> and ES5.
> Javadoc Classpath collisions shouldn't happen anymore.
>
> On Mon, Jun 25, 2018 at 8:51 PM Michael Mior <mm...@apache.org> wrote:
>
> > Thanks!
> > --
> > Michael Mior
> > mmior@apache.org
> >
> >
> >
> > Le lun. 25 juin 2018 à 20:17, Julian Hyde <jh...@apache.org> a écrit :
> >
> > > I’m reviewing and testing just the package-exclusion part now.
> Hopefully
> > > will commit shortly.
> > >
> > > > On Jun 25, 2018, at 2:11 PM, Andrei Sereda <an...@sereda.cc> wrote:
> > > >
> > > > PS. ES2+ES5 changes will be submitted in a couple of days.
> > > >
> > > > On Mon, Jun 25, 2018 at 4:56 PM Andrei Sereda <an...@sereda.cc>
> > wrote:
> > > >
> > > >> Temporary excluded elasticsearch2 package from javadoc. This should
> > > >> unblock users.
> > > >> Tested with JDK 8,9 and 10 (maven 3.5.2 / 3.5.4) macOSX
> > > >>
> > > >> Please double-check PR-741 <
> > https://github.com/apache/calcite/pull/741>
> > > >>
> > > >> CALCITE-2376 (unify ES2 and ES5) has been agreed upon with Christian
> > and
> > > >> Michael. Once implemented, it should automatically fix current
> > classpath
> > > >> issue in javadoc.
> > > >>
> > > >>
> > > >> On Mon, Jun 25, 2018 at 2:23 PM Andrei Sereda <an...@sereda.cc>
> > wrote:
> > > >>
> > > >>> OK. I'm working on it.
> > > >>>
> > > >>> On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <jh...@apache.org>
> > wrote:
> > > >>>
> > > >>>>> On Jun 25, 2018, at 11:11 AM, Andrei Sereda <an...@sereda.cc>
> > > wrote:
> > > >>>>>
> > > >>>>> Would you consider merging this partial javadoc fix (PR 741
> > > >>>>> <https://github.com/apache/calcite/pull/741>) as precursor for
> > > >>>> CALCITE-2376 ?
> > > >>>>> Former PR has also fixes javadoc issues for mongo adapter.
> > > >>>>
> > > >>>> No.
> > > >>>>
> > > >>>> I have 5 commits that I would like to merge but cannot because my
> > > builds
> > > >>>> fail. We need to fix the big problem before we do anything else.
> > > >>>>
> > > >>>>> I'm working on CALCITE-2376 which unifies ES2 and ES5 adapters.
> > I'll
> > > >>>> make
> > > >>>>> sure it is passing both javadoc / site tasks in maven.
> > > >>>>
> > > >>>> Sounds very cool, but I’m not an ES expert. I think you should get
> > > >>>> buy-in from other people who have worked on the ES adapter(s)
> before
> > > you go
> > > >>>> too far down that path.
> > > >>>>
> > > >>>> Also, it is a big solution to a small (but urgent) problem. Let’s
> > fix
> > > >>>> that problem with a quick, small fix.
> > > >>>>
> > > >>>>> On a separate note, do you think it makes sense to add javadoc
> > > >>>> generation
> > > >>>>> to travis build ?
> > > >>>>
> > > >>>> I would love to, but IIRC, travis times out if you add javadoc
> > > >>>> generation.
> > > >>>>
> > > >>>> Julian
> > > >>>>
> > > >>>>
> > >
> > >
> >
>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Andrei Sereda <an...@sereda.cc>.
Submitted PR-744 <https://github.com/apache/calcite/pull/744> to unify ES2
and ES5.
Javadoc Classpath collisions shouldn't happen anymore.
On Mon, Jun 25, 2018 at 8:51 PM Michael Mior <mm...@apache.org> wrote:
> Thanks!
> --
> Michael Mior
> mmior@apache.org
>
>
>
> Le lun. 25 juin 2018 à 20:17, Julian Hyde <jh...@apache.org> a écrit :
>
> > I’m reviewing and testing just the package-exclusion part now. Hopefully
> > will commit shortly.
> >
> > > On Jun 25, 2018, at 2:11 PM, Andrei Sereda <an...@sereda.cc> wrote:
> > >
> > > PS. ES2+ES5 changes will be submitted in a couple of days.
> > >
> > > On Mon, Jun 25, 2018 at 4:56 PM Andrei Sereda <an...@sereda.cc>
> wrote:
> > >
> > >> Temporary excluded elasticsearch2 package from javadoc. This should
> > >> unblock users.
> > >> Tested with JDK 8,9 and 10 (maven 3.5.2 / 3.5.4) macOSX
> > >>
> > >> Please double-check PR-741 <
> https://github.com/apache/calcite/pull/741>
> > >>
> > >> CALCITE-2376 (unify ES2 and ES5) has been agreed upon with Christian
> and
> > >> Michael. Once implemented, it should automatically fix current
> classpath
> > >> issue in javadoc.
> > >>
> > >>
> > >> On Mon, Jun 25, 2018 at 2:23 PM Andrei Sereda <an...@sereda.cc>
> wrote:
> > >>
> > >>> OK. I'm working on it.
> > >>>
> > >>> On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <jh...@apache.org>
> wrote:
> > >>>
> > >>>>> On Jun 25, 2018, at 11:11 AM, Andrei Sereda <an...@sereda.cc>
> > wrote:
> > >>>>>
> > >>>>> Would you consider merging this partial javadoc fix (PR 741
> > >>>>> <https://github.com/apache/calcite/pull/741>) as precursor for
> > >>>> CALCITE-2376 ?
> > >>>>> Former PR has also fixes javadoc issues for mongo adapter.
> > >>>>
> > >>>> No.
> > >>>>
> > >>>> I have 5 commits that I would like to merge but cannot because my
> > builds
> > >>>> fail. We need to fix the big problem before we do anything else.
> > >>>>
> > >>>>> I'm working on CALCITE-2376 which unifies ES2 and ES5 adapters.
> I'll
> > >>>> make
> > >>>>> sure it is passing both javadoc / site tasks in maven.
> > >>>>
> > >>>> Sounds very cool, but I’m not an ES expert. I think you should get
> > >>>> buy-in from other people who have worked on the ES adapter(s) before
> > you go
> > >>>> too far down that path.
> > >>>>
> > >>>> Also, it is a big solution to a small (but urgent) problem. Let’s
> fix
> > >>>> that problem with a quick, small fix.
> > >>>>
> > >>>>> On a separate note, do you think it makes sense to add javadoc
> > >>>> generation
> > >>>>> to travis build ?
> > >>>>
> > >>>> I would love to, but IIRC, travis times out if you add javadoc
> > >>>> generation.
> > >>>>
> > >>>> Julian
> > >>>>
> > >>>>
> >
> >
>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Michael Mior <mm...@apache.org>.
Thanks!
--
Michael Mior
mmior@apache.org
Le lun. 25 juin 2018 à 20:17, Julian Hyde <jh...@apache.org> a écrit :
> I’m reviewing and testing just the package-exclusion part now. Hopefully
> will commit shortly.
>
> > On Jun 25, 2018, at 2:11 PM, Andrei Sereda <an...@sereda.cc> wrote:
> >
> > PS. ES2+ES5 changes will be submitted in a couple of days.
> >
> > On Mon, Jun 25, 2018 at 4:56 PM Andrei Sereda <an...@sereda.cc> wrote:
> >
> >> Temporary excluded elasticsearch2 package from javadoc. This should
> >> unblock users.
> >> Tested with JDK 8,9 and 10 (maven 3.5.2 / 3.5.4) macOSX
> >>
> >> Please double-check PR-741 <https://github.com/apache/calcite/pull/741>
> >>
> >> CALCITE-2376 (unify ES2 and ES5) has been agreed upon with Christian and
> >> Michael. Once implemented, it should automatically fix current classpath
> >> issue in javadoc.
> >>
> >>
> >> On Mon, Jun 25, 2018 at 2:23 PM Andrei Sereda <an...@sereda.cc> wrote:
> >>
> >>> OK. I'm working on it.
> >>>
> >>> On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <jh...@apache.org> wrote:
> >>>
> >>>>> On Jun 25, 2018, at 11:11 AM, Andrei Sereda <an...@sereda.cc>
> wrote:
> >>>>>
> >>>>> Would you consider merging this partial javadoc fix (PR 741
> >>>>> <https://github.com/apache/calcite/pull/741>) as precursor for
> >>>> CALCITE-2376 ?
> >>>>> Former PR has also fixes javadoc issues for mongo adapter.
> >>>>
> >>>> No.
> >>>>
> >>>> I have 5 commits that I would like to merge but cannot because my
> builds
> >>>> fail. We need to fix the big problem before we do anything else.
> >>>>
> >>>>> I'm working on CALCITE-2376 which unifies ES2 and ES5 adapters. I'll
> >>>> make
> >>>>> sure it is passing both javadoc / site tasks in maven.
> >>>>
> >>>> Sounds very cool, but I’m not an ES expert. I think you should get
> >>>> buy-in from other people who have worked on the ES adapter(s) before
> you go
> >>>> too far down that path.
> >>>>
> >>>> Also, it is a big solution to a small (but urgent) problem. Let’s fix
> >>>> that problem with a quick, small fix.
> >>>>
> >>>>> On a separate note, do you think it makes sense to add javadoc
> >>>> generation
> >>>>> to travis build ?
> >>>>
> >>>> I would love to, but IIRC, travis times out if you add javadoc
> >>>> generation.
> >>>>
> >>>> Julian
> >>>>
> >>>>
>
>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Julian Hyde <jh...@apache.org>.
I’m reviewing and testing just the package-exclusion part now. Hopefully will commit shortly.
> On Jun 25, 2018, at 2:11 PM, Andrei Sereda <an...@sereda.cc> wrote:
>
> PS. ES2+ES5 changes will be submitted in a couple of days.
>
> On Mon, Jun 25, 2018 at 4:56 PM Andrei Sereda <an...@sereda.cc> wrote:
>
>> Temporary excluded elasticsearch2 package from javadoc. This should
>> unblock users.
>> Tested with JDK 8,9 and 10 (maven 3.5.2 / 3.5.4) macOSX
>>
>> Please double-check PR-741 <https://github.com/apache/calcite/pull/741>
>>
>> CALCITE-2376 (unify ES2 and ES5) has been agreed upon with Christian and
>> Michael. Once implemented, it should automatically fix current classpath
>> issue in javadoc.
>>
>>
>> On Mon, Jun 25, 2018 at 2:23 PM Andrei Sereda <an...@sereda.cc> wrote:
>>
>>> OK. I'm working on it.
>>>
>>> On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <jh...@apache.org> wrote:
>>>
>>>>> On Jun 25, 2018, at 11:11 AM, Andrei Sereda <an...@sereda.cc> wrote:
>>>>>
>>>>> Would you consider merging this partial javadoc fix (PR 741
>>>>> <https://github.com/apache/calcite/pull/741>) as precursor for
>>>> CALCITE-2376 ?
>>>>> Former PR has also fixes javadoc issues for mongo adapter.
>>>>
>>>> No.
>>>>
>>>> I have 5 commits that I would like to merge but cannot because my builds
>>>> fail. We need to fix the big problem before we do anything else.
>>>>
>>>>> I'm working on CALCITE-2376 which unifies ES2 and ES5 adapters. I'll
>>>> make
>>>>> sure it is passing both javadoc / site tasks in maven.
>>>>
>>>> Sounds very cool, but I’m not an ES expert. I think you should get
>>>> buy-in from other people who have worked on the ES adapter(s) before you go
>>>> too far down that path.
>>>>
>>>> Also, it is a big solution to a small (but urgent) problem. Let’s fix
>>>> that problem with a quick, small fix.
>>>>
>>>>> On a separate note, do you think it makes sense to add javadoc
>>>> generation
>>>>> to travis build ?
>>>>
>>>> I would love to, but IIRC, travis times out if you add javadoc
>>>> generation.
>>>>
>>>> Julian
>>>>
>>>>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Andrei Sereda <an...@sereda.cc>.
PS. ES2+ES5 changes will be submitted in a couple of days.
On Mon, Jun 25, 2018 at 4:56 PM Andrei Sereda <an...@sereda.cc> wrote:
> Temporary excluded elasticsearch2 package from javadoc. This should
> unblock users.
> Tested with JDK 8,9 and 10 (maven 3.5.2 / 3.5.4) macOSX
>
> Please double-check PR-741 <https://github.com/apache/calcite/pull/741>
>
> CALCITE-2376 (unify ES2 and ES5) has been agreed upon with Christian and
> Michael. Once implemented, it should automatically fix current classpath
> issue in javadoc.
>
>
> On Mon, Jun 25, 2018 at 2:23 PM Andrei Sereda <an...@sereda.cc> wrote:
>
>> OK. I'm working on it.
>>
>> On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <jh...@apache.org> wrote:
>>
>>> > On Jun 25, 2018, at 11:11 AM, Andrei Sereda <an...@sereda.cc> wrote:
>>> >
>>> > Would you consider merging this partial javadoc fix (PR 741
>>> > <https://github.com/apache/calcite/pull/741>) as precursor for
>>> CALCITE-2376 ?
>>> > Former PR has also fixes javadoc issues for mongo adapter.
>>>
>>> No.
>>>
>>> I have 5 commits that I would like to merge but cannot because my builds
>>> fail. We need to fix the big problem before we do anything else.
>>>
>>> > I'm working on CALCITE-2376 which unifies ES2 and ES5 adapters. I'll
>>> make
>>> > sure it is passing both javadoc / site tasks in maven.
>>>
>>> Sounds very cool, but I’m not an ES expert. I think you should get
>>> buy-in from other people who have worked on the ES adapter(s) before you go
>>> too far down that path.
>>>
>>> Also, it is a big solution to a small (but urgent) problem. Let’s fix
>>> that problem with a quick, small fix.
>>>
>>> > On a separate note, do you think it makes sense to add javadoc
>>> generation
>>> > to travis build ?
>>>
>>> I would love to, but IIRC, travis times out if you add javadoc
>>> generation.
>>>
>>> Julian
>>>
>>>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Andrei Sereda <an...@sereda.cc>.
Temporary excluded elasticsearch2 package from javadoc. This should unblock
users.
Tested with JDK 8,9 and 10 (maven 3.5.2 / 3.5.4) macOSX
Please double-check PR-741 <https://github.com/apache/calcite/pull/741>
CALCITE-2376 (unify ES2 and ES5) has been agreed upon with Christian and
Michael. Once implemented, it should automatically fix current classpath
issue in javadoc.
On Mon, Jun 25, 2018 at 2:23 PM Andrei Sereda <an...@sereda.cc> wrote:
> OK. I'm working on it.
>
> On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <jh...@apache.org> wrote:
>
>> > On Jun 25, 2018, at 11:11 AM, Andrei Sereda <an...@sereda.cc> wrote:
>> >
>> > Would you consider merging this partial javadoc fix (PR 741
>> > <https://github.com/apache/calcite/pull/741>) as precursor for
>> CALCITE-2376 ?
>> > Former PR has also fixes javadoc issues for mongo adapter.
>>
>> No.
>>
>> I have 5 commits that I would like to merge but cannot because my builds
>> fail. We need to fix the big problem before we do anything else.
>>
>> > I'm working on CALCITE-2376 which unifies ES2 and ES5 adapters. I'll
>> make
>> > sure it is passing both javadoc / site tasks in maven.
>>
>> Sounds very cool, but I’m not an ES expert. I think you should get buy-in
>> from other people who have worked on the ES adapter(s) before you go too
>> far down that path.
>>
>> Also, it is a big solution to a small (but urgent) problem. Let’s fix
>> that problem with a quick, small fix.
>>
>> > On a separate note, do you think it makes sense to add javadoc
>> generation
>> > to travis build ?
>>
>> I would love to, but IIRC, travis times out if you add javadoc generation.
>>
>> Julian
>>
>>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Andrei Sereda <an...@sereda.cc>.
OK. I'm working on it.
On Mon, Jun 25, 2018 at 2:18 PM Julian Hyde <jh...@apache.org> wrote:
> > On Jun 25, 2018, at 11:11 AM, Andrei Sereda <an...@sereda.cc> wrote:
> >
> > Would you consider merging this partial javadoc fix (PR 741
> > <https://github.com/apache/calcite/pull/741>) as precursor for
> CALCITE-2376 ?
> > Former PR has also fixes javadoc issues for mongo adapter.
>
> No.
>
> I have 5 commits that I would like to merge but cannot because my builds
> fail. We need to fix the big problem before we do anything else.
>
> > I'm working on CALCITE-2376 which unifies ES2 and ES5 adapters. I'll make
> > sure it is passing both javadoc / site tasks in maven.
>
> Sounds very cool, but I’m not an ES expert. I think you should get buy-in
> from other people who have worked on the ES adapter(s) before you go too
> far down that path.
>
> Also, it is a big solution to a small (but urgent) problem. Let’s fix that
> problem with a quick, small fix.
>
> > On a separate note, do you think it makes sense to add javadoc generation
> > to travis build ?
>
> I would love to, but IIRC, travis times out if you add javadoc generation.
>
> Julian
>
>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Julian Hyde <jh...@apache.org>.
> On Jun 25, 2018, at 11:11 AM, Andrei Sereda <an...@sereda.cc> wrote:
>
> Would you consider merging this partial javadoc fix (PR 741
> <https://github.com/apache/calcite/pull/741>) as precursor for CALCITE-2376 ?
> Former PR has also fixes javadoc issues for mongo adapter.
No.
I have 5 commits that I would like to merge but cannot because my builds fail. We need to fix the big problem before we do anything else.
> I'm working on CALCITE-2376 which unifies ES2 and ES5 adapters. I'll make
> sure it is passing both javadoc / site tasks in maven.
Sounds very cool, but I’m not an ES expert. I think you should get buy-in from other people who have worked on the ES adapter(s) before you go too far down that path.
Also, it is a big solution to a small (but urgent) problem. Let’s fix that problem with a quick, small fix.
> On a separate note, do you think it makes sense to add javadoc generation
> to travis build ?
I would love to, but IIRC, travis times out if you add javadoc generation.
Julian
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Andrei Sereda <an...@sereda.cc>.
I'm working on CALCITE-2376 which unifies ES2 and ES5 adapters. I'll make
sure it is passing both javadoc / site tasks in maven.
Would you consider merging this partial javadoc fix (PR 741
<https://github.com/apache/calcite/pull/741>) as precursor for CALCITE-2376 ?
Former PR has also fixes javadoc issues for mongo adapter.
On a separate note, do you think it makes sense to add javadoc generation
to travis build ?
On Mon, Jun 25, 2018 at 1:52 PM Julian Hyde <jh...@apache.org> wrote:
> Your changes seem to be generally beneficial and are much appreciated. But
> they do not fix the main problem, which is the missing package on JDK 9,
> 10, 11. (I believe it has always worked on JDK 8.) For example, here is
> output from the run against your branch on JDK 10 Ubuntu:
>
> [ERROR] Failed to execute goal
> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate
> report:
> [ERROR] Exit code: 1 -
> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> error: package org.elasticsearch.node.internal does not exist
> [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
> [ERROR] ^
> [ERROR]
>
> Let’s work on fixing the main problem first, then we’ll consider your PR.
>
> We had issues before (there is a JIRA case, don’t recall the number) where
> ES2 and ES5 could not share the same class path. I think this is another
> manifestation of that — maybe javadoc tries to generate for all modules
> simultaneously, and therefore when ES5’s dependencies have been loaded into
> javadoc, ES2’s dependencies cannot be seen.
>
> The solution may be to disable javadoc for the ES2 adapter. Or perhaps
> just for that one java file.
>
> Julian
>
>
> > On Jun 25, 2018, at 8:10 AM, Volodymyr Vysotskyi <vo...@apache.org>
> wrote:
> >
> > Thanks for the changes, I have checked and it passed on my env.
> >
> > I think it should be also checked with other JDK versions besides 8.
> >
> > Kind regards,
> > Volodymyr Vysotskyi
> >
> >
> > пн, 25 черв. 2018 о 07:05 Andrei Sereda <an...@sereda.cc> пише:
> >
> >> Can you please check the branch in my fork ?
> >>
> >> fork: https://github.com/asereda-gs/calcite
> >> branch: javadoc-fixes
> >>
> >> I've tried to remove most of the warnings and javadoc, javadoc-test and
> >> site all work for me.
> >>
> >> ~/I/c/elasticsearch5 ❯❯❯ mvn --version
> >> javadoc-fixes ✭
> >> Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe;
> >> 2018-06-17T14:33:14-04:00)
> >> Maven home: /usr/local/Cellar/maven/3.5.4/libexec
> >> Java version: 1.8.0_151, vendor: Oracle Corporation, runtime:
> >> /Library/Java/JavaVirtualMachines/jdk1.8.0_151.jdk/Contents/Home/jre
> >> Default locale: en_US, platform encoding: UTF-8
> >> OS name: "mac os x", version: "10.13.2", arch: "x86_64", family: "mac"
> >>
> >>
> >>
> >> On Sun, Jun 24, 2018 at 12:40 PM Volodymyr Vysotskyi <
> volodymyr@apache.org
> >>>
> >> wrote:
> >>
> >>> I have also checked "mvn javadoc:javadoc javadoc:test-javadoc" on my
> >>> machine (Ubuntu, JDK 8, Maven 3.3.9) and it fails on elasticsearch2
> >> module
> >>> (didn't check this problem for the next modules).
> >>>
> >>> "mvn clean -DskipTests site" fails for MongoDB module.
> >>>
> >>> Kind regards,
> >>> Volodymyr Vysotskyi
> >>>
> >>>
> >>> сб, 23 черв. 2018 о 03:12 Julian Hyde <jh...@apache.org> пише:
> >>>
> >>>> Same failure for me on JDK 8/macOS. The most likely other variable is
> >>>> maven: I have 3.5.2 on macOS, 3.5.3 on ubuntu.
> >>>>
> >>>> I’m mystified why no one else is seeing javadoc failures on macOS.
> >>>>
> >>>> Julian
> >>>>
> >>>>
> >>>>> On Jun 22, 2018, at 4:35 PM, Julian Hyde <jh...@apache.org> wrote:
> >>>>>
> >>>>> On JDK 8/ubuntu, “mvn clean -DskipTests site” gives the following
> >>>> failure:
> >>>>>
> >>>>> [ERROR] Failed to execute goal
> >>>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> >>>> project calcite-mongodb: Error generating
> >>>> maven-javadoc-plugin:3.0.1:test-javadoc report:
> >>>>> [ERROR] Exit code: 1 -
> >>>>
> >>>
> >>
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:22:
> >>>> error: cannot find symbol
> >>>>> [ERROR] import org.apache.calcite.test.CalciteAssert;
> >>>>> [ERROR] ^
> >>>>> [ERROR] symbol: class CalciteAssert
> >>>>> [ERROR] location: package org.apache.calcite.test
> >>>>> [ERROR]
> >>>>
> >>>
> >>
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:137:
> >>>> error: package CalciteAssert does not exist
> >>>>> [ERROR] private CalciteAssert.AssertThat assertModel(String model)
> >> {
> >>>>> [ERROR] ^
> >>>>> [ERROR]
> >>>>
> >>>
> >>
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:145:
> >>>> error: package CalciteAssert does not exist
> >>>>> [ERROR] private CalciteAssert.AssertThat assertModel(URL url) {
> >>>>> [ERROR] ^
> >>>>> [ERROR]
> >>>>
> >>>
> >>
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:43:
> >>>> error: reference not found
> >>>>> [ERROR] /** Similar to {@link CalciteAssert#checkResultUnordered},
> >>> but
> >>>> filters strings
> >>>>> [ERROR] ^
> >>>>>
> >>>>>
> >>>>>
> >>>>>> On Jun 22, 2018, at 3:46 PM, Andrei Sereda <an...@sereda.cc>
> >> wrote:
> >>>>>>
> >>>>>> Hmm. Both master and c12cb4b0de work for me. Perhaps the issue is
> >> JDK
> >>> 8
> >>>> vs
> >>>>>> 9 ?
> >>>>>>
> >>>>>> # JDK8 (oracle) / macOS
> >>>>>> $ mvn clean -DskipTests site
> >>>>>>
> >>>>>> On Fri, Jun 22, 2018 at 5:56 PM Julian Hyde <jh...@apache.org>
> >> wrote:
> >>>>>>
> >>>>>>> I just tried it again. The following fails for me (jdk9, ubuntu):
> >>>>>>>
> >>>>>>> $ git checkout c12cb4b0de
> >>>>>>> $ mvn clean -DskipTests site
> >>>>>>>
> >>>>>>> Constructing Javadoc information...
> >>>>>>> 1 error
> >>>>>>> [INFO]
> >>>>>>>
> >>>>
> >> ------------------------------------------------------------------------
> >>>>>>> [INFO] Reactor Summary:
> >>>>>>> [INFO]
> >>>>>>> [INFO] Calcite 1.17.0-SNAPSHOT ............................ FAILURE
> >>>> [02:29
> >>>>>>> min]
> >>>>>>> [INFO] Calcite Linq4j ..................................... SKIPPED
> >>>>>>> [INFO] Calcite Core ....................................... SKIPPED
> >>>>>>> [INFO] Calcite Cassandra .................................. SKIPPED
> >>>>>>> [INFO] Calcite Druid ...................................... SKIPPED
> >>>>>>> [INFO] Calcite Elasticsearch .............................. SKIPPED
> >>>>>>> [INFO] Calcite Elasticsearch5 ............................. SKIPPED
> >>>>>>> [INFO] Calcite Examples ................................... SKIPPED
> >>>>>>> [INFO] Calcite Example CSV ................................ SKIPPED
> >>>>>>> [INFO] Calcite Example Function ........................... SKIPPED
> >>>>>>> [INFO] Calcite File ....................................... SKIPPED
> >>>>>>> [INFO] Calcite Geode ...................................... SKIPPED
> >>>>>>> [INFO] Calcite MongoDB .................................... SKIPPED
> >>>>>>> [INFO] Calcite Pig ........................................ SKIPPED
> >>>>>>> [INFO] Calcite Piglet ..................................... SKIPPED
> >>>>>>> [INFO] Calcite Plus ....................................... SKIPPED
> >>>>>>> [INFO] Calcite Server ..................................... SKIPPED
> >>>>>>> [INFO] Calcite Spark ...................................... SKIPPED
> >>>>>>> [INFO] Calcite Splunk ..................................... SKIPPED
> >>>>>>> [INFO] Calcite Ubenchmark 1.17.0-SNAPSHOT ................. SKIPPED
> >>>>>>> [INFO]
> >>>>>>>
> >>>>
> >> ------------------------------------------------------------------------
> >>>>>>> [INFO] BUILD FAILURE
> >>>>>>> [INFO]
> >>>>>>>
> >>>>
> >> ------------------------------------------------------------------------
> >>>>>>> [INFO] Total time: 02:29 min
> >>>>>>> [INFO] Finished at: 2018-06-22T14:53:13-07:00
> >>>>>>> [INFO]
> >>>>>>>
> >>>>
> >> ------------------------------------------------------------------------
> >>>>>>> [ERROR] Failed to execute goal
> >>>>>>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site)
> >> on
> >>>>>>> project calcite: Error generating
> >>>> maven-javadoc-plugin:3.0.1:test-aggregate
> >>>>>>> report:
> >>>>>>> [ERROR] Exit code: 1 -
> >>>>>>>
> >>>>
> >>>
> >>
> /home/jhyde/open1/calcite.4/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> >>>>>>> error: package org.elasticsearch.node.internal does not exist
> >>>>>>> [ERROR] import
> >>>> org.elasticsearch.node.internal.InternalSettingsPreparer;
> >>>>>>> [ERROR] ^
> >>>>>>> [ERROR]
> >>>>>>> [ERROR] Command line was: /usr/lib/jvm/jdk9/bin/javadoc @options
> >>>> @packages
> >>>>>>> @argfile
> >>>>>>> [ERROR]
> >>>>>>> [ERROR] Refer to the generated Javadoc files in
> >>>>>>> '/home/jhyde/open1/calcite.4/target/site/testapidocs' dir.
> >>>>>>> [ERROR] -> [Help 1]
> >>>>>>> [ERROR]
> >>>>>>> [ERROR] To see the full stack trace of the errors, re-run Maven
> >> with
> >>>> the
> >>>>>>> -e switch.
> >>>>>>> [ERROR] Re-run Maven using the -X switch to enable full debug
> >>> logging.
> >>>>>>> [ERROR]
> >>>>>>> [ERROR] For more information about the errors and possible
> >> solutions,
> >>>>>>> please read the following articles:
> >>>>>>> [ERROR] [Help 1]
> >>>>>>>
> >>>>
> >> http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException
> >>>>>>>
> >>>>>>>
> >>>>>>>> On Jun 22, 2018, at 2:41 PM, Michael Mior <mm...@apache.org>
> >> wrote:
> >>>>>>>>
> >>>>>>>> Odd, mvn clean site still works fine for me.
> >>>>>>>> --
> >>>>>>>> Michael Mior
> >>>>>>>> mmior@apache.org
> >>>>>>>>
> >>>>>>>>
> >>>>>>>>
> >>>>>>>> Le ven. 22 juin 2018 à 17:12, Julian Hyde <jh...@apache.org> a
> >>> écrit
> >>>> :
> >>>>>>>>
> >>>>>>>>> Looks like this change broke “mvn site” (perhaps also “mvn
> >>>>>>>>> javadoc:test-javadoc”).
> >>>>>>>>>
> >>>>>>>>> [ERROR] Failed to execute goal
> >>>>>>>>> org.apache.maven.plugins:maven-site-plugin:3.7:site
> >> (default-site)
> >>> on
> >>>>>>>>> project calcite: Error generating
> >>>>>>> maven-javadoc-plugin:3.0.1:test-aggregate
> >>>>>>>>> report:
> >>>>>>>>> [ERROR] Exit code: 1 -
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> >>>>>>>>> error: package org.elasticsearch.node.internal does not exist
> >>>>>>>>> [ERROR] import
> >>>> org.elasticsearch.node.internal.InternalSettingsPreparer;
> >>>>>>>>> [ERROR] ^
> >>>>>>>>> [ERROR]
> >>>>>>>>>
> >>>>>>>>>> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
> >>>>>>>>>>
> >>>>>>>>>> [CALCITE-2347] running ElasticSearch in embedded mode for unit
> >>>> tests of
> >>>>>>>>> ES adapter (Andrei Sereda)
> >>>>>>>>>>
> >>>>>>>>>> After discussion on dev-list Integration tests (for ES) have
> >> been
> >>>>>>>>> removed. They're now
> >>>>>>>>>> superseded by unit tests (which execute queries against a real
> >>>> elastic
> >>>>>>>>> instance)
> >>>>>>>>>>
> >>>>>>>>>> Added local file (zips-mini.json) which contains a small subset
> >> of
> >>>>>>>>> original zips.json
> >>>>>>>>>> (allows to bootstrap tests faster)
> >>>>>>>>>>
> >>>>>>>>>> Created separate ES JUnit rule which can be re-used across
> >>> different
> >>>>>>>>> tests.
> >>>>>>>>>>
> >>>>>>>>>> Both v2 and v5 of ES adapters are supported.
> >>>>>>>>>>
> >>>>>>>>>> Close apache/calcite#716
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
> >>>>>>>>>> Commit:
> >>>> http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
> >>>>>>>>>> Tree:
> >>> http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
> >>>>>>>>>> Diff:
> >>> http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
> >>>>>>>>>>
> >>>>>>>>>> Branch: refs/heads/master
> >>>>>>>>>> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
> >>>>>>>>>> Parents: 37944bb
> >>>>>>>>>> Author: Andrei Sereda <an...@nospam.com>
> >>>>>>>>>> Authored: Thu May 31 18:19:10 2018 -0400
> >>>>>>>>>> Committer: Michael Mior <mm...@uwaterloo.ca>
> >>>>>>>>>> Committed: Thu Jun 21 06:38:50 2018 -0400
> >>>>>>>>>>
> >>>>>>>>>>
> >>>> ----------------------------------------------------------------------
> >>>>>>>>>> .../AbstractElasticsearchTable.java | 12 +
> >>>>>>>>>> .../elasticsearch/ElasticsearchProject.java | 61 ++-
> >>>>>>>>>> elasticsearch2/pom.xml | 6 +
> >>>>>>>>>> .../Elasticsearch2Enumerator.java | 12 +-
> >>>>>>>>>> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
> >>>>>>>>>> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
> >>>>>>>>>> .../ElasticSearch2AdapterTest.java | 395
> >>>>>>> ++++++++++++++++++
> >>>>>>>>>> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
> >>>>>>>>>> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
> >>>>>>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> >>>>>>>>>> .../calcite/test/Elasticsearch2AdapterIT.java | 270
> >>> -------------
> >>>>>>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
> >>>>>>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
> >>>>>>>>>> elasticsearch5/pom.xml | 31 ++
> >>>>>>>>>> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
> >>>>>>>>>> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
> >>>>>>>>>> .../ElasticSearch5AdapterTest.java | 399
> >>>>>>> +++++++++++++++++++
> >>>>>>>>>> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
> >>>>>>>>>> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
> >>>>>>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> >>>>>>>>>> .../calcite/test/Elasticsearch5AdapterIT.java | 270
> >>> -------------
> >>>>>>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
> >>>>>>>>>> elasticsearch5/src/test/resources/log4j2.xml | 16 +
> >>>>>>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
> >>>>>>>>>> pom.xml | 20 +-
> >>>>>>>>>> 25 files changed, 1866 insertions(+), 670 deletions(-)
> >>>>>>>>>>
> >>>> ----------------------------------------------------------------------
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>>>>>>>>>
> >>>> ----------------------------------------------------------------------
> >>>>>>>>>> diff --git
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>>>>>>>>> index 0980469..8cc5933 100644
> >>>>>>>>>> ---
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>>>>>>>>> +++
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>>>>>>>>> @@ -75,6 +75,18 @@ public abstract class
> >>> AbstractElasticsearchTable
> >>>>>>>>> extends AbstractQueryableTable
> >>>>>>>>>> relOptTable, this, null);
> >>>>>>>>>> }
> >>>>>>>>>>
> >>>>>>>>>> + /**
> >>>>>>>>>> + * In ES 5.x scripted fields start with {@code
> >>>> params._source.foo}
> >>>>>>>>> while in ES2.x
> >>>>>>>>>> + * {@code _source.foo}. Helper method to build correct query
> >>>> based
> >>>>>>> on
> >>>>>>>>> runtime version of elastic.
> >>>>>>>>>> + *
> >>>>>>>>>> + * @see <a href="
> >>>>>>>>> https://github.com/elastic/elasticsearch/issues/20068">_source
> >>>>>>>>> variable</a>
> >>>>>>>>>> + * @see <a href="
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html
> >>>>>>> ">Scripted
> >>>>>>>>> Fields</a>
> >>>>>>>>>> + */
> >>>>>>>>>> + protected String scriptedFieldPrefix() {
> >>>>>>>>>> + // this is default pattern starting 5.x
> >>>>>>>>>> + return "params._source";
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> /** Executes a "find" operation on the underlying type.
> >>>>>>>>>> *
> >>>>>>>>>> * <p>For example,
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>>>>>>>>>
> >>>> ----------------------------------------------------------------------
> >>>>>>>>>> diff --git
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>>>>>>>>> index b42abd7..961c8b0 100644
> >>>>>>>>>> ---
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>>>>>>>>> +++
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>>>>>>>>> @@ -27,11 +27,15 @@ import
> >>>>>>>>> org.apache.calcite.rel.metadata.RelMetadataQuery;
> >>>>>>>>>> import org.apache.calcite.rel.type.RelDataType;
> >>>>>>>>>> import org.apache.calcite.rex.RexNode;
> >>>>>>>>>> import org.apache.calcite.util.Pair;
> >>>>>>>>>> -import org.apache.calcite.util.Util;
> >>>>>>>>>> +
> >>>>>>>>>> +import com.google.common.base.Function;
> >>>>>>>>>> +import com.google.common.collect.Lists;
> >>>>>>>>>>
> >>>>>>>>>> import java.util.ArrayList;
> >>>>>>>>>> import java.util.List;
> >>>>>>>>>>
> >>>>>>>>>> +import javax.annotation.Nullable;
> >>>>>>>>>> +
> >>>>>>>>>> /**
> >>>>>>>>>> * Implementation of {@link org.apache.calcite.rel.core.Project}
> >>>>>>>>>> * relational expression in Elasticsearch.
> >>>>>>>>>> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends
> >>>> Project
> >>>>>>>>> implements ElasticsearchRel {
> >>>>>>>>>> implementor.visitChild(0, getInput());
> >>>>>>>>>>
> >>>>>>>>>> final List<String> inFields =
> >>>>>>>>>> -
> >>>>>>>>>
> >>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> >>>>>>>>>> +
> >>>>>>>>>
> >>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> >>>>>>>>>> final ElasticsearchRules.RexToElasticsearchTranslator
> >> translator
> >>> =
> >>>>>>>>>> - new ElasticsearchRules.RexToElasticsearchTranslator(
> >>>>>>>>>> - (JavaTypeFactory) getCluster().getTypeFactory(),
> >>>>>>> inFields);
> >>>>>>>>>> + new
> >> ElasticsearchRules.RexToElasticsearchTranslator(
> >>>>>>>>>> + (JavaTypeFactory)
> >>>> getCluster().getTypeFactory(),
> >>>>>>>>> inFields);
> >>>>>>>>>>
> >>>>>>>>>> - final List<String> findItems = new ArrayList<>();
> >>>>>>>>>> - final List<String> scriptFieldItems = new ArrayList<>();
> >>>>>>>>>> + final List<String> fields = new ArrayList<>();
> >>>>>>>>>> + final List<String> scriptFields = new ArrayList<>();
> >>>>>>>>>> for (Pair<RexNode, String> pair: getNamedProjects()) {
> >>>>>>>>>> final String name = pair.right;
> >>>>>>>>>> final String expr = pair.left.accept(translator);
> >>>>>>>>>>
> >>>>>>>>>> if (expr.equals("\"" + name + "\"")) {
> >>>>>>>>>> - findItems.add(ElasticsearchRules.quote(name));
> >>>>>>>>>> + fields.add(name);
> >>>>>>>>>> } else if (expr.matches("\"literal\":.+")) {
> >>>>>>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> >>>>>>>>>> - + ":{\"script\": "
> >>>>>>>>>> - + expr.split(":")[1] + "}");
> >>>>>>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
> >>>>>>>>>> + + ":{\"script\": "
> >>>>>>>>>> + + expr.split(":")[1] + "}");
> >>>>>>>>>> } else {
> >>>>>>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> >>>>>>>>>> - + ":{\"script\":\"params._source."
> >>>>>>>>>> - + expr.replaceAll("\"", "") + "\"}");
> >>>>>>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
> >>>>>>>>>> + + ":{\"script\":"
> >>>>>>>>>> + // _source (ES2) vs params._source (ES5)
> >>>>>>>>>> + + "\"" +
> >>>>>>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> >>>>>>>>>> + + expr.replaceAll("\"", "") + "\"}");
> >>>>>>>>>> + }
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + StringBuilder query = new StringBuilder();
> >>>>>>>>>> + if (scriptFields.isEmpty()) {
> >>>>>>>>>> + List<String> newList = Lists.transform(fields, new
> >>>>>>>>> Function<String, String>() {
> >>>>>>>>>> + @Nullable
> >>>>>>>>>> + @Override public String apply(@Nullable String input) {
> >>>>>>>>>> + return ElasticsearchRules.quote(input);
> >>>>>>>>>> + }
> >>>>>>>>>> + });
> >>>>>>>>>> +
> >>>>>>>>>> + final String findString = String.join(", ", newList);
> >>>>>>>>>> + query.append("\"_source\" :
> >>>> [").append(findString).append("]");
> >>>>>>>>>> + } else {
> >>>>>>>>>> + // if scripted fields are present, ES ignores _source
> >>>> attribute
> >>>>>>>>>> + for (String field: fields) {
> >>>>>>>>>> + scriptFields.add(ElasticsearchRules.quote(field) +
> >>>>>>>>> ":{\"script\": "
> >>>>>>>>>> + // _source (ES2) vs params._source (ES5)
> >>>>>>>>>> + + "\"" +
> >>>>>>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> >>>>>>>>>> + + field + "\"}");
> >>>>>>>>>> }
> >>>>>>>>>> + query.append("\"script_fields\": {" + String.join(", ",
> >>>>>>>>> scriptFields) + "}");
> >>>>>>>>>> }
> >>>>>>>>>> - final String findString = Util.toString(findItems, "", ",
> >> ",
> >>>> "");
> >>>>>>>>>> - final String scriptFieldString = "\"script_fields\": {"
> >>>>>>>>>> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
> >>>>>>>>>> - final String fieldString = "\"_source\" : [" + findString +
> >>> "]"
> >>>>>>>>>> - + ", " + scriptFieldString;
> >>>>>>>>>>
> >>>>>>>>>> for (String opfield : implementor.list) {
> >>>>>>>>>> if (opfield.startsWith("\"_source\"")) {
> >>>>>>>>>> implementor.list.remove(opfield);
> >>>>>>>>>> }
> >>>>>>>>>> }
> >>>>>>>>>> - implementor.add(fieldString);
> >>>>>>>>>> + implementor.add(query.toString());
> >>>>>>>>>> }
> >>>>>>>>>> }
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
> >>>>>>>>>>
> >>>> ----------------------------------------------------------------------
> >>>>>>>>>> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
> >>>>>>>>>> index f24622c..6fbee03 100644
> >>>>>>>>>> --- a/elasticsearch2/pom.xml
> >>>>>>>>>> +++ b/elasticsearch2/pom.xml
> >>>>>>>>>> @@ -73,6 +73,12 @@ limitations under the License.
> >>>>>>>>>> <version>${elasticsearch-java-driver.version}</version>
> >>>>>>>>>> </dependency>
> >>>>>>>>>> <dependency>
> >>>>>>>>>> + <!-- Lang groovy dependency is needed for testing with
> >>>> embedded
> >>>>>>>>> ES (scripted fields like loc[0]) -->
> >>>>>>>>>> + <groupId>org.elasticsearch.module</groupId>
> >>>>>>>>>> + <artifactId>lang-groovy</artifactId>
> >>>>>>>>>> + <scope>test</scope>
> >>>>>>>>>> + </dependency>
> >>>>>>>>>> + <dependency>
> >>>>>>>>>> <groupId>com.carrotsearch</groupId>
> >>>>>>>>>> <artifactId>hppc</artifactId>
> >>>>>>>>>> <version>${hppc.version}</version>
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>>>>>>>>>
> >>>> ----------------------------------------------------------------------
> >>>>>>>>>> diff --git
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>>>>>>>>> index 84370ab..c3d2ac0 100644
> >>>>>>>>>> ---
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>>>>>>>>> +++
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>>>>>>>>> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
> >>>>>>>>>> import java.util.Date;
> >>>>>>>>>> import java.util.Iterator;
> >>>>>>>>>> import java.util.List;
> >>>>>>>>>> +import java.util.Locale;
> >>>>>>>>>> import java.util.Map;
> >>>>>>>>>>
> >>>>>>>>>> /**
> >>>>>>>>>> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator
> >>>> implements
> >>>>>>>>> Enumerator<Object> {
> >>>>>>>>>> private static Function1<SearchHit, Object[]> listGetter(
> >>>>>>>>>> final List<Map.Entry<String, Class>> fields) {
> >>>>>>>>>> return new Function1<SearchHit, Object[]>() {
> >>>>>>>>>> - public Object[] apply(SearchHit searchHitFields) {
> >>>>>>>>>> + public Object[] apply(SearchHit hit) {
> >>>>>>>>>> Object[] objects = new Object[fields.size()];
> >>>>>>>>>> for (int i = 0; i < fields.size(); i++) {
> >>>>>>>>>> final Map.Entry<String, Class> field = fields.get(i);
> >>>>>>>>>> final String name = field.getKey();
> >>>>>>>>>> - if (searchHitFields.fields().isEmpty()) {
> >>>>>>>>>> - objects[i] =
> >>>>>>> convert(searchHitFields.getSource().get(name),
> >>>>>>>>> field.getValue());
> >>>>>>>>>> + if (hit.fields().isEmpty()) {
> >>>>>>>>>> + objects[i] = convert(hit.getSource().get(name),
> >>>>>>>>> field.getValue());
> >>>>>>>>>> + } else if (hit.fields().containsKey(name)) {
> >>>>>>>>>> + objects[i] = convert(hit.field(name).getValue(),
> >>>>>>>>> field.getValue());
> >>>>>>>>>> } else {
> >>>>>>>>>> - objects[i] =
> >>>>>>>>> convert(searchHitFields.field(name).getValue(),
> >> field.getValue());
> >>>>>>>>>> + throw new IllegalStateException(
> >>>>>>>>>> + String.format(Locale.getDefault(), "No
> >> result
> >>>> for
> >>>>>>>>> %s", field));
> >>>>>>>>>> }
> >>>>>>>>>> }
> >>>>>>>>>> return objects;
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>>>>>>>>>
> >>>> ----------------------------------------------------------------------
> >>>>>>>>>> diff --git
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>>>>>>>>> index 668402b..46e3fc5 100644
> >>>>>>>>>> ---
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>>>>>>>>> +++
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>>>>>>>>> @@ -22,6 +22,9 @@ import
> >>>> org.apache.calcite.schema.impl.AbstractSchema;
> >>>>>>>>>>
> >>>>>>>>>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
> >>>>>>>>>>
> >>>>>>>>>> +import com.google.common.annotations.VisibleForTesting;
> >>>>>>>>>> +import com.google.common.base.Preconditions;
> >>>>>>>>>> +
> >>>>>>>>>> import com.google.common.collect.ImmutableList;
> >>>>>>>>>> import com.google.common.collect.ImmutableMap;
> >>>>>>>>>>
> >>>>>>>>>> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
> >>>>>>>>> AbstractSchema
> >>>>>>>>>> }
> >>>>>>>>>> }
> >>>>>>>>>>
> >>>>>>>>>> + /**
> >>>>>>>>>> + * Allows schema to be instantiated from existing elastic
> >>> search
> >>>>>>>>> client.
> >>>>>>>>>> + * This constructor is used in tests.
> >>>>>>>>>> + */
> >>>>>>>>>> + @VisibleForTesting
> >>>>>>>>>> + Elasticsearch2Schema(Client client, String index) {
> >>>>>>>>>> + this.client = Preconditions.checkNotNull(client, "client");
> >>>>>>>>>> + this.index = Preconditions.checkNotNull(index, "index");
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> @Override protected Map<String, Table> getTableMap() {
> >>>>>>>>>> final ImmutableMap.Builder<String, Table> builder =
> >>>>>>>>> ImmutableMap.builder();
> >>>>>>>>>>
> >>>>>>>>>> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
> >>>>>>>>> AbstractSchema
> >>>>>>>>>>
> >>>>>>>>>> final List<DiscoveryNode> nodes =
> >>>>>>>>> ImmutableList.copyOf(transportClient.connectedNodes());
> >>>>>>>>>> if (nodes.isEmpty()) {
> >>>>>>>>>> - throw new RuntimeException("Cannot connect to any
> >>>> elasticsearch
> >>>>>>>>> nodes");
> >>>>>>>>>> + throw new IllegalStateException("Cannot connect to any
> >>>>>>>>> elasticsearch node: "
> >>>>>>>>>> + + transportNodes);
> >>>>>>>>>> }
> >>>>>>>>>>
> >>>>>>>>>> client = transportClient;
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>>>>>>>>>
> >>>> ----------------------------------------------------------------------
> >>>>>>>>>> diff --git
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>>>>>>>>> index 636aa5f..2928835 100644
> >>>>>>>>>> ---
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>>>>>>>>> +++
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>>>>>>>>> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
> >>>>>>>>> AbstractElasticsearchTable {
> >>>>>>>>>> this.client = client;
> >>>>>>>>>> }
> >>>>>>>>>>
> >>>>>>>>>> + /**
> >>>>>>>>>> + * ES version 2.x. To access document attributes ES2 uses
> >>> {@code
> >>>>>>>>> _source.foo} syntax.
> >>>>>>>>>> + */
> >>>>>>>>>> + @Override protected String scriptedFieldPrefix() {
> >>>>>>>>>> + return "_source";
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> @Override protected Enumerable<Object> find(String index,
> >>>> List<String>
> >>>>>>>>> ops,
> >>>>>>>>>> - List<Map.Entry<String, Class>> fields) {
> >>>>>>>>>> +
> >>>> List<Map.Entry<String,
> >>>>>>>>> Class>> fields) {
> >>>>>>>>>> final String dbName = index;
> >>>>>>>>>>
> >>>>>>>>>> final String queryString = "{" + Util.toString(ops, "", ", ",
> >>> "") +
> >>>>>>>>> "}";
> >>>>>>>>>>
> >>>>>>>>>>
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>>>>>>>>>
> >>>> ----------------------------------------------------------------------
> >>>>>>>>>> diff --git
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>>>>>>>>> new file mode 100644
> >>>>>>>>>> index 0000000..287e094
> >>>>>>>>>> --- /dev/null
> >>>>>>>>>> +++
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>>>>>>>>> @@ -0,0 +1,395 @@
> >>>>>>>>>> +/*
> >>>>>>>>>> + * Licensed to the Apache Software Foundation (ASF) under one
> >> or
> >>>> more
> >>>>>>>>>> + * contributor license agreements. See the NOTICE file
> >>> distributed
> >>>>>>> with
> >>>>>>>>>> + * this work for additional information regarding copyright
> >>>> ownership.
> >>>>>>>>>> + * The ASF licenses this file to you under the Apache License,
> >>>> Version
> >>>>>>>>> 2.0
> >>>>>>>>>> + * (the "License"); you may not use this file except in
> >>> compliance
> >>>>>>> with
> >>>>>>>>>> + * the License. You may obtain a copy of the License at
> >>>>>>>>>> + *
> >>>>>>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
> >>>>>>>>>> + *
> >>>>>>>>>> + * Unless required by applicable law or agreed to in writing,
> >>>> software
> >>>>>>>>>> + * distributed under the License is distributed on an "AS IS"
> >>>> BASIS,
> >>>>>>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
> >>> or
> >>>>>>>>> implied.
> >>>>>>>>>> + * See the License for the specific language governing
> >>> permissions
> >>>> and
> >>>>>>>>>> + * limitations under the License.
> >>>>>>>>>> + */
> >>>>>>>>>> +package org.apache.calcite.adapter.elasticsearch2;
> >>>>>>>>>> +
> >>>>>>>>>> +import org.apache.calcite.jdbc.CalciteConnection;
> >>>>>>>>>> +import org.apache.calcite.schema.SchemaPlus;
> >>>>>>>>>> +import org.apache.calcite.schema.impl.ViewTable;
> >>>>>>>>>> +import org.apache.calcite.schema.impl.ViewTableMacro;
> >>>>>>>>>> +import org.apache.calcite.test.CalciteAssert;
> >>>>>>>>>> +import org.apache.calcite.test.ElasticChecker;
> >>>>>>>>>> +
> >>>>>>>>>> +import com.google.common.io.LineProcessor;
> >>>>>>>>>> +import com.google.common.io.Resources;
> >>>>>>>>>> +
> >>>>>>>>>> +import org.elasticsearch.action.bulk.BulkItemResponse;
> >>>>>>>>>> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
> >>>>>>>>>> +import org.elasticsearch.action.bulk.BulkResponse;
> >>>>>>>>>> +import org.elasticsearch.common.xcontent.XContentBuilder;
> >>>>>>>>>> +import org.elasticsearch.common.xcontent.XContentFactory;
> >>>>>>>>>> +
> >>>>>>>>>> +import org.junit.BeforeClass;
> >>>>>>>>>> +import org.junit.ClassRule;
> >>>>>>>>>> +import org.junit.Test;
> >>>>>>>>>> +
> >>>>>>>>>> +import java.io.IOException;
> >>>>>>>>>> +import java.nio.charset.StandardCharsets;
> >>>>>>>>>> +import java.sql.Connection;
> >>>>>>>>>> +import java.sql.DriverManager;
> >>>>>>>>>> +import java.sql.SQLException;
> >>>>>>>>>> +import java.util.Arrays;
> >>>>>>>>>> +import java.util.Collections;
> >>>>>>>>>> +import java.util.Locale;
> >>>>>>>>>> +
> >>>>>>>>>> +/**
> >>>>>>>>>> + * Set of tests for ES adapter. Uses real instance via {@link
> >>>>>>>>> EmbeddedElasticRule}. Document
> >>>>>>>>>> + * source is local {@code zips-mini.json} file (located in the
> >>>>>>>>> classpath).
> >>>>>>>>>> + */
> >>>>>>>>>> +public class ElasticSearch2AdapterTest {
> >>>>>>>>>> +
> >>>>>>>>>> + @ClassRule //init once for all tests
> >>>>>>>>>> + public static final EmbeddedElasticRule NODE =
> >>>>>>>>> EmbeddedElasticRule.create();
> >>>>>>>>>> +
> >>>>>>>>>> + private static final String ZIPS = "zips";
> >>>>>>>>>> +
> >>>>>>>>>> + /**
> >>>>>>>>>> + * Used to create {@code zips} index and insert some data
> >>>>>>>>>> + */
> >>>>>>>>>> + @BeforeClass
> >>>>>>>>>> + public static void setupInstance() throws Exception {
> >>>>>>>>>> + // define mapping so fields are searchable (term query)
> >>>>>>>>>> + XContentBuilder mapping =
> >>>>>>>>> XContentFactory.jsonBuilder().startObject()
> >>>>>>>>>> + .startObject("properties")
> >>>>>>>>>> + .startObject("city").field("type", "string")
> >>>>>>>>>> + .field("index", "not_analyzed").endObject()
> >>>>>>>>>> + .startObject("state").field("type", "string")
> >>>>>>>>>> + .field("index", "not_analyzed").endObject()
> >>>>>>>>>> + .startObject("pop").field("type",
> >> "long").endObject()
> >>>>>>>>>> + .endObject()
> >>>>>>>>>> + .endObject();
> >>>>>>>>>> +
> >>>>>>>>>> + // create index
> >>>>>>>>>> + NODE.client().admin().indices()
> >>>>>>>>>> + .prepareCreate(ZIPS)
> >>>>>>>>>> + .addMapping(ZIPS, mapping)
> >>>>>>>>>> + .get();
> >>>>>>>>>> +
> >>>>>>>>>> + BulkRequestBuilder bulk =
> >>>>>>>>> NODE.client().prepareBulk().setRefresh(true);
> >>>>>>>>>> +
> >>>>>>>>>> + // load records from file
> >>>>>>>>>> +
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
> >>>>>>>>>> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
> >>>>>>>>>> + @Override public boolean processLine(String line)
> >>>> throws
> >>>>>>>>> IOException {
> >>>>>>>>>> + line = line.replaceAll("_id", "id"); // _id is
> >> a
> >>>>>>>>> reserved attribute in ES
> >>>>>>>>>> + bulk.add(NODE.client().prepareIndex(ZIPS,
> >>>>>>>>> ZIPS).setSource(line));
> >>>>>>>>>> + return true;
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + @Override public Void getResult() {
> >>>>>>>>>> + return null;
> >>>>>>>>>> + }
> >>>>>>>>>> + });
> >>>>>>>>>> +
> >>>>>>>>>> + if (bulk.numberOfActions() == 0) {
> >>>>>>>>>> + throw new IllegalStateException("No records to be
> >>> indexed");
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + BulkResponse response = bulk.execute().get();
> >>>>>>>>>> +
> >>>>>>>>>> + if (response.hasFailures()) {
> >>>>>>>>>> + throw new IllegalStateException(
> >>>>>>>>>> + String.format(Locale.getDefault(), "Failed to
> >>>> populate
> >>>>>>>>> %s:\n%s", NODE.httpAddress(),
> >>>>>>>>>> +
> >>>>>>>>>
> >>> Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
> >>>>>>>>>> +
> >>>>>>>>>
> >>>>>>>
> >>>>
> >>>
> >>
> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + private CalciteAssert.ConnectionFactory
> >> newConnectionFactory()
> >>> {
> >>>>>>>>>> + return new CalciteAssert.ConnectionFactory() {
> >>>>>>>>>> + @Override public Connection createConnection() throws
> >>>>>>>>> SQLException {
> >>>>>>>>>> + final Connection connection =
> >>>>>>>>> DriverManager.getConnection("jdbc:calcite:");
> >>>>>>>>>> + final SchemaPlus root =
> >>>>>>>>> connection.unwrap(CalciteConnection.class).getRootSchema();
> >>>>>>>>>> +
> >>>>>>>>>> + root.add("elastic", new
> >>> Elasticsearch2Schema(NODE.client(),
> >>>>>>>>> ZIPS));
> >>>>>>>>>> +
> >>>>>>>>>> + // add calcite view programmatically
> >>>>>>>>>> + final String viewSql = "select cast(_MAP['city'] AS
> >>>>>>>>> varchar(20)) AS \"city\", "
> >>>>>>>>>> + + " cast(_MAP['loc'][0] AS float) AS
> >>>> \"longitude\",\n"
> >>>>>>>>>> + + " cast(_MAP['loc'][1] AS float) AS
> >>>> \"latitude\",\n"
> >>>>>>>>>> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
> >>>>>>>>>> + + " cast(_MAP['state'] AS varchar(2)) AS
> >>> \"state\",
> >>>> "
> >>>>>>>>>> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
> >>>>>>>>>> + + "from \"elastic\".\"zips\"";
> >>>>>>>>>> +
> >>>>>>>>>> + ViewTableMacro macro = ViewTable.viewMacro(root,
> >> viewSql,
> >>>>>>>>>> + Collections.singletonList("elastic"),
> >>>>>>>>> Arrays.asList("elastic", "view"), false);
> >>>>>>>>>> + root.add("ZIPS", macro);
> >>>>>>>>>> +
> >>>>>>>>>> + return connection;
> >>>>>>>>>> + }
> >>>>>>>>>> + };
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + private CalciteAssert.AssertThat calciteAssert() {
> >>>>>>>>>> + return CalciteAssert.that()
> >>>>>>>>>> + .with(newConnectionFactory());
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + /**
> >>>>>>>>>> + * Tests using calcite view
> >>>>>>>>>> + */
> >>>>>>>>>> + @Test
> >>>>>>>>>> + public void view() throws Exception {
> >>>>>>>>>> + calciteAssert()
> >>>>>>>>>> + .query("select * from zips where \"city\" =
> >>> 'BROOKLYN'")
> >>>>>>>>>> + .returns("city=BROOKLYN; longitude=-73.956985;
> >>>>>>>>> latitude=40.646694; "
> >>>>>>>>>> + + "pop=111396; state=NY; id=11226\n")
> >>>>>>>>>> + .returnsCount(1);
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + @Test
> >>>>>>>>>> + public void emptyResult() {
> >>>>>>>>>> + CalciteAssert.that()
> >>>>>>>>>> + .with(newConnectionFactory())
> >>>>>>>>>> + .query("select * from zips limit 0")
> >>>>>>>>>> + .returnsCount(0);
> >>>>>>>>>> +
> >>>>>>>>>> + CalciteAssert.that()
> >>>>>>>>>> + .with(newConnectionFactory())
> >>>>>>>>>> + .query("select * from \"elastic\".\"zips\" where
> >>>>>>>>> _MAP['Foo'] = '_MISSING_'")
> >>>>>>>>>> + .returnsCount(0);
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + @Test
> >>>>>>>>>> + public void basic() throws Exception {
> >>>>>>>>>> + CalciteAssert.that()
> >>>>>>>>>> + .with(newConnectionFactory())
> >>>>>>>>>> + .query("select * from \"elastic\".\"zips\" where
> >>>>>>>>> _MAP['city'] = 'BROOKLYN'")
> >>>>>>>>>> + .returnsCount(1);
> >>>>>>>>>> +
> >>>>>>>>>> + CalciteAssert.that()
> >>>>>>>>>> + .with(newConnectionFactory())
> >>>>>>>>>> + .query("select * from \"elastic\".\"zips\" where"
> >>>>>>>>>> + + " _MAP['city'] in ('BROOKLYN',
> >>>> 'WASHINGTON')")
> >>>>>>>>>> + .returnsCount(2);
> >>>>>>>>>> +
> >>>>>>>>>> + // lower-case
> >>>>>>>>>> + CalciteAssert.that()
> >>>>>>>>>> + .with(newConnectionFactory())
> >>>>>>>>>> + .query("select * from \"elastic\".\"zips\" where "
> >>>>>>>>>> + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
> >>>>>>>>> 'BROOK') ")
> >>>>>>>>>> + .returnsCount(0);
> >>>>>>>>>> +
> >>>>>>>>>> + // missing field
> >>>>>>>>>> + CalciteAssert.that()
> >>>>>>>>>> + .with(newConnectionFactory())
> >>>>>>>>>> + .query("select * from \"elastic\".\"zips\" where
> >>>>>>>>> _MAP['CITY'] = 'BROOKLYN'")
> >>>>>>>>>> + .returnsCount(0);
> >>>>>>>>>> +
> >>>>>>>>>> + // limit works
> >>>>>>>>>> + CalciteAssert.that()
> >>>>>>>>>> + .with(newConnectionFactory())
> >>>>>>>>>> + .query("select * from \"elastic\".\"zips\" limit
> >> 42")
> >>>>>>>>>> + .returnsCount(42);
> >>>>>>>>>> +
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + @Test public void testSort() {
> >>>>>>>>>> + final String explain =
> >>>> "PLAN=ElasticsearchToEnumerableConverter\n"
> >>>>>>>>>> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> >>>>>>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> >>>>>>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>>>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0,
> >>> 'loc'),
> >>>>>>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> >>>>>>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> >>>>>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>>>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0,
> >> 'id')):VARCHAR(5)
> >>>>>>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>> \"ISO-8859-1$en_US$primary\"])\n"
> >>>>>>>>>> + + " ElasticsearchTableScan(table=[[elastic,
> >>>> zips]])";
> >>>>>>>>>> +
> >>>>>>>>>> + calciteAssert()
> >>>>>>>>>> + .query("select * from zips order by \"state\"")
> >>>>>>>>>> + .returnsCount(10)
> >>>>>>>>>> + .explainContains(explain);
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + @Test public void testSortLimit() {
> >>>>>>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> >>>>>>>>>> + + "order by \"state\", \"id\" offset 2 rows fetch
> >>> next
> >>>> 3
> >>>>>>>>> rows only";
> >>>>>>>>>> + calciteAssert()
> >>>>>>>>>> + .query(sql)
> >>>>>>>>>> + .returnsUnordered("state=AK; id=99801",
> >>>>>>>>>> + "state=AL; id=35215",
> >>>>>>>>>> + "state=AL; id=35401")
> >>>>>>>>>> + .queryContains(
> >>>>>>>>>> + ElasticChecker.elasticsearchChecker(
> >>>>>>>>>> + "\"_source\" : [\"state\",
> >> \"id\"]",
> >>>>>>>>>> + "\"sort\": [ {\"state\": \"asc\"},
> >>>>>>> {\"id\":
> >>>>>>>>> \"asc\"}]",
> >>>>>>>>>> + "\"from\": 2",
> >>>>>>>>>> + "\"size\": 3"));
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> +
> >>>>>>>>>> +
> >>>>>>>>>> + @Test public void testOffsetLimit() {
> >>>>>>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> >>>>>>>>>> + + "offset 2 fetch next 3 rows only";
> >>>>>>>>>> + calciteAssert()
> >>>>>>>>>> + .query(sql)
> >>>>>>>>>> + .runs()
> >>>>>>>>>> + .queryContains(
> >>>>>>>>>> + ElasticChecker.elasticsearchChecker(
> >>>>>>>>>> + "\"from\": 2",
> >>>>>>>>>> + "\"size\": 3",
> >>>>>>>>>> + "\"_source\" : [\"state\",
> >>> \"id\"]"));
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + @Test public void testLimit() {
> >>>>>>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> >>>>>>>>>> + + "fetch next 3 rows only";
> >>>>>>>>>> +
> >>>>>>>>>> + calciteAssert()
> >>>>>>>>>> + .query(sql)
> >>>>>>>>>> + .runs()
> >>>>>>>>>> + .queryContains(
> >>>>>>>>>> + ElasticChecker.elasticsearchChecker(
> >>>>>>>>>> + "\"size\": 3",
> >>>>>>>>>> + "\"_source\" : [\"state\",
> >>> \"id\"]"));
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + @Test public void testFilterSort() {
> >>>>>>>>>> + final String sql = "select * from zips\n"
> >>>>>>>>>> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
> >>>>>>>>>> + + "order by \"state\", \"id\"";
> >>>>>>>>>> + final String explain =
> >>>> "PLAN=ElasticsearchToEnumerableConverter\n"
> >>>>>>>>>> + + " ElasticsearchSort(sort0=[$4], sort1=[$5],
> >>>> dir0=[ASC],
> >>>>>>>>> dir1=[ASC])\n"
> >>>>>>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> >>>>>>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>>>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0,
> >>> 'loc'),
> >>>>>>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> >>>>>>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> >>>>>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>>>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0,
> >> 'id')):VARCHAR(5)
> >>>>>>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>> \"ISO-8859-1$en_US$primary\"])\n"
> >>>>>>>>>> + + "
> >>>>>>> ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
> >>>>>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>>>>>> \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0,
> >>>> 'id')):VARCHAR(5)
> >>>>>>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> >> \"ISO-8859-1$en_US$primary\",
> >>>>>>>>> '70000'))])\n"
> >>>>>>>>>> + + " ElasticsearchTableScan(table=[[elastic,
> >>>>>>> zips]])";
> >>>>>>>>>> + calciteAssert()
> >>>>>>>>>> + .query(sql)
> >>>>>>>>>> + .returnsOrdered("city=LOS ANGELES;
> >>>> longitude=-118.258189;
> >>>>>>>>> latitude=34.007856; "
> >>>>>>>>>> + + "pop=96074; state=CA; id=90011",
> >>>>>>>>>> + "city=BELL GARDENS; longitude=-118.17205;
> >>>>>>>>> latitude=33.969177; "
> >>>>>>>>>> + + "pop=99568; state=CA; id=90201",
> >>>>>>>>>> + "city=NORWALK; longitude=-118.081767;
> >>>>>>>>> latitude=33.90564; "
> >>>>>>>>>> + + "pop=94188; state=CA; id=90650")
> >>>>>>>>>> + .queryContains(
> >>>>>>>>>> +
> >>> ElasticChecker.elasticsearchChecker("\"query\"
> >>>> : "
> >>>>>>>>>> + +
> >>>>>>>>> "{\"constant_score\":{\"filter\":{\"bool\":"
> >>>>>>>>>> + +
> >>>>>>>>> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
> >>>>>>>>>> + +
> >>>>>>>>> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
> >>>>>>>>>> + "\"script_fields\":
> >>>>>>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> >>>>>>>>>> + +
> >>>>>>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> >>>>>>>>>> + + "\"city\":{\"script\":
> >>>>>>>>> \"_source.city\"}, "
> >>>>>>>>>> + + "\"pop\":{\"script\":
> >>>>>>>>> \"_source.pop\"}, "
> >>>>>>>>>> + + "\"state\":{\"script\":
> >>>>>>>>> \"_source.state\"}, "
> >>>>>>>>>> + + "\"id\":{\"script\": \"_
> >>>>>>> source.id
> >>>>>>>>> \"}}",
> >>>>>>>>>> + "\"sort\": [ {\"state\": \"asc\"},
> >>>>>>> {\"id\":
> >>>>>>>>> \"asc\"}]"))
> >>>>>>>>>> + .explainContains(explain);
> >>>>>>>>>> + }
> >>>>>>>>>> +
> >>>>>>>>>> + @Test public void testFilterSortDesc() {
> >>>>>>>>>> + final String sql = "select * from zips\n"
> >>>>>>>>>> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
> >>>>>>>>>> + + "order by \"state\" desc, \"pop\"";
> >>>>>>>>>> + calciteAssert()
> >>>>>>>>>> + .query(sql)
> >>>>>>>>>> + .limit(4)
> >>>>>>>>>> + .returnsOrdered(
> >>>>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Julian Hyde <jh...@apache.org>.
Your changes seem to be generally beneficial and are much appreciated. But they do not fix the main problem, which is the missing package on JDK 9, 10, 11. (I believe it has always worked on JDK 8.) For example, here is output from the run against your branch on JDK 10 Ubuntu:
[ERROR] Failed to execute goal org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate report:
[ERROR] Exit code: 1 - /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29: error: package org.elasticsearch.node.internal does not exist
[ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
[ERROR] ^
[ERROR]
Let’s work on fixing the main problem first, then we’ll consider your PR.
We had issues before (there is a JIRA case, don’t recall the number) where ES2 and ES5 could not share the same class path. I think this is another manifestation of that — maybe javadoc tries to generate for all modules simultaneously, and therefore when ES5’s dependencies have been loaded into javadoc, ES2’s dependencies cannot be seen.
The solution may be to disable javadoc for the ES2 adapter. Or perhaps just for that one java file.
Julian
> On Jun 25, 2018, at 8:10 AM, Volodymyr Vysotskyi <vo...@apache.org> wrote:
>
> Thanks for the changes, I have checked and it passed on my env.
>
> I think it should be also checked with other JDK versions besides 8.
>
> Kind regards,
> Volodymyr Vysotskyi
>
>
> пн, 25 черв. 2018 о 07:05 Andrei Sereda <an...@sereda.cc> пише:
>
>> Can you please check the branch in my fork ?
>>
>> fork: https://github.com/asereda-gs/calcite
>> branch: javadoc-fixes
>>
>> I've tried to remove most of the warnings and javadoc, javadoc-test and
>> site all work for me.
>>
>> ~/I/c/elasticsearch5 ❯❯❯ mvn --version
>> javadoc-fixes ✭
>> Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe;
>> 2018-06-17T14:33:14-04:00)
>> Maven home: /usr/local/Cellar/maven/3.5.4/libexec
>> Java version: 1.8.0_151, vendor: Oracle Corporation, runtime:
>> /Library/Java/JavaVirtualMachines/jdk1.8.0_151.jdk/Contents/Home/jre
>> Default locale: en_US, platform encoding: UTF-8
>> OS name: "mac os x", version: "10.13.2", arch: "x86_64", family: "mac"
>>
>>
>>
>> On Sun, Jun 24, 2018 at 12:40 PM Volodymyr Vysotskyi <volodymyr@apache.org
>>>
>> wrote:
>>
>>> I have also checked "mvn javadoc:javadoc javadoc:test-javadoc" on my
>>> machine (Ubuntu, JDK 8, Maven 3.3.9) and it fails on elasticsearch2
>> module
>>> (didn't check this problem for the next modules).
>>>
>>> "mvn clean -DskipTests site" fails for MongoDB module.
>>>
>>> Kind regards,
>>> Volodymyr Vysotskyi
>>>
>>>
>>> сб, 23 черв. 2018 о 03:12 Julian Hyde <jh...@apache.org> пише:
>>>
>>>> Same failure for me on JDK 8/macOS. The most likely other variable is
>>>> maven: I have 3.5.2 on macOS, 3.5.3 on ubuntu.
>>>>
>>>> I’m mystified why no one else is seeing javadoc failures on macOS.
>>>>
>>>> Julian
>>>>
>>>>
>>>>> On Jun 22, 2018, at 4:35 PM, Julian Hyde <jh...@apache.org> wrote:
>>>>>
>>>>> On JDK 8/ubuntu, “mvn clean -DskipTests site” gives the following
>>>> failure:
>>>>>
>>>>> [ERROR] Failed to execute goal
>>>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
>>>> project calcite-mongodb: Error generating
>>>> maven-javadoc-plugin:3.0.1:test-javadoc report:
>>>>> [ERROR] Exit code: 1 -
>>>>
>>>
>> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:22:
>>>> error: cannot find symbol
>>>>> [ERROR] import org.apache.calcite.test.CalciteAssert;
>>>>> [ERROR] ^
>>>>> [ERROR] symbol: class CalciteAssert
>>>>> [ERROR] location: package org.apache.calcite.test
>>>>> [ERROR]
>>>>
>>>
>> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:137:
>>>> error: package CalciteAssert does not exist
>>>>> [ERROR] private CalciteAssert.AssertThat assertModel(String model)
>> {
>>>>> [ERROR] ^
>>>>> [ERROR]
>>>>
>>>
>> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:145:
>>>> error: package CalciteAssert does not exist
>>>>> [ERROR] private CalciteAssert.AssertThat assertModel(URL url) {
>>>>> [ERROR] ^
>>>>> [ERROR]
>>>>
>>>
>> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:43:
>>>> error: reference not found
>>>>> [ERROR] /** Similar to {@link CalciteAssert#checkResultUnordered},
>>> but
>>>> filters strings
>>>>> [ERROR] ^
>>>>>
>>>>>
>>>>>
>>>>>> On Jun 22, 2018, at 3:46 PM, Andrei Sereda <an...@sereda.cc>
>> wrote:
>>>>>>
>>>>>> Hmm. Both master and c12cb4b0de work for me. Perhaps the issue is
>> JDK
>>> 8
>>>> vs
>>>>>> 9 ?
>>>>>>
>>>>>> # JDK8 (oracle) / macOS
>>>>>> $ mvn clean -DskipTests site
>>>>>>
>>>>>> On Fri, Jun 22, 2018 at 5:56 PM Julian Hyde <jh...@apache.org>
>> wrote:
>>>>>>
>>>>>>> I just tried it again. The following fails for me (jdk9, ubuntu):
>>>>>>>
>>>>>>> $ git checkout c12cb4b0de
>>>>>>> $ mvn clean -DskipTests site
>>>>>>>
>>>>>>> Constructing Javadoc information...
>>>>>>> 1 error
>>>>>>> [INFO]
>>>>>>>
>>>>
>> ------------------------------------------------------------------------
>>>>>>> [INFO] Reactor Summary:
>>>>>>> [INFO]
>>>>>>> [INFO] Calcite 1.17.0-SNAPSHOT ............................ FAILURE
>>>> [02:29
>>>>>>> min]
>>>>>>> [INFO] Calcite Linq4j ..................................... SKIPPED
>>>>>>> [INFO] Calcite Core ....................................... SKIPPED
>>>>>>> [INFO] Calcite Cassandra .................................. SKIPPED
>>>>>>> [INFO] Calcite Druid ...................................... SKIPPED
>>>>>>> [INFO] Calcite Elasticsearch .............................. SKIPPED
>>>>>>> [INFO] Calcite Elasticsearch5 ............................. SKIPPED
>>>>>>> [INFO] Calcite Examples ................................... SKIPPED
>>>>>>> [INFO] Calcite Example CSV ................................ SKIPPED
>>>>>>> [INFO] Calcite Example Function ........................... SKIPPED
>>>>>>> [INFO] Calcite File ....................................... SKIPPED
>>>>>>> [INFO] Calcite Geode ...................................... SKIPPED
>>>>>>> [INFO] Calcite MongoDB .................................... SKIPPED
>>>>>>> [INFO] Calcite Pig ........................................ SKIPPED
>>>>>>> [INFO] Calcite Piglet ..................................... SKIPPED
>>>>>>> [INFO] Calcite Plus ....................................... SKIPPED
>>>>>>> [INFO] Calcite Server ..................................... SKIPPED
>>>>>>> [INFO] Calcite Spark ...................................... SKIPPED
>>>>>>> [INFO] Calcite Splunk ..................................... SKIPPED
>>>>>>> [INFO] Calcite Ubenchmark 1.17.0-SNAPSHOT ................. SKIPPED
>>>>>>> [INFO]
>>>>>>>
>>>>
>> ------------------------------------------------------------------------
>>>>>>> [INFO] BUILD FAILURE
>>>>>>> [INFO]
>>>>>>>
>>>>
>> ------------------------------------------------------------------------
>>>>>>> [INFO] Total time: 02:29 min
>>>>>>> [INFO] Finished at: 2018-06-22T14:53:13-07:00
>>>>>>> [INFO]
>>>>>>>
>>>>
>> ------------------------------------------------------------------------
>>>>>>> [ERROR] Failed to execute goal
>>>>>>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site)
>> on
>>>>>>> project calcite: Error generating
>>>> maven-javadoc-plugin:3.0.1:test-aggregate
>>>>>>> report:
>>>>>>> [ERROR] Exit code: 1 -
>>>>>>>
>>>>
>>>
>> /home/jhyde/open1/calcite.4/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
>>>>>>> error: package org.elasticsearch.node.internal does not exist
>>>>>>> [ERROR] import
>>>> org.elasticsearch.node.internal.InternalSettingsPreparer;
>>>>>>> [ERROR] ^
>>>>>>> [ERROR]
>>>>>>> [ERROR] Command line was: /usr/lib/jvm/jdk9/bin/javadoc @options
>>>> @packages
>>>>>>> @argfile
>>>>>>> [ERROR]
>>>>>>> [ERROR] Refer to the generated Javadoc files in
>>>>>>> '/home/jhyde/open1/calcite.4/target/site/testapidocs' dir.
>>>>>>> [ERROR] -> [Help 1]
>>>>>>> [ERROR]
>>>>>>> [ERROR] To see the full stack trace of the errors, re-run Maven
>> with
>>>> the
>>>>>>> -e switch.
>>>>>>> [ERROR] Re-run Maven using the -X switch to enable full debug
>>> logging.
>>>>>>> [ERROR]
>>>>>>> [ERROR] For more information about the errors and possible
>> solutions,
>>>>>>> please read the following articles:
>>>>>>> [ERROR] [Help 1]
>>>>>>>
>>>>
>> http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException
>>>>>>>
>>>>>>>
>>>>>>>> On Jun 22, 2018, at 2:41 PM, Michael Mior <mm...@apache.org>
>> wrote:
>>>>>>>>
>>>>>>>> Odd, mvn clean site still works fine for me.
>>>>>>>> --
>>>>>>>> Michael Mior
>>>>>>>> mmior@apache.org
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> Le ven. 22 juin 2018 à 17:12, Julian Hyde <jh...@apache.org> a
>>> écrit
>>>> :
>>>>>>>>
>>>>>>>>> Looks like this change broke “mvn site” (perhaps also “mvn
>>>>>>>>> javadoc:test-javadoc”).
>>>>>>>>>
>>>>>>>>> [ERROR] Failed to execute goal
>>>>>>>>> org.apache.maven.plugins:maven-site-plugin:3.7:site
>> (default-site)
>>> on
>>>>>>>>> project calcite: Error generating
>>>>>>> maven-javadoc-plugin:3.0.1:test-aggregate
>>>>>>>>> report:
>>>>>>>>> [ERROR] Exit code: 1 -
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
>>>>>>>>> error: package org.elasticsearch.node.internal does not exist
>>>>>>>>> [ERROR] import
>>>> org.elasticsearch.node.internal.InternalSettingsPreparer;
>>>>>>>>> [ERROR] ^
>>>>>>>>> [ERROR]
>>>>>>>>>
>>>>>>>>>> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
>>>>>>>>>>
>>>>>>>>>> [CALCITE-2347] running ElasticSearch in embedded mode for unit
>>>> tests of
>>>>>>>>> ES adapter (Andrei Sereda)
>>>>>>>>>>
>>>>>>>>>> After discussion on dev-list Integration tests (for ES) have
>> been
>>>>>>>>> removed. They're now
>>>>>>>>>> superseded by unit tests (which execute queries against a real
>>>> elastic
>>>>>>>>> instance)
>>>>>>>>>>
>>>>>>>>>> Added local file (zips-mini.json) which contains a small subset
>> of
>>>>>>>>> original zips.json
>>>>>>>>>> (allows to bootstrap tests faster)
>>>>>>>>>>
>>>>>>>>>> Created separate ES JUnit rule which can be re-used across
>>> different
>>>>>>>>> tests.
>>>>>>>>>>
>>>>>>>>>> Both v2 and v5 of ES adapters are supported.
>>>>>>>>>>
>>>>>>>>>> Close apache/calcite#716
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
>>>>>>>>>> Commit:
>>>> http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
>>>>>>>>>> Tree:
>>> http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
>>>>>>>>>> Diff:
>>> http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
>>>>>>>>>>
>>>>>>>>>> Branch: refs/heads/master
>>>>>>>>>> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
>>>>>>>>>> Parents: 37944bb
>>>>>>>>>> Author: Andrei Sereda <an...@nospam.com>
>>>>>>>>>> Authored: Thu May 31 18:19:10 2018 -0400
>>>>>>>>>> Committer: Michael Mior <mm...@uwaterloo.ca>
>>>>>>>>>> Committed: Thu Jun 21 06:38:50 2018 -0400
>>>>>>>>>>
>>>>>>>>>>
>>>> ----------------------------------------------------------------------
>>>>>>>>>> .../AbstractElasticsearchTable.java | 12 +
>>>>>>>>>> .../elasticsearch/ElasticsearchProject.java | 61 ++-
>>>>>>>>>> elasticsearch2/pom.xml | 6 +
>>>>>>>>>> .../Elasticsearch2Enumerator.java | 12 +-
>>>>>>>>>> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
>>>>>>>>>> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
>>>>>>>>>> .../ElasticSearch2AdapterTest.java | 395
>>>>>>> ++++++++++++++++++
>>>>>>>>>> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
>>>>>>>>>> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
>>>>>>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
>>>>>>>>>> .../calcite/test/Elasticsearch2AdapterIT.java | 270
>>> -------------
>>>>>>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
>>>>>>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
>>>>>>>>>> elasticsearch5/pom.xml | 31 ++
>>>>>>>>>> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
>>>>>>>>>> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
>>>>>>>>>> .../ElasticSearch5AdapterTest.java | 399
>>>>>>> +++++++++++++++++++
>>>>>>>>>> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
>>>>>>>>>> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
>>>>>>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
>>>>>>>>>> .../calcite/test/Elasticsearch5AdapterIT.java | 270
>>> -------------
>>>>>>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
>>>>>>>>>> elasticsearch5/src/test/resources/log4j2.xml | 16 +
>>>>>>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
>>>>>>>>>> pom.xml | 20 +-
>>>>>>>>>> 25 files changed, 1866 insertions(+), 670 deletions(-)
>>>>>>>>>>
>>>> ----------------------------------------------------------------------
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>>>>>>>
>>>> ----------------------------------------------------------------------
>>>>>>>>>> diff --git
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>>>>>>> index 0980469..8cc5933 100644
>>>>>>>>>> ---
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>>>>>>> +++
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>>>>>>> @@ -75,6 +75,18 @@ public abstract class
>>> AbstractElasticsearchTable
>>>>>>>>> extends AbstractQueryableTable
>>>>>>>>>> relOptTable, this, null);
>>>>>>>>>> }
>>>>>>>>>>
>>>>>>>>>> + /**
>>>>>>>>>> + * In ES 5.x scripted fields start with {@code
>>>> params._source.foo}
>>>>>>>>> while in ES2.x
>>>>>>>>>> + * {@code _source.foo}. Helper method to build correct query
>>>> based
>>>>>>> on
>>>>>>>>> runtime version of elastic.
>>>>>>>>>> + *
>>>>>>>>>> + * @see <a href="
>>>>>>>>> https://github.com/elastic/elasticsearch/issues/20068">_source
>>>>>>>>> variable</a>
>>>>>>>>>> + * @see <a href="
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html
>>>>>>> ">Scripted
>>>>>>>>> Fields</a>
>>>>>>>>>> + */
>>>>>>>>>> + protected String scriptedFieldPrefix() {
>>>>>>>>>> + // this is default pattern starting 5.x
>>>>>>>>>> + return "params._source";
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> /** Executes a "find" operation on the underlying type.
>>>>>>>>>> *
>>>>>>>>>> * <p>For example,
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>>>>>>>
>>>> ----------------------------------------------------------------------
>>>>>>>>>> diff --git
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>>>>>>> index b42abd7..961c8b0 100644
>>>>>>>>>> ---
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>>>>>>> +++
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>>>>>>> @@ -27,11 +27,15 @@ import
>>>>>>>>> org.apache.calcite.rel.metadata.RelMetadataQuery;
>>>>>>>>>> import org.apache.calcite.rel.type.RelDataType;
>>>>>>>>>> import org.apache.calcite.rex.RexNode;
>>>>>>>>>> import org.apache.calcite.util.Pair;
>>>>>>>>>> -import org.apache.calcite.util.Util;
>>>>>>>>>> +
>>>>>>>>>> +import com.google.common.base.Function;
>>>>>>>>>> +import com.google.common.collect.Lists;
>>>>>>>>>>
>>>>>>>>>> import java.util.ArrayList;
>>>>>>>>>> import java.util.List;
>>>>>>>>>>
>>>>>>>>>> +import javax.annotation.Nullable;
>>>>>>>>>> +
>>>>>>>>>> /**
>>>>>>>>>> * Implementation of {@link org.apache.calcite.rel.core.Project}
>>>>>>>>>> * relational expression in Elasticsearch.
>>>>>>>>>> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends
>>>> Project
>>>>>>>>> implements ElasticsearchRel {
>>>>>>>>>> implementor.visitChild(0, getInput());
>>>>>>>>>>
>>>>>>>>>> final List<String> inFields =
>>>>>>>>>> -
>>>>>>>>>
>>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
>>>>>>>>>> +
>>>>>>>>>
>>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
>>>>>>>>>> final ElasticsearchRules.RexToElasticsearchTranslator
>> translator
>>> =
>>>>>>>>>> - new ElasticsearchRules.RexToElasticsearchTranslator(
>>>>>>>>>> - (JavaTypeFactory) getCluster().getTypeFactory(),
>>>>>>> inFields);
>>>>>>>>>> + new
>> ElasticsearchRules.RexToElasticsearchTranslator(
>>>>>>>>>> + (JavaTypeFactory)
>>>> getCluster().getTypeFactory(),
>>>>>>>>> inFields);
>>>>>>>>>>
>>>>>>>>>> - final List<String> findItems = new ArrayList<>();
>>>>>>>>>> - final List<String> scriptFieldItems = new ArrayList<>();
>>>>>>>>>> + final List<String> fields = new ArrayList<>();
>>>>>>>>>> + final List<String> scriptFields = new ArrayList<>();
>>>>>>>>>> for (Pair<RexNode, String> pair: getNamedProjects()) {
>>>>>>>>>> final String name = pair.right;
>>>>>>>>>> final String expr = pair.left.accept(translator);
>>>>>>>>>>
>>>>>>>>>> if (expr.equals("\"" + name + "\"")) {
>>>>>>>>>> - findItems.add(ElasticsearchRules.quote(name));
>>>>>>>>>> + fields.add(name);
>>>>>>>>>> } else if (expr.matches("\"literal\":.+")) {
>>>>>>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
>>>>>>>>>> - + ":{\"script\": "
>>>>>>>>>> - + expr.split(":")[1] + "}");
>>>>>>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
>>>>>>>>>> + + ":{\"script\": "
>>>>>>>>>> + + expr.split(":")[1] + "}");
>>>>>>>>>> } else {
>>>>>>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
>>>>>>>>>> - + ":{\"script\":\"params._source."
>>>>>>>>>> - + expr.replaceAll("\"", "") + "\"}");
>>>>>>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
>>>>>>>>>> + + ":{\"script\":"
>>>>>>>>>> + // _source (ES2) vs params._source (ES5)
>>>>>>>>>> + + "\"" +
>>>>>>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
>>>>>>>>>> + + expr.replaceAll("\"", "") + "\"}");
>>>>>>>>>> + }
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + StringBuilder query = new StringBuilder();
>>>>>>>>>> + if (scriptFields.isEmpty()) {
>>>>>>>>>> + List<String> newList = Lists.transform(fields, new
>>>>>>>>> Function<String, String>() {
>>>>>>>>>> + @Nullable
>>>>>>>>>> + @Override public String apply(@Nullable String input) {
>>>>>>>>>> + return ElasticsearchRules.quote(input);
>>>>>>>>>> + }
>>>>>>>>>> + });
>>>>>>>>>> +
>>>>>>>>>> + final String findString = String.join(", ", newList);
>>>>>>>>>> + query.append("\"_source\" :
>>>> [").append(findString).append("]");
>>>>>>>>>> + } else {
>>>>>>>>>> + // if scripted fields are present, ES ignores _source
>>>> attribute
>>>>>>>>>> + for (String field: fields) {
>>>>>>>>>> + scriptFields.add(ElasticsearchRules.quote(field) +
>>>>>>>>> ":{\"script\": "
>>>>>>>>>> + // _source (ES2) vs params._source (ES5)
>>>>>>>>>> + + "\"" +
>>>>>>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
>>>>>>>>>> + + field + "\"}");
>>>>>>>>>> }
>>>>>>>>>> + query.append("\"script_fields\": {" + String.join(", ",
>>>>>>>>> scriptFields) + "}");
>>>>>>>>>> }
>>>>>>>>>> - final String findString = Util.toString(findItems, "", ",
>> ",
>>>> "");
>>>>>>>>>> - final String scriptFieldString = "\"script_fields\": {"
>>>>>>>>>> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
>>>>>>>>>> - final String fieldString = "\"_source\" : [" + findString +
>>> "]"
>>>>>>>>>> - + ", " + scriptFieldString;
>>>>>>>>>>
>>>>>>>>>> for (String opfield : implementor.list) {
>>>>>>>>>> if (opfield.startsWith("\"_source\"")) {
>>>>>>>>>> implementor.list.remove(opfield);
>>>>>>>>>> }
>>>>>>>>>> }
>>>>>>>>>> - implementor.add(fieldString);
>>>>>>>>>> + implementor.add(query.toString());
>>>>>>>>>> }
>>>>>>>>>> }
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
>>>>>>>>>>
>>>> ----------------------------------------------------------------------
>>>>>>>>>> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
>>>>>>>>>> index f24622c..6fbee03 100644
>>>>>>>>>> --- a/elasticsearch2/pom.xml
>>>>>>>>>> +++ b/elasticsearch2/pom.xml
>>>>>>>>>> @@ -73,6 +73,12 @@ limitations under the License.
>>>>>>>>>> <version>${elasticsearch-java-driver.version}</version>
>>>>>>>>>> </dependency>
>>>>>>>>>> <dependency>
>>>>>>>>>> + <!-- Lang groovy dependency is needed for testing with
>>>> embedded
>>>>>>>>> ES (scripted fields like loc[0]) -->
>>>>>>>>>> + <groupId>org.elasticsearch.module</groupId>
>>>>>>>>>> + <artifactId>lang-groovy</artifactId>
>>>>>>>>>> + <scope>test</scope>
>>>>>>>>>> + </dependency>
>>>>>>>>>> + <dependency>
>>>>>>>>>> <groupId>com.carrotsearch</groupId>
>>>>>>>>>> <artifactId>hppc</artifactId>
>>>>>>>>>> <version>${hppc.version}</version>
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>>>>>>>
>>>> ----------------------------------------------------------------------
>>>>>>>>>> diff --git
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>>>>>>> index 84370ab..c3d2ac0 100644
>>>>>>>>>> ---
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>>>>>>> +++
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>>>>>>> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
>>>>>>>>>> import java.util.Date;
>>>>>>>>>> import java.util.Iterator;
>>>>>>>>>> import java.util.List;
>>>>>>>>>> +import java.util.Locale;
>>>>>>>>>> import java.util.Map;
>>>>>>>>>>
>>>>>>>>>> /**
>>>>>>>>>> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator
>>>> implements
>>>>>>>>> Enumerator<Object> {
>>>>>>>>>> private static Function1<SearchHit, Object[]> listGetter(
>>>>>>>>>> final List<Map.Entry<String, Class>> fields) {
>>>>>>>>>> return new Function1<SearchHit, Object[]>() {
>>>>>>>>>> - public Object[] apply(SearchHit searchHitFields) {
>>>>>>>>>> + public Object[] apply(SearchHit hit) {
>>>>>>>>>> Object[] objects = new Object[fields.size()];
>>>>>>>>>> for (int i = 0; i < fields.size(); i++) {
>>>>>>>>>> final Map.Entry<String, Class> field = fields.get(i);
>>>>>>>>>> final String name = field.getKey();
>>>>>>>>>> - if (searchHitFields.fields().isEmpty()) {
>>>>>>>>>> - objects[i] =
>>>>>>> convert(searchHitFields.getSource().get(name),
>>>>>>>>> field.getValue());
>>>>>>>>>> + if (hit.fields().isEmpty()) {
>>>>>>>>>> + objects[i] = convert(hit.getSource().get(name),
>>>>>>>>> field.getValue());
>>>>>>>>>> + } else if (hit.fields().containsKey(name)) {
>>>>>>>>>> + objects[i] = convert(hit.field(name).getValue(),
>>>>>>>>> field.getValue());
>>>>>>>>>> } else {
>>>>>>>>>> - objects[i] =
>>>>>>>>> convert(searchHitFields.field(name).getValue(),
>> field.getValue());
>>>>>>>>>> + throw new IllegalStateException(
>>>>>>>>>> + String.format(Locale.getDefault(), "No
>> result
>>>> for
>>>>>>>>> %s", field));
>>>>>>>>>> }
>>>>>>>>>> }
>>>>>>>>>> return objects;
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>>>>>>>
>>>> ----------------------------------------------------------------------
>>>>>>>>>> diff --git
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>>>>>>> index 668402b..46e3fc5 100644
>>>>>>>>>> ---
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>>>>>>> +++
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>>>>>>> @@ -22,6 +22,9 @@ import
>>>> org.apache.calcite.schema.impl.AbstractSchema;
>>>>>>>>>>
>>>>>>>>>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
>>>>>>>>>>
>>>>>>>>>> +import com.google.common.annotations.VisibleForTesting;
>>>>>>>>>> +import com.google.common.base.Preconditions;
>>>>>>>>>> +
>>>>>>>>>> import com.google.common.collect.ImmutableList;
>>>>>>>>>> import com.google.common.collect.ImmutableMap;
>>>>>>>>>>
>>>>>>>>>> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
>>>>>>>>> AbstractSchema
>>>>>>>>>> }
>>>>>>>>>> }
>>>>>>>>>>
>>>>>>>>>> + /**
>>>>>>>>>> + * Allows schema to be instantiated from existing elastic
>>> search
>>>>>>>>> client.
>>>>>>>>>> + * This constructor is used in tests.
>>>>>>>>>> + */
>>>>>>>>>> + @VisibleForTesting
>>>>>>>>>> + Elasticsearch2Schema(Client client, String index) {
>>>>>>>>>> + this.client = Preconditions.checkNotNull(client, "client");
>>>>>>>>>> + this.index = Preconditions.checkNotNull(index, "index");
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> @Override protected Map<String, Table> getTableMap() {
>>>>>>>>>> final ImmutableMap.Builder<String, Table> builder =
>>>>>>>>> ImmutableMap.builder();
>>>>>>>>>>
>>>>>>>>>> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
>>>>>>>>> AbstractSchema
>>>>>>>>>>
>>>>>>>>>> final List<DiscoveryNode> nodes =
>>>>>>>>> ImmutableList.copyOf(transportClient.connectedNodes());
>>>>>>>>>> if (nodes.isEmpty()) {
>>>>>>>>>> - throw new RuntimeException("Cannot connect to any
>>>> elasticsearch
>>>>>>>>> nodes");
>>>>>>>>>> + throw new IllegalStateException("Cannot connect to any
>>>>>>>>> elasticsearch node: "
>>>>>>>>>> + + transportNodes);
>>>>>>>>>> }
>>>>>>>>>>
>>>>>>>>>> client = transportClient;
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>>>>>>>
>>>> ----------------------------------------------------------------------
>>>>>>>>>> diff --git
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>>>>>>> index 636aa5f..2928835 100644
>>>>>>>>>> ---
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>>>>>>> +++
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>>>>>>> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
>>>>>>>>> AbstractElasticsearchTable {
>>>>>>>>>> this.client = client;
>>>>>>>>>> }
>>>>>>>>>>
>>>>>>>>>> + /**
>>>>>>>>>> + * ES version 2.x. To access document attributes ES2 uses
>>> {@code
>>>>>>>>> _source.foo} syntax.
>>>>>>>>>> + */
>>>>>>>>>> + @Override protected String scriptedFieldPrefix() {
>>>>>>>>>> + return "_source";
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> @Override protected Enumerable<Object> find(String index,
>>>> List<String>
>>>>>>>>> ops,
>>>>>>>>>> - List<Map.Entry<String, Class>> fields) {
>>>>>>>>>> +
>>>> List<Map.Entry<String,
>>>>>>>>> Class>> fields) {
>>>>>>>>>> final String dbName = index;
>>>>>>>>>>
>>>>>>>>>> final String queryString = "{" + Util.toString(ops, "", ", ",
>>> "") +
>>>>>>>>> "}";
>>>>>>>>>>
>>>>>>>>>>
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>>>>>>>
>>>> ----------------------------------------------------------------------
>>>>>>>>>> diff --git
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>>>>>>> new file mode 100644
>>>>>>>>>> index 0000000..287e094
>>>>>>>>>> --- /dev/null
>>>>>>>>>> +++
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>>>>>>> @@ -0,0 +1,395 @@
>>>>>>>>>> +/*
>>>>>>>>>> + * Licensed to the Apache Software Foundation (ASF) under one
>> or
>>>> more
>>>>>>>>>> + * contributor license agreements. See the NOTICE file
>>> distributed
>>>>>>> with
>>>>>>>>>> + * this work for additional information regarding copyright
>>>> ownership.
>>>>>>>>>> + * The ASF licenses this file to you under the Apache License,
>>>> Version
>>>>>>>>> 2.0
>>>>>>>>>> + * (the "License"); you may not use this file except in
>>> compliance
>>>>>>> with
>>>>>>>>>> + * the License. You may obtain a copy of the License at
>>>>>>>>>> + *
>>>>>>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>>>>>>>>> + *
>>>>>>>>>> + * Unless required by applicable law or agreed to in writing,
>>>> software
>>>>>>>>>> + * distributed under the License is distributed on an "AS IS"
>>>> BASIS,
>>>>>>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
>>> or
>>>>>>>>> implied.
>>>>>>>>>> + * See the License for the specific language governing
>>> permissions
>>>> and
>>>>>>>>>> + * limitations under the License.
>>>>>>>>>> + */
>>>>>>>>>> +package org.apache.calcite.adapter.elasticsearch2;
>>>>>>>>>> +
>>>>>>>>>> +import org.apache.calcite.jdbc.CalciteConnection;
>>>>>>>>>> +import org.apache.calcite.schema.SchemaPlus;
>>>>>>>>>> +import org.apache.calcite.schema.impl.ViewTable;
>>>>>>>>>> +import org.apache.calcite.schema.impl.ViewTableMacro;
>>>>>>>>>> +import org.apache.calcite.test.CalciteAssert;
>>>>>>>>>> +import org.apache.calcite.test.ElasticChecker;
>>>>>>>>>> +
>>>>>>>>>> +import com.google.common.io.LineProcessor;
>>>>>>>>>> +import com.google.common.io.Resources;
>>>>>>>>>> +
>>>>>>>>>> +import org.elasticsearch.action.bulk.BulkItemResponse;
>>>>>>>>>> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
>>>>>>>>>> +import org.elasticsearch.action.bulk.BulkResponse;
>>>>>>>>>> +import org.elasticsearch.common.xcontent.XContentBuilder;
>>>>>>>>>> +import org.elasticsearch.common.xcontent.XContentFactory;
>>>>>>>>>> +
>>>>>>>>>> +import org.junit.BeforeClass;
>>>>>>>>>> +import org.junit.ClassRule;
>>>>>>>>>> +import org.junit.Test;
>>>>>>>>>> +
>>>>>>>>>> +import java.io.IOException;
>>>>>>>>>> +import java.nio.charset.StandardCharsets;
>>>>>>>>>> +import java.sql.Connection;
>>>>>>>>>> +import java.sql.DriverManager;
>>>>>>>>>> +import java.sql.SQLException;
>>>>>>>>>> +import java.util.Arrays;
>>>>>>>>>> +import java.util.Collections;
>>>>>>>>>> +import java.util.Locale;
>>>>>>>>>> +
>>>>>>>>>> +/**
>>>>>>>>>> + * Set of tests for ES adapter. Uses real instance via {@link
>>>>>>>>> EmbeddedElasticRule}. Document
>>>>>>>>>> + * source is local {@code zips-mini.json} file (located in the
>>>>>>>>> classpath).
>>>>>>>>>> + */
>>>>>>>>>> +public class ElasticSearch2AdapterTest {
>>>>>>>>>> +
>>>>>>>>>> + @ClassRule //init once for all tests
>>>>>>>>>> + public static final EmbeddedElasticRule NODE =
>>>>>>>>> EmbeddedElasticRule.create();
>>>>>>>>>> +
>>>>>>>>>> + private static final String ZIPS = "zips";
>>>>>>>>>> +
>>>>>>>>>> + /**
>>>>>>>>>> + * Used to create {@code zips} index and insert some data
>>>>>>>>>> + */
>>>>>>>>>> + @BeforeClass
>>>>>>>>>> + public static void setupInstance() throws Exception {
>>>>>>>>>> + // define mapping so fields are searchable (term query)
>>>>>>>>>> + XContentBuilder mapping =
>>>>>>>>> XContentFactory.jsonBuilder().startObject()
>>>>>>>>>> + .startObject("properties")
>>>>>>>>>> + .startObject("city").field("type", "string")
>>>>>>>>>> + .field("index", "not_analyzed").endObject()
>>>>>>>>>> + .startObject("state").field("type", "string")
>>>>>>>>>> + .field("index", "not_analyzed").endObject()
>>>>>>>>>> + .startObject("pop").field("type",
>> "long").endObject()
>>>>>>>>>> + .endObject()
>>>>>>>>>> + .endObject();
>>>>>>>>>> +
>>>>>>>>>> + // create index
>>>>>>>>>> + NODE.client().admin().indices()
>>>>>>>>>> + .prepareCreate(ZIPS)
>>>>>>>>>> + .addMapping(ZIPS, mapping)
>>>>>>>>>> + .get();
>>>>>>>>>> +
>>>>>>>>>> + BulkRequestBuilder bulk =
>>>>>>>>> NODE.client().prepareBulk().setRefresh(true);
>>>>>>>>>> +
>>>>>>>>>> + // load records from file
>>>>>>>>>> +
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
>>>>>>>>>> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
>>>>>>>>>> + @Override public boolean processLine(String line)
>>>> throws
>>>>>>>>> IOException {
>>>>>>>>>> + line = line.replaceAll("_id", "id"); // _id is
>> a
>>>>>>>>> reserved attribute in ES
>>>>>>>>>> + bulk.add(NODE.client().prepareIndex(ZIPS,
>>>>>>>>> ZIPS).setSource(line));
>>>>>>>>>> + return true;
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Override public Void getResult() {
>>>>>>>>>> + return null;
>>>>>>>>>> + }
>>>>>>>>>> + });
>>>>>>>>>> +
>>>>>>>>>> + if (bulk.numberOfActions() == 0) {
>>>>>>>>>> + throw new IllegalStateException("No records to be
>>> indexed");
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + BulkResponse response = bulk.execute().get();
>>>>>>>>>> +
>>>>>>>>>> + if (response.hasFailures()) {
>>>>>>>>>> + throw new IllegalStateException(
>>>>>>>>>> + String.format(Locale.getDefault(), "Failed to
>>>> populate
>>>>>>>>> %s:\n%s", NODE.httpAddress(),
>>>>>>>>>> +
>>>>>>>>>
>>> Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
>>>>>>>>>> +
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + private CalciteAssert.ConnectionFactory
>> newConnectionFactory()
>>> {
>>>>>>>>>> + return new CalciteAssert.ConnectionFactory() {
>>>>>>>>>> + @Override public Connection createConnection() throws
>>>>>>>>> SQLException {
>>>>>>>>>> + final Connection connection =
>>>>>>>>> DriverManager.getConnection("jdbc:calcite:");
>>>>>>>>>> + final SchemaPlus root =
>>>>>>>>> connection.unwrap(CalciteConnection.class).getRootSchema();
>>>>>>>>>> +
>>>>>>>>>> + root.add("elastic", new
>>> Elasticsearch2Schema(NODE.client(),
>>>>>>>>> ZIPS));
>>>>>>>>>> +
>>>>>>>>>> + // add calcite view programmatically
>>>>>>>>>> + final String viewSql = "select cast(_MAP['city'] AS
>>>>>>>>> varchar(20)) AS \"city\", "
>>>>>>>>>> + + " cast(_MAP['loc'][0] AS float) AS
>>>> \"longitude\",\n"
>>>>>>>>>> + + " cast(_MAP['loc'][1] AS float) AS
>>>> \"latitude\",\n"
>>>>>>>>>> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
>>>>>>>>>> + + " cast(_MAP['state'] AS varchar(2)) AS
>>> \"state\",
>>>> "
>>>>>>>>>> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
>>>>>>>>>> + + "from \"elastic\".\"zips\"";
>>>>>>>>>> +
>>>>>>>>>> + ViewTableMacro macro = ViewTable.viewMacro(root,
>> viewSql,
>>>>>>>>>> + Collections.singletonList("elastic"),
>>>>>>>>> Arrays.asList("elastic", "view"), false);
>>>>>>>>>> + root.add("ZIPS", macro);
>>>>>>>>>> +
>>>>>>>>>> + return connection;
>>>>>>>>>> + }
>>>>>>>>>> + };
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + private CalciteAssert.AssertThat calciteAssert() {
>>>>>>>>>> + return CalciteAssert.that()
>>>>>>>>>> + .with(newConnectionFactory());
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + /**
>>>>>>>>>> + * Tests using calcite view
>>>>>>>>>> + */
>>>>>>>>>> + @Test
>>>>>>>>>> + public void view() throws Exception {
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query("select * from zips where \"city\" =
>>> 'BROOKLYN'")
>>>>>>>>>> + .returns("city=BROOKLYN; longitude=-73.956985;
>>>>>>>>> latitude=40.646694; "
>>>>>>>>>> + + "pop=111396; state=NY; id=11226\n")
>>>>>>>>>> + .returnsCount(1);
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test
>>>>>>>>>> + public void emptyResult() {
>>>>>>>>>> + CalciteAssert.that()
>>>>>>>>>> + .with(newConnectionFactory())
>>>>>>>>>> + .query("select * from zips limit 0")
>>>>>>>>>> + .returnsCount(0);
>>>>>>>>>> +
>>>>>>>>>> + CalciteAssert.that()
>>>>>>>>>> + .with(newConnectionFactory())
>>>>>>>>>> + .query("select * from \"elastic\".\"zips\" where
>>>>>>>>> _MAP['Foo'] = '_MISSING_'")
>>>>>>>>>> + .returnsCount(0);
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test
>>>>>>>>>> + public void basic() throws Exception {
>>>>>>>>>> + CalciteAssert.that()
>>>>>>>>>> + .with(newConnectionFactory())
>>>>>>>>>> + .query("select * from \"elastic\".\"zips\" where
>>>>>>>>> _MAP['city'] = 'BROOKLYN'")
>>>>>>>>>> + .returnsCount(1);
>>>>>>>>>> +
>>>>>>>>>> + CalciteAssert.that()
>>>>>>>>>> + .with(newConnectionFactory())
>>>>>>>>>> + .query("select * from \"elastic\".\"zips\" where"
>>>>>>>>>> + + " _MAP['city'] in ('BROOKLYN',
>>>> 'WASHINGTON')")
>>>>>>>>>> + .returnsCount(2);
>>>>>>>>>> +
>>>>>>>>>> + // lower-case
>>>>>>>>>> + CalciteAssert.that()
>>>>>>>>>> + .with(newConnectionFactory())
>>>>>>>>>> + .query("select * from \"elastic\".\"zips\" where "
>>>>>>>>>> + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
>>>>>>>>> 'BROOK') ")
>>>>>>>>>> + .returnsCount(0);
>>>>>>>>>> +
>>>>>>>>>> + // missing field
>>>>>>>>>> + CalciteAssert.that()
>>>>>>>>>> + .with(newConnectionFactory())
>>>>>>>>>> + .query("select * from \"elastic\".\"zips\" where
>>>>>>>>> _MAP['CITY'] = 'BROOKLYN'")
>>>>>>>>>> + .returnsCount(0);
>>>>>>>>>> +
>>>>>>>>>> + // limit works
>>>>>>>>>> + CalciteAssert.that()
>>>>>>>>>> + .with(newConnectionFactory())
>>>>>>>>>> + .query("select * from \"elastic\".\"zips\" limit
>> 42")
>>>>>>>>>> + .returnsCount(42);
>>>>>>>>>> +
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test public void testSort() {
>>>>>>>>>> + final String explain =
>>>> "PLAN=ElasticsearchToEnumerableConverter\n"
>>>>>>>>>> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
>>>>>>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
>>>>>>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>>>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0,
>>> 'loc'),
>>>>>>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
>>>>>>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
>>>>>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>>>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0,
>> 'id')):VARCHAR(5)
>>>>>>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
>>>> \"ISO-8859-1$en_US$primary\"])\n"
>>>>>>>>>> + + " ElasticsearchTableScan(table=[[elastic,
>>>> zips]])";
>>>>>>>>>> +
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query("select * from zips order by \"state\"")
>>>>>>>>>> + .returnsCount(10)
>>>>>>>>>> + .explainContains(explain);
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test public void testSortLimit() {
>>>>>>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>>>>>>>>> + + "order by \"state\", \"id\" offset 2 rows fetch
>>> next
>>>> 3
>>>>>>>>> rows only";
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query(sql)
>>>>>>>>>> + .returnsUnordered("state=AK; id=99801",
>>>>>>>>>> + "state=AL; id=35215",
>>>>>>>>>> + "state=AL; id=35401")
>>>>>>>>>> + .queryContains(
>>>>>>>>>> + ElasticChecker.elasticsearchChecker(
>>>>>>>>>> + "\"_source\" : [\"state\",
>> \"id\"]",
>>>>>>>>>> + "\"sort\": [ {\"state\": \"asc\"},
>>>>>>> {\"id\":
>>>>>>>>> \"asc\"}]",
>>>>>>>>>> + "\"from\": 2",
>>>>>>>>>> + "\"size\": 3"));
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> +
>>>>>>>>>> +
>>>>>>>>>> + @Test public void testOffsetLimit() {
>>>>>>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>>>>>>>>> + + "offset 2 fetch next 3 rows only";
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query(sql)
>>>>>>>>>> + .runs()
>>>>>>>>>> + .queryContains(
>>>>>>>>>> + ElasticChecker.elasticsearchChecker(
>>>>>>>>>> + "\"from\": 2",
>>>>>>>>>> + "\"size\": 3",
>>>>>>>>>> + "\"_source\" : [\"state\",
>>> \"id\"]"));
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test public void testLimit() {
>>>>>>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>>>>>>>>> + + "fetch next 3 rows only";
>>>>>>>>>> +
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query(sql)
>>>>>>>>>> + .runs()
>>>>>>>>>> + .queryContains(
>>>>>>>>>> + ElasticChecker.elasticsearchChecker(
>>>>>>>>>> + "\"size\": 3",
>>>>>>>>>> + "\"_source\" : [\"state\",
>>> \"id\"]"));
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test public void testFilterSort() {
>>>>>>>>>> + final String sql = "select * from zips\n"
>>>>>>>>>> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
>>>>>>>>>> + + "order by \"state\", \"id\"";
>>>>>>>>>> + final String explain =
>>>> "PLAN=ElasticsearchToEnumerableConverter\n"
>>>>>>>>>> + + " ElasticsearchSort(sort0=[$4], sort1=[$5],
>>>> dir0=[ASC],
>>>>>>>>> dir1=[ASC])\n"
>>>>>>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
>>>>>>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>>>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0,
>>> 'loc'),
>>>>>>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
>>>>>>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
>>>>>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>>>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0,
>> 'id')):VARCHAR(5)
>>>>>>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
>>>> \"ISO-8859-1$en_US$primary\"])\n"
>>>>>>>>>> + + "
>>>>>>> ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
>>>>>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>>>>>> \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0,
>>>> 'id')):VARCHAR(5)
>>>>>>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\",
>>>>>>>>> '70000'))])\n"
>>>>>>>>>> + + " ElasticsearchTableScan(table=[[elastic,
>>>>>>> zips]])";
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query(sql)
>>>>>>>>>> + .returnsOrdered("city=LOS ANGELES;
>>>> longitude=-118.258189;
>>>>>>>>> latitude=34.007856; "
>>>>>>>>>> + + "pop=96074; state=CA; id=90011",
>>>>>>>>>> + "city=BELL GARDENS; longitude=-118.17205;
>>>>>>>>> latitude=33.969177; "
>>>>>>>>>> + + "pop=99568; state=CA; id=90201",
>>>>>>>>>> + "city=NORWALK; longitude=-118.081767;
>>>>>>>>> latitude=33.90564; "
>>>>>>>>>> + + "pop=94188; state=CA; id=90650")
>>>>>>>>>> + .queryContains(
>>>>>>>>>> +
>>> ElasticChecker.elasticsearchChecker("\"query\"
>>>> : "
>>>>>>>>>> + +
>>>>>>>>> "{\"constant_score\":{\"filter\":{\"bool\":"
>>>>>>>>>> + +
>>>>>>>>> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
>>>>>>>>>> + +
>>>>>>>>> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
>>>>>>>>>> + "\"script_fields\":
>>>>>>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>>>>>>>>> + +
>>>>>>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>>>>>>>>> + + "\"city\":{\"script\":
>>>>>>>>> \"_source.city\"}, "
>>>>>>>>>> + + "\"pop\":{\"script\":
>>>>>>>>> \"_source.pop\"}, "
>>>>>>>>>> + + "\"state\":{\"script\":
>>>>>>>>> \"_source.state\"}, "
>>>>>>>>>> + + "\"id\":{\"script\": \"_
>>>>>>> source.id
>>>>>>>>> \"}}",
>>>>>>>>>> + "\"sort\": [ {\"state\": \"asc\"},
>>>>>>> {\"id\":
>>>>>>>>> \"asc\"}]"))
>>>>>>>>>> + .explainContains(explain);
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test public void testFilterSortDesc() {
>>>>>>>>>> + final String sql = "select * from zips\n"
>>>>>>>>>> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
>>>>>>>>>> + + "order by \"state\" desc, \"pop\"";
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query(sql)
>>>>>>>>>> + .limit(4)
>>>>>>>>>> + .returnsOrdered(
>>>>>>>>>> + "city=LOS ANGELES; longitude=-118.258189;
>>>>>>>>> latitude=34.007856; pop=96074; state=CA; id=90011",
>>>>>>>>>> + "city=BELL GARDENS; longitude=-118.17205;
>>>>>>>>> latitude=33.969177; pop=99568; state=CA; id=90201");
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test public void testFilterRedundant() {
>>>>>>>>>> + final String sql = "select * from zips\n"
>>>>>>>>>> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and
>>>>>>>>> \"state\" = 'OK'";
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query(sql)
>>>>>>>>>> + .runs()
>>>>>>>>>> + .queryContains(
>>>>>>>>>> + ElasticChecker.elasticsearchChecker(""
>>>>>>>>>> + + "\"query\" :
>>>>>>>>> {\"constant_score\":{\"filter\":{\"bool\":"
>>>>>>>>>> + +
>>>>>>>>> "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
>>>>>>>>>> + "\"script_fields\":
>>>>>>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>>>>>>>>> + +
>>>>>>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>>>>>>>>> + + "\"city\":{\"script\":
>>>>>>>>> \"_source.city\"}, "
>>>>>>>>>> + + "\"pop\":{\"script\":
>>>>>>>>> \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
>>>>>>>>>> + +
>>>> "\"id\":{\"script\":
>>>>>>>>> \"_source.id\"}}"
>>>>>>>>>> + ));
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test public void testInPlan() {
>>>>>>>>>> + final String[] searches = {
>>>>>>>>>> + "\"query\" :
>>>>>>>>> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
>>>>>>>>>> + +
>>>>>>>>>
>>>>>>>
>>>>
>>>
>> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
>>>>>>>>>> + + "{\"pop\":99568}}]}}]}}}}",
>>>>>>>>>> + "\"script_fields\":
>>>>>>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>>>>>>>>> + +
>>> "\"latitude\":{\"script\":\"_source.loc[1]\"},
>>>> "
>>>>>>>>>> + + "\"city\":{\"script\": \"_source.city\"}, "
>>>>>>>>>> + + "\"pop\":{\"script\": \"_source.pop\"}, "
>>>>>>>>>> + + "\"state\":{\"script\":
>> \"_source.state\"}, "
>>>>>>>>>> + + "\"id\":{\"script\": \"_source.id\"}}"
>>>>>>>>>> + };
>>>>>>>>>> +
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query("select * from zips where \"pop\" in (96074,
>>>>>>> 99568)")
>>>>>>>>>> + .returnsUnordered(
>>>>>>>>>> + "city=BELL GARDENS; longitude=-118.17205;
>>>>>>>>> latitude=33.969177; pop=99568; state=CA; id=90201",
>>>>>>>>>> + "city=LOS ANGELES; longitude=-118.258189;
>>>>>>>>> latitude=34.007856; pop=96074; state=CA; id=90011"
>>>>>>>>>> + )
>>>>>>>>>> +
>>>>>>>>> .queryContains(ElasticChecker.elasticsearchChecker(searches));
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test public void testZips() {
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query("select \"state\", \"city\" from zips")
>>>>>>>>>> + .returnsCount(10);
>>>>>>>>>> + }
>>>>>>>>>> +
>>>>>>>>>> + @Test public void testProject() {
>>>>>>>>>> + final String sql = "select \"state\", \"city\", 0 as
>>>> \"zero\"\n"
>>>>>>>>>> + + "from zips\n"
>>>>>>>>>> + + "order by \"state\", \"city\"";
>>>>>>>>>> +
>>>>>>>>>> + calciteAssert()
>>>>>>>>>> + .query(sql)
>>>>>>>>>> + .limit(2)
>>>>>>>>>> + .returnsUnordered("state=AK; city=ANCHORAGE;
>> zero=0",
>>>>>>>>>> + "state=AK; city=FAIRBANKS; zero=0")
>>>>>>>>>> + .queryContains(
>>>>>>>>>> +
>>>>>>>>> ElasticChecker.elasticsearchChecker("\"script_fields\": "
>>>>>>>>>> + + "{\"zero\":{\"script\":
>>>> \"0\"},
>>>>>>> "
>>>>>>>>>> + + "\"state\":{\"script\":
>>>>>>>>> \"_source.state\"}, "
>>>>>>>>>> + + "\"city\":{\"script\":
>>>>>>>>> \"_source.city\"}}",
>>>>>>>>>> + "\"sort
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Volodymyr Vysotskyi <vo...@apache.org>.
Thanks for the changes, I have checked and it passed on my env.
I think it should be also checked with other JDK versions besides 8.
Kind regards,
Volodymyr Vysotskyi
пн, 25 черв. 2018 о 07:05 Andrei Sereda <an...@sereda.cc> пише:
> Can you please check the branch in my fork ?
>
> fork: https://github.com/asereda-gs/calcite
> branch: javadoc-fixes
>
> I've tried to remove most of the warnings and javadoc, javadoc-test and
> site all work for me.
>
> ~/I/c/elasticsearch5 ❯❯❯ mvn --version
> javadoc-fixes ✭
> Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe;
> 2018-06-17T14:33:14-04:00)
> Maven home: /usr/local/Cellar/maven/3.5.4/libexec
> Java version: 1.8.0_151, vendor: Oracle Corporation, runtime:
> /Library/Java/JavaVirtualMachines/jdk1.8.0_151.jdk/Contents/Home/jre
> Default locale: en_US, platform encoding: UTF-8
> OS name: "mac os x", version: "10.13.2", arch: "x86_64", family: "mac"
>
>
>
> On Sun, Jun 24, 2018 at 12:40 PM Volodymyr Vysotskyi <volodymyr@apache.org
> >
> wrote:
>
> > I have also checked "mvn javadoc:javadoc javadoc:test-javadoc" on my
> > machine (Ubuntu, JDK 8, Maven 3.3.9) and it fails on elasticsearch2
> module
> > (didn't check this problem for the next modules).
> >
> > "mvn clean -DskipTests site" fails for MongoDB module.
> >
> > Kind regards,
> > Volodymyr Vysotskyi
> >
> >
> > сб, 23 черв. 2018 о 03:12 Julian Hyde <jh...@apache.org> пише:
> >
> > > Same failure for me on JDK 8/macOS. The most likely other variable is
> > > maven: I have 3.5.2 on macOS, 3.5.3 on ubuntu.
> > >
> > > I’m mystified why no one else is seeing javadoc failures on macOS.
> > >
> > > Julian
> > >
> > >
> > > > On Jun 22, 2018, at 4:35 PM, Julian Hyde <jh...@apache.org> wrote:
> > > >
> > > > On JDK 8/ubuntu, “mvn clean -DskipTests site” gives the following
> > > failure:
> > > >
> > > > [ERROR] Failed to execute goal
> > > org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> > > project calcite-mongodb: Error generating
> > > maven-javadoc-plugin:3.0.1:test-javadoc report:
> > > > [ERROR] Exit code: 1 -
> > >
> >
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:22:
> > > error: cannot find symbol
> > > > [ERROR] import org.apache.calcite.test.CalciteAssert;
> > > > [ERROR] ^
> > > > [ERROR] symbol: class CalciteAssert
> > > > [ERROR] location: package org.apache.calcite.test
> > > > [ERROR]
> > >
> >
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:137:
> > > error: package CalciteAssert does not exist
> > > > [ERROR] private CalciteAssert.AssertThat assertModel(String model)
> {
> > > > [ERROR] ^
> > > > [ERROR]
> > >
> >
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:145:
> > > error: package CalciteAssert does not exist
> > > > [ERROR] private CalciteAssert.AssertThat assertModel(URL url) {
> > > > [ERROR] ^
> > > > [ERROR]
> > >
> >
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:43:
> > > error: reference not found
> > > > [ERROR] /** Similar to {@link CalciteAssert#checkResultUnordered},
> > but
> > > filters strings
> > > > [ERROR] ^
> > > >
> > > >
> > > >
> > > >> On Jun 22, 2018, at 3:46 PM, Andrei Sereda <an...@sereda.cc>
> wrote:
> > > >>
> > > >> Hmm. Both master and c12cb4b0de work for me. Perhaps the issue is
> JDK
> > 8
> > > vs
> > > >> 9 ?
> > > >>
> > > >> # JDK8 (oracle) / macOS
> > > >> $ mvn clean -DskipTests site
> > > >>
> > > >> On Fri, Jun 22, 2018 at 5:56 PM Julian Hyde <jh...@apache.org>
> wrote:
> > > >>
> > > >>> I just tried it again. The following fails for me (jdk9, ubuntu):
> > > >>>
> > > >>> $ git checkout c12cb4b0de
> > > >>> $ mvn clean -DskipTests site
> > > >>>
> > > >>> Constructing Javadoc information...
> > > >>> 1 error
> > > >>> [INFO]
> > > >>>
> > >
> ------------------------------------------------------------------------
> > > >>> [INFO] Reactor Summary:
> > > >>> [INFO]
> > > >>> [INFO] Calcite 1.17.0-SNAPSHOT ............................ FAILURE
> > > [02:29
> > > >>> min]
> > > >>> [INFO] Calcite Linq4j ..................................... SKIPPED
> > > >>> [INFO] Calcite Core ....................................... SKIPPED
> > > >>> [INFO] Calcite Cassandra .................................. SKIPPED
> > > >>> [INFO] Calcite Druid ...................................... SKIPPED
> > > >>> [INFO] Calcite Elasticsearch .............................. SKIPPED
> > > >>> [INFO] Calcite Elasticsearch5 ............................. SKIPPED
> > > >>> [INFO] Calcite Examples ................................... SKIPPED
> > > >>> [INFO] Calcite Example CSV ................................ SKIPPED
> > > >>> [INFO] Calcite Example Function ........................... SKIPPED
> > > >>> [INFO] Calcite File ....................................... SKIPPED
> > > >>> [INFO] Calcite Geode ...................................... SKIPPED
> > > >>> [INFO] Calcite MongoDB .................................... SKIPPED
> > > >>> [INFO] Calcite Pig ........................................ SKIPPED
> > > >>> [INFO] Calcite Piglet ..................................... SKIPPED
> > > >>> [INFO] Calcite Plus ....................................... SKIPPED
> > > >>> [INFO] Calcite Server ..................................... SKIPPED
> > > >>> [INFO] Calcite Spark ...................................... SKIPPED
> > > >>> [INFO] Calcite Splunk ..................................... SKIPPED
> > > >>> [INFO] Calcite Ubenchmark 1.17.0-SNAPSHOT ................. SKIPPED
> > > >>> [INFO]
> > > >>>
> > >
> ------------------------------------------------------------------------
> > > >>> [INFO] BUILD FAILURE
> > > >>> [INFO]
> > > >>>
> > >
> ------------------------------------------------------------------------
> > > >>> [INFO] Total time: 02:29 min
> > > >>> [INFO] Finished at: 2018-06-22T14:53:13-07:00
> > > >>> [INFO]
> > > >>>
> > >
> ------------------------------------------------------------------------
> > > >>> [ERROR] Failed to execute goal
> > > >>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site)
> on
> > > >>> project calcite: Error generating
> > > maven-javadoc-plugin:3.0.1:test-aggregate
> > > >>> report:
> > > >>> [ERROR] Exit code: 1 -
> > > >>>
> > >
> >
> /home/jhyde/open1/calcite.4/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> > > >>> error: package org.elasticsearch.node.internal does not exist
> > > >>> [ERROR] import
> > > org.elasticsearch.node.internal.InternalSettingsPreparer;
> > > >>> [ERROR] ^
> > > >>> [ERROR]
> > > >>> [ERROR] Command line was: /usr/lib/jvm/jdk9/bin/javadoc @options
> > > @packages
> > > >>> @argfile
> > > >>> [ERROR]
> > > >>> [ERROR] Refer to the generated Javadoc files in
> > > >>> '/home/jhyde/open1/calcite.4/target/site/testapidocs' dir.
> > > >>> [ERROR] -> [Help 1]
> > > >>> [ERROR]
> > > >>> [ERROR] To see the full stack trace of the errors, re-run Maven
> with
> > > the
> > > >>> -e switch.
> > > >>> [ERROR] Re-run Maven using the -X switch to enable full debug
> > logging.
> > > >>> [ERROR]
> > > >>> [ERROR] For more information about the errors and possible
> solutions,
> > > >>> please read the following articles:
> > > >>> [ERROR] [Help 1]
> > > >>>
> > >
> http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException
> > > >>>
> > > >>>
> > > >>>> On Jun 22, 2018, at 2:41 PM, Michael Mior <mm...@apache.org>
> wrote:
> > > >>>>
> > > >>>> Odd, mvn clean site still works fine for me.
> > > >>>> --
> > > >>>> Michael Mior
> > > >>>> mmior@apache.org
> > > >>>>
> > > >>>>
> > > >>>>
> > > >>>> Le ven. 22 juin 2018 à 17:12, Julian Hyde <jh...@apache.org> a
> > écrit
> > > :
> > > >>>>
> > > >>>>> Looks like this change broke “mvn site” (perhaps also “mvn
> > > >>>>> javadoc:test-javadoc”).
> > > >>>>>
> > > >>>>> [ERROR] Failed to execute goal
> > > >>>>> org.apache.maven.plugins:maven-site-plugin:3.7:site
> (default-site)
> > on
> > > >>>>> project calcite: Error generating
> > > >>> maven-javadoc-plugin:3.0.1:test-aggregate
> > > >>>>> report:
> > > >>>>> [ERROR] Exit code: 1 -
> > > >>>>>
> > > >>>
> > >
> >
> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> > > >>>>> error: package org.elasticsearch.node.internal does not exist
> > > >>>>> [ERROR] import
> > > org.elasticsearch.node.internal.InternalSettingsPreparer;
> > > >>>>> [ERROR] ^
> > > >>>>> [ERROR]
> > > >>>>>
> > > >>>>>> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
> > > >>>>>>
> > > >>>>>> [CALCITE-2347] running ElasticSearch in embedded mode for unit
> > > tests of
> > > >>>>> ES adapter (Andrei Sereda)
> > > >>>>>>
> > > >>>>>> After discussion on dev-list Integration tests (for ES) have
> been
> > > >>>>> removed. They're now
> > > >>>>>> superseded by unit tests (which execute queries against a real
> > > elastic
> > > >>>>> instance)
> > > >>>>>>
> > > >>>>>> Added local file (zips-mini.json) which contains a small subset
> of
> > > >>>>> original zips.json
> > > >>>>>> (allows to bootstrap tests faster)
> > > >>>>>>
> > > >>>>>> Created separate ES JUnit rule which can be re-used across
> > different
> > > >>>>> tests.
> > > >>>>>>
> > > >>>>>> Both v2 and v5 of ES adapters are supported.
> > > >>>>>>
> > > >>>>>> Close apache/calcite#716
> > > >>>>>>
> > > >>>>>>
> > > >>>>>> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
> > > >>>>>> Commit:
> > > http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
> > > >>>>>> Tree:
> > http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
> > > >>>>>> Diff:
> > http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
> > > >>>>>>
> > > >>>>>> Branch: refs/heads/master
> > > >>>>>> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
> > > >>>>>> Parents: 37944bb
> > > >>>>>> Author: Andrei Sereda <an...@nospam.com>
> > > >>>>>> Authored: Thu May 31 18:19:10 2018 -0400
> > > >>>>>> Committer: Michael Mior <mm...@uwaterloo.ca>
> > > >>>>>> Committed: Thu Jun 21 06:38:50 2018 -0400
> > > >>>>>>
> > > >>>>>>
> > > ----------------------------------------------------------------------
> > > >>>>>> .../AbstractElasticsearchTable.java | 12 +
> > > >>>>>> .../elasticsearch/ElasticsearchProject.java | 61 ++-
> > > >>>>>> elasticsearch2/pom.xml | 6 +
> > > >>>>>> .../Elasticsearch2Enumerator.java | 12 +-
> > > >>>>>> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
> > > >>>>>> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
> > > >>>>>> .../ElasticSearch2AdapterTest.java | 395
> > > >>> ++++++++++++++++++
> > > >>>>>> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
> > > >>>>>> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
> > > >>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> > > >>>>>> .../calcite/test/Elasticsearch2AdapterIT.java | 270
> > -------------
> > > >>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
> > > >>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
> > > >>>>>> elasticsearch5/pom.xml | 31 ++
> > > >>>>>> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
> > > >>>>>> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
> > > >>>>>> .../ElasticSearch5AdapterTest.java | 399
> > > >>> +++++++++++++++++++
> > > >>>>>> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
> > > >>>>>> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
> > > >>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> > > >>>>>> .../calcite/test/Elasticsearch5AdapterIT.java | 270
> > -------------
> > > >>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
> > > >>>>>> elasticsearch5/src/test/resources/log4j2.xml | 16 +
> > > >>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
> > > >>>>>> pom.xml | 20 +-
> > > >>>>>> 25 files changed, 1866 insertions(+), 670 deletions(-)
> > > >>>>>>
> > > ----------------------------------------------------------------------
> > > >>>>>>
> > > >>>>>>
> > > >>>>>>
> > > >>>>>
> > > >>>
> > >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > > >>>>>>
> > > ----------------------------------------------------------------------
> > > >>>>>> diff --git
> > > >>>>>
> > > >>>
> > >
> >
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > > >>>>>
> > > >>>
> > >
> >
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > > >>>>>> index 0980469..8cc5933 100644
> > > >>>>>> ---
> > > >>>>>
> > > >>>
> > >
> >
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > > >>>>>> +++
> > > >>>>>
> > > >>>
> > >
> >
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > > >>>>>> @@ -75,6 +75,18 @@ public abstract class
> > AbstractElasticsearchTable
> > > >>>>> extends AbstractQueryableTable
> > > >>>>>> relOptTable, this, null);
> > > >>>>>> }
> > > >>>>>>
> > > >>>>>> + /**
> > > >>>>>> + * In ES 5.x scripted fields start with {@code
> > > params._source.foo}
> > > >>>>> while in ES2.x
> > > >>>>>> + * {@code _source.foo}. Helper method to build correct query
> > > based
> > > >>> on
> > > >>>>> runtime version of elastic.
> > > >>>>>> + *
> > > >>>>>> + * @see <a href="
> > > >>>>> https://github.com/elastic/elasticsearch/issues/20068">_source
> > > >>>>> variable</a>
> > > >>>>>> + * @see <a href="
> > > >>>>>
> > > >>>
> > >
> >
> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html
> > > >>> ">Scripted
> > > >>>>> Fields</a>
> > > >>>>>> + */
> > > >>>>>> + protected String scriptedFieldPrefix() {
> > > >>>>>> + // this is default pattern starting 5.x
> > > >>>>>> + return "params._source";
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> /** Executes a "find" operation on the underlying type.
> > > >>>>>> *
> > > >>>>>> * <p>For example,
> > > >>>>>>
> > > >>>>>>
> > > >>>>>
> > > >>>
> > >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > > >>>>>>
> > > ----------------------------------------------------------------------
> > > >>>>>> diff --git
> > > >>>>>
> > > >>>
> > >
> >
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > > >>>>>
> > > >>>
> > >
> >
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > > >>>>>> index b42abd7..961c8b0 100644
> > > >>>>>> ---
> > > >>>>>
> > > >>>
> > >
> >
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > > >>>>>> +++
> > > >>>>>
> > > >>>
> > >
> >
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > > >>>>>> @@ -27,11 +27,15 @@ import
> > > >>>>> org.apache.calcite.rel.metadata.RelMetadataQuery;
> > > >>>>>> import org.apache.calcite.rel.type.RelDataType;
> > > >>>>>> import org.apache.calcite.rex.RexNode;
> > > >>>>>> import org.apache.calcite.util.Pair;
> > > >>>>>> -import org.apache.calcite.util.Util;
> > > >>>>>> +
> > > >>>>>> +import com.google.common.base.Function;
> > > >>>>>> +import com.google.common.collect.Lists;
> > > >>>>>>
> > > >>>>>> import java.util.ArrayList;
> > > >>>>>> import java.util.List;
> > > >>>>>>
> > > >>>>>> +import javax.annotation.Nullable;
> > > >>>>>> +
> > > >>>>>> /**
> > > >>>>>> * Implementation of {@link org.apache.calcite.rel.core.Project}
> > > >>>>>> * relational expression in Elasticsearch.
> > > >>>>>> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends
> > > Project
> > > >>>>> implements ElasticsearchRel {
> > > >>>>>> implementor.visitChild(0, getInput());
> > > >>>>>>
> > > >>>>>> final List<String> inFields =
> > > >>>>>> -
> > > >>>>>
> > ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> > > >>>>>> +
> > > >>>>>
> > ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> > > >>>>>> final ElasticsearchRules.RexToElasticsearchTranslator
> translator
> > =
> > > >>>>>> - new ElasticsearchRules.RexToElasticsearchTranslator(
> > > >>>>>> - (JavaTypeFactory) getCluster().getTypeFactory(),
> > > >>> inFields);
> > > >>>>>> + new
> ElasticsearchRules.RexToElasticsearchTranslator(
> > > >>>>>> + (JavaTypeFactory)
> > > getCluster().getTypeFactory(),
> > > >>>>> inFields);
> > > >>>>>>
> > > >>>>>> - final List<String> findItems = new ArrayList<>();
> > > >>>>>> - final List<String> scriptFieldItems = new ArrayList<>();
> > > >>>>>> + final List<String> fields = new ArrayList<>();
> > > >>>>>> + final List<String> scriptFields = new ArrayList<>();
> > > >>>>>> for (Pair<RexNode, String> pair: getNamedProjects()) {
> > > >>>>>> final String name = pair.right;
> > > >>>>>> final String expr = pair.left.accept(translator);
> > > >>>>>>
> > > >>>>>> if (expr.equals("\"" + name + "\"")) {
> > > >>>>>> - findItems.add(ElasticsearchRules.quote(name));
> > > >>>>>> + fields.add(name);
> > > >>>>>> } else if (expr.matches("\"literal\":.+")) {
> > > >>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> > > >>>>>> - + ":{\"script\": "
> > > >>>>>> - + expr.split(":")[1] + "}");
> > > >>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
> > > >>>>>> + + ":{\"script\": "
> > > >>>>>> + + expr.split(":")[1] + "}");
> > > >>>>>> } else {
> > > >>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> > > >>>>>> - + ":{\"script\":\"params._source."
> > > >>>>>> - + expr.replaceAll("\"", "") + "\"}");
> > > >>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
> > > >>>>>> + + ":{\"script\":"
> > > >>>>>> + // _source (ES2) vs params._source (ES5)
> > > >>>>>> + + "\"" +
> > > >>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> > > >>>>>> + + expr.replaceAll("\"", "") + "\"}");
> > > >>>>>> + }
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + StringBuilder query = new StringBuilder();
> > > >>>>>> + if (scriptFields.isEmpty()) {
> > > >>>>>> + List<String> newList = Lists.transform(fields, new
> > > >>>>> Function<String, String>() {
> > > >>>>>> + @Nullable
> > > >>>>>> + @Override public String apply(@Nullable String input) {
> > > >>>>>> + return ElasticsearchRules.quote(input);
> > > >>>>>> + }
> > > >>>>>> + });
> > > >>>>>> +
> > > >>>>>> + final String findString = String.join(", ", newList);
> > > >>>>>> + query.append("\"_source\" :
> > > [").append(findString).append("]");
> > > >>>>>> + } else {
> > > >>>>>> + // if scripted fields are present, ES ignores _source
> > > attribute
> > > >>>>>> + for (String field: fields) {
> > > >>>>>> + scriptFields.add(ElasticsearchRules.quote(field) +
> > > >>>>> ":{\"script\": "
> > > >>>>>> + // _source (ES2) vs params._source (ES5)
> > > >>>>>> + + "\"" +
> > > >>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> > > >>>>>> + + field + "\"}");
> > > >>>>>> }
> > > >>>>>> + query.append("\"script_fields\": {" + String.join(", ",
> > > >>>>> scriptFields) + "}");
> > > >>>>>> }
> > > >>>>>> - final String findString = Util.toString(findItems, "", ",
> ",
> > > "");
> > > >>>>>> - final String scriptFieldString = "\"script_fields\": {"
> > > >>>>>> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
> > > >>>>>> - final String fieldString = "\"_source\" : [" + findString +
> > "]"
> > > >>>>>> - + ", " + scriptFieldString;
> > > >>>>>>
> > > >>>>>> for (String opfield : implementor.list) {
> > > >>>>>> if (opfield.startsWith("\"_source\"")) {
> > > >>>>>> implementor.list.remove(opfield);
> > > >>>>>> }
> > > >>>>>> }
> > > >>>>>> - implementor.add(fieldString);
> > > >>>>>> + implementor.add(query.toString());
> > > >>>>>> }
> > > >>>>>> }
> > > >>>>>>
> > > >>>>>>
> > > >>>>>>
> > > >>>>>
> > > >>>
> > >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
> > > >>>>>>
> > > ----------------------------------------------------------------------
> > > >>>>>> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
> > > >>>>>> index f24622c..6fbee03 100644
> > > >>>>>> --- a/elasticsearch2/pom.xml
> > > >>>>>> +++ b/elasticsearch2/pom.xml
> > > >>>>>> @@ -73,6 +73,12 @@ limitations under the License.
> > > >>>>>> <version>${elasticsearch-java-driver.version}</version>
> > > >>>>>> </dependency>
> > > >>>>>> <dependency>
> > > >>>>>> + <!-- Lang groovy dependency is needed for testing with
> > > embedded
> > > >>>>> ES (scripted fields like loc[0]) -->
> > > >>>>>> + <groupId>org.elasticsearch.module</groupId>
> > > >>>>>> + <artifactId>lang-groovy</artifactId>
> > > >>>>>> + <scope>test</scope>
> > > >>>>>> + </dependency>
> > > >>>>>> + <dependency>
> > > >>>>>> <groupId>com.carrotsearch</groupId>
> > > >>>>>> <artifactId>hppc</artifactId>
> > > >>>>>> <version>${hppc.version}</version>
> > > >>>>>>
> > > >>>>>>
> > > >>>>>
> > > >>>
> > >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > > >>>>>>
> > > ----------------------------------------------------------------------
> > > >>>>>> diff --git
> > > >>>>>
> > > >>>
> > >
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > > >>>>>
> > > >>>
> > >
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > > >>>>>> index 84370ab..c3d2ac0 100644
> > > >>>>>> ---
> > > >>>>>
> > > >>>
> > >
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > > >>>>>> +++
> > > >>>>>
> > > >>>
> > >
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > > >>>>>> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
> > > >>>>>> import java.util.Date;
> > > >>>>>> import java.util.Iterator;
> > > >>>>>> import java.util.List;
> > > >>>>>> +import java.util.Locale;
> > > >>>>>> import java.util.Map;
> > > >>>>>>
> > > >>>>>> /**
> > > >>>>>> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator
> > > implements
> > > >>>>> Enumerator<Object> {
> > > >>>>>> private static Function1<SearchHit, Object[]> listGetter(
> > > >>>>>> final List<Map.Entry<String, Class>> fields) {
> > > >>>>>> return new Function1<SearchHit, Object[]>() {
> > > >>>>>> - public Object[] apply(SearchHit searchHitFields) {
> > > >>>>>> + public Object[] apply(SearchHit hit) {
> > > >>>>>> Object[] objects = new Object[fields.size()];
> > > >>>>>> for (int i = 0; i < fields.size(); i++) {
> > > >>>>>> final Map.Entry<String, Class> field = fields.get(i);
> > > >>>>>> final String name = field.getKey();
> > > >>>>>> - if (searchHitFields.fields().isEmpty()) {
> > > >>>>>> - objects[i] =
> > > >>> convert(searchHitFields.getSource().get(name),
> > > >>>>> field.getValue());
> > > >>>>>> + if (hit.fields().isEmpty()) {
> > > >>>>>> + objects[i] = convert(hit.getSource().get(name),
> > > >>>>> field.getValue());
> > > >>>>>> + } else if (hit.fields().containsKey(name)) {
> > > >>>>>> + objects[i] = convert(hit.field(name).getValue(),
> > > >>>>> field.getValue());
> > > >>>>>> } else {
> > > >>>>>> - objects[i] =
> > > >>>>> convert(searchHitFields.field(name).getValue(),
> field.getValue());
> > > >>>>>> + throw new IllegalStateException(
> > > >>>>>> + String.format(Locale.getDefault(), "No
> result
> > > for
> > > >>>>> %s", field));
> > > >>>>>> }
> > > >>>>>> }
> > > >>>>>> return objects;
> > > >>>>>>
> > > >>>>>>
> > > >>>>>
> > > >>>
> > >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > > >>>>>>
> > > ----------------------------------------------------------------------
> > > >>>>>> diff --git
> > > >>>>>
> > > >>>
> > >
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > > >>>>>
> > > >>>
> > >
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > > >>>>>> index 668402b..46e3fc5 100644
> > > >>>>>> ---
> > > >>>>>
> > > >>>
> > >
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > > >>>>>> +++
> > > >>>>>
> > > >>>
> > >
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > > >>>>>> @@ -22,6 +22,9 @@ import
> > > org.apache.calcite.schema.impl.AbstractSchema;
> > > >>>>>>
> > > >>>>>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
> > > >>>>>>
> > > >>>>>> +import com.google.common.annotations.VisibleForTesting;
> > > >>>>>> +import com.google.common.base.Preconditions;
> > > >>>>>> +
> > > >>>>>> import com.google.common.collect.ImmutableList;
> > > >>>>>> import com.google.common.collect.ImmutableMap;
> > > >>>>>>
> > > >>>>>> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
> > > >>>>> AbstractSchema
> > > >>>>>> }
> > > >>>>>> }
> > > >>>>>>
> > > >>>>>> + /**
> > > >>>>>> + * Allows schema to be instantiated from existing elastic
> > search
> > > >>>>> client.
> > > >>>>>> + * This constructor is used in tests.
> > > >>>>>> + */
> > > >>>>>> + @VisibleForTesting
> > > >>>>>> + Elasticsearch2Schema(Client client, String index) {
> > > >>>>>> + this.client = Preconditions.checkNotNull(client, "client");
> > > >>>>>> + this.index = Preconditions.checkNotNull(index, "index");
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> @Override protected Map<String, Table> getTableMap() {
> > > >>>>>> final ImmutableMap.Builder<String, Table> builder =
> > > >>>>> ImmutableMap.builder();
> > > >>>>>>
> > > >>>>>> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
> > > >>>>> AbstractSchema
> > > >>>>>>
> > > >>>>>> final List<DiscoveryNode> nodes =
> > > >>>>> ImmutableList.copyOf(transportClient.connectedNodes());
> > > >>>>>> if (nodes.isEmpty()) {
> > > >>>>>> - throw new RuntimeException("Cannot connect to any
> > > elasticsearch
> > > >>>>> nodes");
> > > >>>>>> + throw new IllegalStateException("Cannot connect to any
> > > >>>>> elasticsearch node: "
> > > >>>>>> + + transportNodes);
> > > >>>>>> }
> > > >>>>>>
> > > >>>>>> client = transportClient;
> > > >>>>>>
> > > >>>>>>
> > > >>>>>
> > > >>>
> > >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > > >>>>>>
> > > ----------------------------------------------------------------------
> > > >>>>>> diff --git
> > > >>>>>
> > > >>>
> > >
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > > >>>>>
> > > >>>
> > >
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > > >>>>>> index 636aa5f..2928835 100644
> > > >>>>>> ---
> > > >>>>>
> > > >>>
> > >
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > > >>>>>> +++
> > > >>>>>
> > > >>>
> > >
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > > >>>>>> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
> > > >>>>> AbstractElasticsearchTable {
> > > >>>>>> this.client = client;
> > > >>>>>> }
> > > >>>>>>
> > > >>>>>> + /**
> > > >>>>>> + * ES version 2.x. To access document attributes ES2 uses
> > {@code
> > > >>>>> _source.foo} syntax.
> > > >>>>>> + */
> > > >>>>>> + @Override protected String scriptedFieldPrefix() {
> > > >>>>>> + return "_source";
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> @Override protected Enumerable<Object> find(String index,
> > > List<String>
> > > >>>>> ops,
> > > >>>>>> - List<Map.Entry<String, Class>> fields) {
> > > >>>>>> +
> > > List<Map.Entry<String,
> > > >>>>> Class>> fields) {
> > > >>>>>> final String dbName = index;
> > > >>>>>>
> > > >>>>>> final String queryString = "{" + Util.toString(ops, "", ", ",
> > "") +
> > > >>>>> "}";
> > > >>>>>>
> > > >>>>>>
> > > >>>>>
> > > >>>
> > >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > > >>>>>>
> > > ----------------------------------------------------------------------
> > > >>>>>> diff --git
> > > >>>>>
> > > >>>
> > >
> >
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > > >>>>>
> > > >>>
> > >
> >
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > > >>>>>> new file mode 100644
> > > >>>>>> index 0000000..287e094
> > > >>>>>> --- /dev/null
> > > >>>>>> +++
> > > >>>>>
> > > >>>
> > >
> >
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > > >>>>>> @@ -0,0 +1,395 @@
> > > >>>>>> +/*
> > > >>>>>> + * Licensed to the Apache Software Foundation (ASF) under one
> or
> > > more
> > > >>>>>> + * contributor license agreements. See the NOTICE file
> > distributed
> > > >>> with
> > > >>>>>> + * this work for additional information regarding copyright
> > > ownership.
> > > >>>>>> + * The ASF licenses this file to you under the Apache License,
> > > Version
> > > >>>>> 2.0
> > > >>>>>> + * (the "License"); you may not use this file except in
> > compliance
> > > >>> with
> > > >>>>>> + * the License. You may obtain a copy of the License at
> > > >>>>>> + *
> > > >>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
> > > >>>>>> + *
> > > >>>>>> + * Unless required by applicable law or agreed to in writing,
> > > software
> > > >>>>>> + * distributed under the License is distributed on an "AS IS"
> > > BASIS,
> > > >>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
> > or
> > > >>>>> implied.
> > > >>>>>> + * See the License for the specific language governing
> > permissions
> > > and
> > > >>>>>> + * limitations under the License.
> > > >>>>>> + */
> > > >>>>>> +package org.apache.calcite.adapter.elasticsearch2;
> > > >>>>>> +
> > > >>>>>> +import org.apache.calcite.jdbc.CalciteConnection;
> > > >>>>>> +import org.apache.calcite.schema.SchemaPlus;
> > > >>>>>> +import org.apache.calcite.schema.impl.ViewTable;
> > > >>>>>> +import org.apache.calcite.schema.impl.ViewTableMacro;
> > > >>>>>> +import org.apache.calcite.test.CalciteAssert;
> > > >>>>>> +import org.apache.calcite.test.ElasticChecker;
> > > >>>>>> +
> > > >>>>>> +import com.google.common.io.LineProcessor;
> > > >>>>>> +import com.google.common.io.Resources;
> > > >>>>>> +
> > > >>>>>> +import org.elasticsearch.action.bulk.BulkItemResponse;
> > > >>>>>> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
> > > >>>>>> +import org.elasticsearch.action.bulk.BulkResponse;
> > > >>>>>> +import org.elasticsearch.common.xcontent.XContentBuilder;
> > > >>>>>> +import org.elasticsearch.common.xcontent.XContentFactory;
> > > >>>>>> +
> > > >>>>>> +import org.junit.BeforeClass;
> > > >>>>>> +import org.junit.ClassRule;
> > > >>>>>> +import org.junit.Test;
> > > >>>>>> +
> > > >>>>>> +import java.io.IOException;
> > > >>>>>> +import java.nio.charset.StandardCharsets;
> > > >>>>>> +import java.sql.Connection;
> > > >>>>>> +import java.sql.DriverManager;
> > > >>>>>> +import java.sql.SQLException;
> > > >>>>>> +import java.util.Arrays;
> > > >>>>>> +import java.util.Collections;
> > > >>>>>> +import java.util.Locale;
> > > >>>>>> +
> > > >>>>>> +/**
> > > >>>>>> + * Set of tests for ES adapter. Uses real instance via {@link
> > > >>>>> EmbeddedElasticRule}. Document
> > > >>>>>> + * source is local {@code zips-mini.json} file (located in the
> > > >>>>> classpath).
> > > >>>>>> + */
> > > >>>>>> +public class ElasticSearch2AdapterTest {
> > > >>>>>> +
> > > >>>>>> + @ClassRule //init once for all tests
> > > >>>>>> + public static final EmbeddedElasticRule NODE =
> > > >>>>> EmbeddedElasticRule.create();
> > > >>>>>> +
> > > >>>>>> + private static final String ZIPS = "zips";
> > > >>>>>> +
> > > >>>>>> + /**
> > > >>>>>> + * Used to create {@code zips} index and insert some data
> > > >>>>>> + */
> > > >>>>>> + @BeforeClass
> > > >>>>>> + public static void setupInstance() throws Exception {
> > > >>>>>> + // define mapping so fields are searchable (term query)
> > > >>>>>> + XContentBuilder mapping =
> > > >>>>> XContentFactory.jsonBuilder().startObject()
> > > >>>>>> + .startObject("properties")
> > > >>>>>> + .startObject("city").field("type", "string")
> > > >>>>>> + .field("index", "not_analyzed").endObject()
> > > >>>>>> + .startObject("state").field("type", "string")
> > > >>>>>> + .field("index", "not_analyzed").endObject()
> > > >>>>>> + .startObject("pop").field("type",
> "long").endObject()
> > > >>>>>> + .endObject()
> > > >>>>>> + .endObject();
> > > >>>>>> +
> > > >>>>>> + // create index
> > > >>>>>> + NODE.client().admin().indices()
> > > >>>>>> + .prepareCreate(ZIPS)
> > > >>>>>> + .addMapping(ZIPS, mapping)
> > > >>>>>> + .get();
> > > >>>>>> +
> > > >>>>>> + BulkRequestBuilder bulk =
> > > >>>>> NODE.client().prepareBulk().setRefresh(true);
> > > >>>>>> +
> > > >>>>>> + // load records from file
> > > >>>>>> +
> > > >>>>>
> > > >>>
> > >
> >
> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
> > > >>>>>> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
> > > >>>>>> + @Override public boolean processLine(String line)
> > > throws
> > > >>>>> IOException {
> > > >>>>>> + line = line.replaceAll("_id", "id"); // _id is
> a
> > > >>>>> reserved attribute in ES
> > > >>>>>> + bulk.add(NODE.client().prepareIndex(ZIPS,
> > > >>>>> ZIPS).setSource(line));
> > > >>>>>> + return true;
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Override public Void getResult() {
> > > >>>>>> + return null;
> > > >>>>>> + }
> > > >>>>>> + });
> > > >>>>>> +
> > > >>>>>> + if (bulk.numberOfActions() == 0) {
> > > >>>>>> + throw new IllegalStateException("No records to be
> > indexed");
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + BulkResponse response = bulk.execute().get();
> > > >>>>>> +
> > > >>>>>> + if (response.hasFailures()) {
> > > >>>>>> + throw new IllegalStateException(
> > > >>>>>> + String.format(Locale.getDefault(), "Failed to
> > > populate
> > > >>>>> %s:\n%s", NODE.httpAddress(),
> > > >>>>>> +
> > > >>>>>
> > Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
> > > >>>>>> +
> > > >>>>>
> > > >>>
> > >
> >
> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + private CalciteAssert.ConnectionFactory
> newConnectionFactory()
> > {
> > > >>>>>> + return new CalciteAssert.ConnectionFactory() {
> > > >>>>>> + @Override public Connection createConnection() throws
> > > >>>>> SQLException {
> > > >>>>>> + final Connection connection =
> > > >>>>> DriverManager.getConnection("jdbc:calcite:");
> > > >>>>>> + final SchemaPlus root =
> > > >>>>> connection.unwrap(CalciteConnection.class).getRootSchema();
> > > >>>>>> +
> > > >>>>>> + root.add("elastic", new
> > Elasticsearch2Schema(NODE.client(),
> > > >>>>> ZIPS));
> > > >>>>>> +
> > > >>>>>> + // add calcite view programmatically
> > > >>>>>> + final String viewSql = "select cast(_MAP['city'] AS
> > > >>>>> varchar(20)) AS \"city\", "
> > > >>>>>> + + " cast(_MAP['loc'][0] AS float) AS
> > > \"longitude\",\n"
> > > >>>>>> + + " cast(_MAP['loc'][1] AS float) AS
> > > \"latitude\",\n"
> > > >>>>>> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
> > > >>>>>> + + " cast(_MAP['state'] AS varchar(2)) AS
> > \"state\",
> > > "
> > > >>>>>> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
> > > >>>>>> + + "from \"elastic\".\"zips\"";
> > > >>>>>> +
> > > >>>>>> + ViewTableMacro macro = ViewTable.viewMacro(root,
> viewSql,
> > > >>>>>> + Collections.singletonList("elastic"),
> > > >>>>> Arrays.asList("elastic", "view"), false);
> > > >>>>>> + root.add("ZIPS", macro);
> > > >>>>>> +
> > > >>>>>> + return connection;
> > > >>>>>> + }
> > > >>>>>> + };
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + private CalciteAssert.AssertThat calciteAssert() {
> > > >>>>>> + return CalciteAssert.that()
> > > >>>>>> + .with(newConnectionFactory());
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + /**
> > > >>>>>> + * Tests using calcite view
> > > >>>>>> + */
> > > >>>>>> + @Test
> > > >>>>>> + public void view() throws Exception {
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query("select * from zips where \"city\" =
> > 'BROOKLYN'")
> > > >>>>>> + .returns("city=BROOKLYN; longitude=-73.956985;
> > > >>>>> latitude=40.646694; "
> > > >>>>>> + + "pop=111396; state=NY; id=11226\n")
> > > >>>>>> + .returnsCount(1);
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test
> > > >>>>>> + public void emptyResult() {
> > > >>>>>> + CalciteAssert.that()
> > > >>>>>> + .with(newConnectionFactory())
> > > >>>>>> + .query("select * from zips limit 0")
> > > >>>>>> + .returnsCount(0);
> > > >>>>>> +
> > > >>>>>> + CalciteAssert.that()
> > > >>>>>> + .with(newConnectionFactory())
> > > >>>>>> + .query("select * from \"elastic\".\"zips\" where
> > > >>>>> _MAP['Foo'] = '_MISSING_'")
> > > >>>>>> + .returnsCount(0);
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test
> > > >>>>>> + public void basic() throws Exception {
> > > >>>>>> + CalciteAssert.that()
> > > >>>>>> + .with(newConnectionFactory())
> > > >>>>>> + .query("select * from \"elastic\".\"zips\" where
> > > >>>>> _MAP['city'] = 'BROOKLYN'")
> > > >>>>>> + .returnsCount(1);
> > > >>>>>> +
> > > >>>>>> + CalciteAssert.that()
> > > >>>>>> + .with(newConnectionFactory())
> > > >>>>>> + .query("select * from \"elastic\".\"zips\" where"
> > > >>>>>> + + " _MAP['city'] in ('BROOKLYN',
> > > 'WASHINGTON')")
> > > >>>>>> + .returnsCount(2);
> > > >>>>>> +
> > > >>>>>> + // lower-case
> > > >>>>>> + CalciteAssert.that()
> > > >>>>>> + .with(newConnectionFactory())
> > > >>>>>> + .query("select * from \"elastic\".\"zips\" where "
> > > >>>>>> + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
> > > >>>>> 'BROOK') ")
> > > >>>>>> + .returnsCount(0);
> > > >>>>>> +
> > > >>>>>> + // missing field
> > > >>>>>> + CalciteAssert.that()
> > > >>>>>> + .with(newConnectionFactory())
> > > >>>>>> + .query("select * from \"elastic\".\"zips\" where
> > > >>>>> _MAP['CITY'] = 'BROOKLYN'")
> > > >>>>>> + .returnsCount(0);
> > > >>>>>> +
> > > >>>>>> + // limit works
> > > >>>>>> + CalciteAssert.that()
> > > >>>>>> + .with(newConnectionFactory())
> > > >>>>>> + .query("select * from \"elastic\".\"zips\" limit
> 42")
> > > >>>>>> + .returnsCount(42);
> > > >>>>>> +
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test public void testSort() {
> > > >>>>>> + final String explain =
> > > "PLAN=ElasticsearchToEnumerableConverter\n"
> > > >>>>>> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> > > >>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> > > >>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> > > >>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0,
> > 'loc'),
> > > >>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> > > >>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> > > >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> > > >>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0,
> 'id')):VARCHAR(5)
> > > >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> > > \"ISO-8859-1$en_US$primary\"])\n"
> > > >>>>>> + + " ElasticsearchTableScan(table=[[elastic,
> > > zips]])";
> > > >>>>>> +
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query("select * from zips order by \"state\"")
> > > >>>>>> + .returnsCount(10)
> > > >>>>>> + .explainContains(explain);
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test public void testSortLimit() {
> > > >>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> > > >>>>>> + + "order by \"state\", \"id\" offset 2 rows fetch
> > next
> > > 3
> > > >>>>> rows only";
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query(sql)
> > > >>>>>> + .returnsUnordered("state=AK; id=99801",
> > > >>>>>> + "state=AL; id=35215",
> > > >>>>>> + "state=AL; id=35401")
> > > >>>>>> + .queryContains(
> > > >>>>>> + ElasticChecker.elasticsearchChecker(
> > > >>>>>> + "\"_source\" : [\"state\",
> \"id\"]",
> > > >>>>>> + "\"sort\": [ {\"state\": \"asc\"},
> > > >>> {\"id\":
> > > >>>>> \"asc\"}]",
> > > >>>>>> + "\"from\": 2",
> > > >>>>>> + "\"size\": 3"));
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> +
> > > >>>>>> +
> > > >>>>>> + @Test public void testOffsetLimit() {
> > > >>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> > > >>>>>> + + "offset 2 fetch next 3 rows only";
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query(sql)
> > > >>>>>> + .runs()
> > > >>>>>> + .queryContains(
> > > >>>>>> + ElasticChecker.elasticsearchChecker(
> > > >>>>>> + "\"from\": 2",
> > > >>>>>> + "\"size\": 3",
> > > >>>>>> + "\"_source\" : [\"state\",
> > \"id\"]"));
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test public void testLimit() {
> > > >>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> > > >>>>>> + + "fetch next 3 rows only";
> > > >>>>>> +
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query(sql)
> > > >>>>>> + .runs()
> > > >>>>>> + .queryContains(
> > > >>>>>> + ElasticChecker.elasticsearchChecker(
> > > >>>>>> + "\"size\": 3",
> > > >>>>>> + "\"_source\" : [\"state\",
> > \"id\"]"));
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test public void testFilterSort() {
> > > >>>>>> + final String sql = "select * from zips\n"
> > > >>>>>> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
> > > >>>>>> + + "order by \"state\", \"id\"";
> > > >>>>>> + final String explain =
> > > "PLAN=ElasticsearchToEnumerableConverter\n"
> > > >>>>>> + + " ElasticsearchSort(sort0=[$4], sort1=[$5],
> > > dir0=[ASC],
> > > >>>>> dir1=[ASC])\n"
> > > >>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> > > >>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> > > >>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0,
> > 'loc'),
> > > >>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> > > >>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> > > >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> > > >>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0,
> 'id')):VARCHAR(5)
> > > >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> > > \"ISO-8859-1$en_US$primary\"])\n"
> > > >>>>>> + + "
> > > >>> ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
> > > >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> > > >>>>> \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0,
> > > 'id')):VARCHAR(5)
> > > >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\",
> > > >>>>> '70000'))])\n"
> > > >>>>>> + + " ElasticsearchTableScan(table=[[elastic,
> > > >>> zips]])";
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query(sql)
> > > >>>>>> + .returnsOrdered("city=LOS ANGELES;
> > > longitude=-118.258189;
> > > >>>>> latitude=34.007856; "
> > > >>>>>> + + "pop=96074; state=CA; id=90011",
> > > >>>>>> + "city=BELL GARDENS; longitude=-118.17205;
> > > >>>>> latitude=33.969177; "
> > > >>>>>> + + "pop=99568; state=CA; id=90201",
> > > >>>>>> + "city=NORWALK; longitude=-118.081767;
> > > >>>>> latitude=33.90564; "
> > > >>>>>> + + "pop=94188; state=CA; id=90650")
> > > >>>>>> + .queryContains(
> > > >>>>>> +
> > ElasticChecker.elasticsearchChecker("\"query\"
> > > : "
> > > >>>>>> + +
> > > >>>>> "{\"constant_score\":{\"filter\":{\"bool\":"
> > > >>>>>> + +
> > > >>>>> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
> > > >>>>>> + +
> > > >>>>> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
> > > >>>>>> + "\"script_fields\":
> > > >>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> > > >>>>>> + +
> > > >>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> > > >>>>>> + + "\"city\":{\"script\":
> > > >>>>> \"_source.city\"}, "
> > > >>>>>> + + "\"pop\":{\"script\":
> > > >>>>> \"_source.pop\"}, "
> > > >>>>>> + + "\"state\":{\"script\":
> > > >>>>> \"_source.state\"}, "
> > > >>>>>> + + "\"id\":{\"script\": \"_
> > > >>> source.id
> > > >>>>> \"}}",
> > > >>>>>> + "\"sort\": [ {\"state\": \"asc\"},
> > > >>> {\"id\":
> > > >>>>> \"asc\"}]"))
> > > >>>>>> + .explainContains(explain);
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test public void testFilterSortDesc() {
> > > >>>>>> + final String sql = "select * from zips\n"
> > > >>>>>> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
> > > >>>>>> + + "order by \"state\" desc, \"pop\"";
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query(sql)
> > > >>>>>> + .limit(4)
> > > >>>>>> + .returnsOrdered(
> > > >>>>>> + "city=LOS ANGELES; longitude=-118.258189;
> > > >>>>> latitude=34.007856; pop=96074; state=CA; id=90011",
> > > >>>>>> + "city=BELL GARDENS; longitude=-118.17205;
> > > >>>>> latitude=33.969177; pop=99568; state=CA; id=90201");
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test public void testFilterRedundant() {
> > > >>>>>> + final String sql = "select * from zips\n"
> > > >>>>>> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and
> > > >>>>> \"state\" = 'OK'";
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query(sql)
> > > >>>>>> + .runs()
> > > >>>>>> + .queryContains(
> > > >>>>>> + ElasticChecker.elasticsearchChecker(""
> > > >>>>>> + + "\"query\" :
> > > >>>>> {\"constant_score\":{\"filter\":{\"bool\":"
> > > >>>>>> + +
> > > >>>>> "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
> > > >>>>>> + "\"script_fields\":
> > > >>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> > > >>>>>> + +
> > > >>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> > > >>>>>> + + "\"city\":{\"script\":
> > > >>>>> \"_source.city\"}, "
> > > >>>>>> + + "\"pop\":{\"script\":
> > > >>>>> \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
> > > >>>>>> + +
> > > "\"id\":{\"script\":
> > > >>>>> \"_source.id\"}}"
> > > >>>>>> + ));
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test public void testInPlan() {
> > > >>>>>> + final String[] searches = {
> > > >>>>>> + "\"query\" :
> > > >>>>> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
> > > >>>>>> + +
> > > >>>>>
> > > >>>
> > >
> >
> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
> > > >>>>>> + + "{\"pop\":99568}}]}}]}}}}",
> > > >>>>>> + "\"script_fields\":
> > > >>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> > > >>>>>> + +
> > "\"latitude\":{\"script\":\"_source.loc[1]\"},
> > > "
> > > >>>>>> + + "\"city\":{\"script\": \"_source.city\"}, "
> > > >>>>>> + + "\"pop\":{\"script\": \"_source.pop\"}, "
> > > >>>>>> + + "\"state\":{\"script\":
> \"_source.state\"}, "
> > > >>>>>> + + "\"id\":{\"script\": \"_source.id\"}}"
> > > >>>>>> + };
> > > >>>>>> +
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query("select * from zips where \"pop\" in (96074,
> > > >>> 99568)")
> > > >>>>>> + .returnsUnordered(
> > > >>>>>> + "city=BELL GARDENS; longitude=-118.17205;
> > > >>>>> latitude=33.969177; pop=99568; state=CA; id=90201",
> > > >>>>>> + "city=LOS ANGELES; longitude=-118.258189;
> > > >>>>> latitude=34.007856; pop=96074; state=CA; id=90011"
> > > >>>>>> + )
> > > >>>>>> +
> > > >>>>> .queryContains(ElasticChecker.elasticsearchChecker(searches));
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test public void testZips() {
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query("select \"state\", \"city\" from zips")
> > > >>>>>> + .returnsCount(10);
> > > >>>>>> + }
> > > >>>>>> +
> > > >>>>>> + @Test public void testProject() {
> > > >>>>>> + final String sql = "select \"state\", \"city\", 0 as
> > > \"zero\"\n"
> > > >>>>>> + + "from zips\n"
> > > >>>>>> + + "order by \"state\", \"city\"";
> > > >>>>>> +
> > > >>>>>> + calciteAssert()
> > > >>>>>> + .query(sql)
> > > >>>>>> + .limit(2)
> > > >>>>>> + .returnsUnordered("state=AK; city=ANCHORAGE;
> zero=0",
> > > >>>>>> + "state=AK; city=FAIRBANKS; zero=0")
> > > >>>>>> + .queryContains(
> > > >>>>>> +
> > > >>>>> ElasticChecker.elasticsearchChecker("\"script_fields\": "
> > > >>>>>> + + "{\"zero\":{\"script\":
> > > \"0\"},
> > > >>> "
> > > >>>>>> + + "\"state\":{\"script\":
> > > >>>>> \"_source.state\"}, "
> > > >>>>>> + + "\"city\":{\"script\":
> > > >>>>> \"_source.city\"}}",
> > > >>>>>> + "\"sort
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Andrei Sereda <an...@sereda.cc>.
Can you please check the branch in my fork ?
fork: https://github.com/asereda-gs/calcite
branch: javadoc-fixes
I've tried to remove most of the warnings and javadoc, javadoc-test and
site all work for me.
~/I/c/elasticsearch5 ❯❯❯ mvn --version
javadoc-fixes ✭
Apache Maven 3.5.4 (1edded0938998edf8bf061f1ceb3cfdeccf443fe;
2018-06-17T14:33:14-04:00)
Maven home: /usr/local/Cellar/maven/3.5.4/libexec
Java version: 1.8.0_151, vendor: Oracle Corporation, runtime:
/Library/Java/JavaVirtualMachines/jdk1.8.0_151.jdk/Contents/Home/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "mac os x", version: "10.13.2", arch: "x86_64", family: "mac"
On Sun, Jun 24, 2018 at 12:40 PM Volodymyr Vysotskyi <vo...@apache.org>
wrote:
> I have also checked "mvn javadoc:javadoc javadoc:test-javadoc" on my
> machine (Ubuntu, JDK 8, Maven 3.3.9) and it fails on elasticsearch2 module
> (didn't check this problem for the next modules).
>
> "mvn clean -DskipTests site" fails for MongoDB module.
>
> Kind regards,
> Volodymyr Vysotskyi
>
>
> сб, 23 черв. 2018 о 03:12 Julian Hyde <jh...@apache.org> пише:
>
> > Same failure for me on JDK 8/macOS. The most likely other variable is
> > maven: I have 3.5.2 on macOS, 3.5.3 on ubuntu.
> >
> > I’m mystified why no one else is seeing javadoc failures on macOS.
> >
> > Julian
> >
> >
> > > On Jun 22, 2018, at 4:35 PM, Julian Hyde <jh...@apache.org> wrote:
> > >
> > > On JDK 8/ubuntu, “mvn clean -DskipTests site” gives the following
> > failure:
> > >
> > > [ERROR] Failed to execute goal
> > org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> > project calcite-mongodb: Error generating
> > maven-javadoc-plugin:3.0.1:test-javadoc report:
> > > [ERROR] Exit code: 1 -
> >
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:22:
> > error: cannot find symbol
> > > [ERROR] import org.apache.calcite.test.CalciteAssert;
> > > [ERROR] ^
> > > [ERROR] symbol: class CalciteAssert
> > > [ERROR] location: package org.apache.calcite.test
> > > [ERROR]
> >
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:137:
> > error: package CalciteAssert does not exist
> > > [ERROR] private CalciteAssert.AssertThat assertModel(String model) {
> > > [ERROR] ^
> > > [ERROR]
> >
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:145:
> > error: package CalciteAssert does not exist
> > > [ERROR] private CalciteAssert.AssertThat assertModel(URL url) {
> > > [ERROR] ^
> > > [ERROR]
> >
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:43:
> > error: reference not found
> > > [ERROR] /** Similar to {@link CalciteAssert#checkResultUnordered},
> but
> > filters strings
> > > [ERROR] ^
> > >
> > >
> > >
> > >> On Jun 22, 2018, at 3:46 PM, Andrei Sereda <an...@sereda.cc> wrote:
> > >>
> > >> Hmm. Both master and c12cb4b0de work for me. Perhaps the issue is JDK
> 8
> > vs
> > >> 9 ?
> > >>
> > >> # JDK8 (oracle) / macOS
> > >> $ mvn clean -DskipTests site
> > >>
> > >> On Fri, Jun 22, 2018 at 5:56 PM Julian Hyde <jh...@apache.org> wrote:
> > >>
> > >>> I just tried it again. The following fails for me (jdk9, ubuntu):
> > >>>
> > >>> $ git checkout c12cb4b0de
> > >>> $ mvn clean -DskipTests site
> > >>>
> > >>> Constructing Javadoc information...
> > >>> 1 error
> > >>> [INFO]
> > >>>
> > ------------------------------------------------------------------------
> > >>> [INFO] Reactor Summary:
> > >>> [INFO]
> > >>> [INFO] Calcite 1.17.0-SNAPSHOT ............................ FAILURE
> > [02:29
> > >>> min]
> > >>> [INFO] Calcite Linq4j ..................................... SKIPPED
> > >>> [INFO] Calcite Core ....................................... SKIPPED
> > >>> [INFO] Calcite Cassandra .................................. SKIPPED
> > >>> [INFO] Calcite Druid ...................................... SKIPPED
> > >>> [INFO] Calcite Elasticsearch .............................. SKIPPED
> > >>> [INFO] Calcite Elasticsearch5 ............................. SKIPPED
> > >>> [INFO] Calcite Examples ................................... SKIPPED
> > >>> [INFO] Calcite Example CSV ................................ SKIPPED
> > >>> [INFO] Calcite Example Function ........................... SKIPPED
> > >>> [INFO] Calcite File ....................................... SKIPPED
> > >>> [INFO] Calcite Geode ...................................... SKIPPED
> > >>> [INFO] Calcite MongoDB .................................... SKIPPED
> > >>> [INFO] Calcite Pig ........................................ SKIPPED
> > >>> [INFO] Calcite Piglet ..................................... SKIPPED
> > >>> [INFO] Calcite Plus ....................................... SKIPPED
> > >>> [INFO] Calcite Server ..................................... SKIPPED
> > >>> [INFO] Calcite Spark ...................................... SKIPPED
> > >>> [INFO] Calcite Splunk ..................................... SKIPPED
> > >>> [INFO] Calcite Ubenchmark 1.17.0-SNAPSHOT ................. SKIPPED
> > >>> [INFO]
> > >>>
> > ------------------------------------------------------------------------
> > >>> [INFO] BUILD FAILURE
> > >>> [INFO]
> > >>>
> > ------------------------------------------------------------------------
> > >>> [INFO] Total time: 02:29 min
> > >>> [INFO] Finished at: 2018-06-22T14:53:13-07:00
> > >>> [INFO]
> > >>>
> > ------------------------------------------------------------------------
> > >>> [ERROR] Failed to execute goal
> > >>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> > >>> project calcite: Error generating
> > maven-javadoc-plugin:3.0.1:test-aggregate
> > >>> report:
> > >>> [ERROR] Exit code: 1 -
> > >>>
> >
> /home/jhyde/open1/calcite.4/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> > >>> error: package org.elasticsearch.node.internal does not exist
> > >>> [ERROR] import
> > org.elasticsearch.node.internal.InternalSettingsPreparer;
> > >>> [ERROR] ^
> > >>> [ERROR]
> > >>> [ERROR] Command line was: /usr/lib/jvm/jdk9/bin/javadoc @options
> > @packages
> > >>> @argfile
> > >>> [ERROR]
> > >>> [ERROR] Refer to the generated Javadoc files in
> > >>> '/home/jhyde/open1/calcite.4/target/site/testapidocs' dir.
> > >>> [ERROR] -> [Help 1]
> > >>> [ERROR]
> > >>> [ERROR] To see the full stack trace of the errors, re-run Maven with
> > the
> > >>> -e switch.
> > >>> [ERROR] Re-run Maven using the -X switch to enable full debug
> logging.
> > >>> [ERROR]
> > >>> [ERROR] For more information about the errors and possible solutions,
> > >>> please read the following articles:
> > >>> [ERROR] [Help 1]
> > >>>
> > http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException
> > >>>
> > >>>
> > >>>> On Jun 22, 2018, at 2:41 PM, Michael Mior <mm...@apache.org> wrote:
> > >>>>
> > >>>> Odd, mvn clean site still works fine for me.
> > >>>> --
> > >>>> Michael Mior
> > >>>> mmior@apache.org
> > >>>>
> > >>>>
> > >>>>
> > >>>> Le ven. 22 juin 2018 à 17:12, Julian Hyde <jh...@apache.org> a
> écrit
> > :
> > >>>>
> > >>>>> Looks like this change broke “mvn site” (perhaps also “mvn
> > >>>>> javadoc:test-javadoc”).
> > >>>>>
> > >>>>> [ERROR] Failed to execute goal
> > >>>>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site)
> on
> > >>>>> project calcite: Error generating
> > >>> maven-javadoc-plugin:3.0.1:test-aggregate
> > >>>>> report:
> > >>>>> [ERROR] Exit code: 1 -
> > >>>>>
> > >>>
> >
> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> > >>>>> error: package org.elasticsearch.node.internal does not exist
> > >>>>> [ERROR] import
> > org.elasticsearch.node.internal.InternalSettingsPreparer;
> > >>>>> [ERROR] ^
> > >>>>> [ERROR]
> > >>>>>
> > >>>>>> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
> > >>>>>>
> > >>>>>> [CALCITE-2347] running ElasticSearch in embedded mode for unit
> > tests of
> > >>>>> ES adapter (Andrei Sereda)
> > >>>>>>
> > >>>>>> After discussion on dev-list Integration tests (for ES) have been
> > >>>>> removed. They're now
> > >>>>>> superseded by unit tests (which execute queries against a real
> > elastic
> > >>>>> instance)
> > >>>>>>
> > >>>>>> Added local file (zips-mini.json) which contains a small subset of
> > >>>>> original zips.json
> > >>>>>> (allows to bootstrap tests faster)
> > >>>>>>
> > >>>>>> Created separate ES JUnit rule which can be re-used across
> different
> > >>>>> tests.
> > >>>>>>
> > >>>>>> Both v2 and v5 of ES adapters are supported.
> > >>>>>>
> > >>>>>> Close apache/calcite#716
> > >>>>>>
> > >>>>>>
> > >>>>>> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
> > >>>>>> Commit:
> > http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
> > >>>>>> Tree:
> http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
> > >>>>>> Diff:
> http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
> > >>>>>>
> > >>>>>> Branch: refs/heads/master
> > >>>>>> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
> > >>>>>> Parents: 37944bb
> > >>>>>> Author: Andrei Sereda <an...@nospam.com>
> > >>>>>> Authored: Thu May 31 18:19:10 2018 -0400
> > >>>>>> Committer: Michael Mior <mm...@uwaterloo.ca>
> > >>>>>> Committed: Thu Jun 21 06:38:50 2018 -0400
> > >>>>>>
> > >>>>>>
> > ----------------------------------------------------------------------
> > >>>>>> .../AbstractElasticsearchTable.java | 12 +
> > >>>>>> .../elasticsearch/ElasticsearchProject.java | 61 ++-
> > >>>>>> elasticsearch2/pom.xml | 6 +
> > >>>>>> .../Elasticsearch2Enumerator.java | 12 +-
> > >>>>>> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
> > >>>>>> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
> > >>>>>> .../ElasticSearch2AdapterTest.java | 395
> > >>> ++++++++++++++++++
> > >>>>>> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
> > >>>>>> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
> > >>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> > >>>>>> .../calcite/test/Elasticsearch2AdapterIT.java | 270
> -------------
> > >>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
> > >>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
> > >>>>>> elasticsearch5/pom.xml | 31 ++
> > >>>>>> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
> > >>>>>> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
> > >>>>>> .../ElasticSearch5AdapterTest.java | 399
> > >>> +++++++++++++++++++
> > >>>>>> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
> > >>>>>> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
> > >>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> > >>>>>> .../calcite/test/Elasticsearch5AdapterIT.java | 270
> -------------
> > >>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
> > >>>>>> elasticsearch5/src/test/resources/log4j2.xml | 16 +
> > >>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
> > >>>>>> pom.xml | 20 +-
> > >>>>>> 25 files changed, 1866 insertions(+), 670 deletions(-)
> > >>>>>>
> > ----------------------------------------------------------------------
> > >>>>>>
> > >>>>>>
> > >>>>>>
> > >>>>>
> > >>>
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > >>>>>>
> > ----------------------------------------------------------------------
> > >>>>>> diff --git
> > >>>>>
> > >>>
> >
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > >>>>>
> > >>>
> >
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > >>>>>> index 0980469..8cc5933 100644
> > >>>>>> ---
> > >>>>>
> > >>>
> >
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > >>>>>> +++
> > >>>>>
> > >>>
> >
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > >>>>>> @@ -75,6 +75,18 @@ public abstract class
> AbstractElasticsearchTable
> > >>>>> extends AbstractQueryableTable
> > >>>>>> relOptTable, this, null);
> > >>>>>> }
> > >>>>>>
> > >>>>>> + /**
> > >>>>>> + * In ES 5.x scripted fields start with {@code
> > params._source.foo}
> > >>>>> while in ES2.x
> > >>>>>> + * {@code _source.foo}. Helper method to build correct query
> > based
> > >>> on
> > >>>>> runtime version of elastic.
> > >>>>>> + *
> > >>>>>> + * @see <a href="
> > >>>>> https://github.com/elastic/elasticsearch/issues/20068">_source
> > >>>>> variable</a>
> > >>>>>> + * @see <a href="
> > >>>>>
> > >>>
> >
> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html
> > >>> ">Scripted
> > >>>>> Fields</a>
> > >>>>>> + */
> > >>>>>> + protected String scriptedFieldPrefix() {
> > >>>>>> + // this is default pattern starting 5.x
> > >>>>>> + return "params._source";
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> /** Executes a "find" operation on the underlying type.
> > >>>>>> *
> > >>>>>> * <p>For example,
> > >>>>>>
> > >>>>>>
> > >>>>>
> > >>>
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > >>>>>>
> > ----------------------------------------------------------------------
> > >>>>>> diff --git
> > >>>>>
> > >>>
> >
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > >>>>>
> > >>>
> >
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > >>>>>> index b42abd7..961c8b0 100644
> > >>>>>> ---
> > >>>>>
> > >>>
> >
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > >>>>>> +++
> > >>>>>
> > >>>
> >
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > >>>>>> @@ -27,11 +27,15 @@ import
> > >>>>> org.apache.calcite.rel.metadata.RelMetadataQuery;
> > >>>>>> import org.apache.calcite.rel.type.RelDataType;
> > >>>>>> import org.apache.calcite.rex.RexNode;
> > >>>>>> import org.apache.calcite.util.Pair;
> > >>>>>> -import org.apache.calcite.util.Util;
> > >>>>>> +
> > >>>>>> +import com.google.common.base.Function;
> > >>>>>> +import com.google.common.collect.Lists;
> > >>>>>>
> > >>>>>> import java.util.ArrayList;
> > >>>>>> import java.util.List;
> > >>>>>>
> > >>>>>> +import javax.annotation.Nullable;
> > >>>>>> +
> > >>>>>> /**
> > >>>>>> * Implementation of {@link org.apache.calcite.rel.core.Project}
> > >>>>>> * relational expression in Elasticsearch.
> > >>>>>> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends
> > Project
> > >>>>> implements ElasticsearchRel {
> > >>>>>> implementor.visitChild(0, getInput());
> > >>>>>>
> > >>>>>> final List<String> inFields =
> > >>>>>> -
> > >>>>>
> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> > >>>>>> +
> > >>>>>
> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> > >>>>>> final ElasticsearchRules.RexToElasticsearchTranslator translator
> =
> > >>>>>> - new ElasticsearchRules.RexToElasticsearchTranslator(
> > >>>>>> - (JavaTypeFactory) getCluster().getTypeFactory(),
> > >>> inFields);
> > >>>>>> + new ElasticsearchRules.RexToElasticsearchTranslator(
> > >>>>>> + (JavaTypeFactory)
> > getCluster().getTypeFactory(),
> > >>>>> inFields);
> > >>>>>>
> > >>>>>> - final List<String> findItems = new ArrayList<>();
> > >>>>>> - final List<String> scriptFieldItems = new ArrayList<>();
> > >>>>>> + final List<String> fields = new ArrayList<>();
> > >>>>>> + final List<String> scriptFields = new ArrayList<>();
> > >>>>>> for (Pair<RexNode, String> pair: getNamedProjects()) {
> > >>>>>> final String name = pair.right;
> > >>>>>> final String expr = pair.left.accept(translator);
> > >>>>>>
> > >>>>>> if (expr.equals("\"" + name + "\"")) {
> > >>>>>> - findItems.add(ElasticsearchRules.quote(name));
> > >>>>>> + fields.add(name);
> > >>>>>> } else if (expr.matches("\"literal\":.+")) {
> > >>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> > >>>>>> - + ":{\"script\": "
> > >>>>>> - + expr.split(":")[1] + "}");
> > >>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
> > >>>>>> + + ":{\"script\": "
> > >>>>>> + + expr.split(":")[1] + "}");
> > >>>>>> } else {
> > >>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> > >>>>>> - + ":{\"script\":\"params._source."
> > >>>>>> - + expr.replaceAll("\"", "") + "\"}");
> > >>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
> > >>>>>> + + ":{\"script\":"
> > >>>>>> + // _source (ES2) vs params._source (ES5)
> > >>>>>> + + "\"" +
> > >>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> > >>>>>> + + expr.replaceAll("\"", "") + "\"}");
> > >>>>>> + }
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + StringBuilder query = new StringBuilder();
> > >>>>>> + if (scriptFields.isEmpty()) {
> > >>>>>> + List<String> newList = Lists.transform(fields, new
> > >>>>> Function<String, String>() {
> > >>>>>> + @Nullable
> > >>>>>> + @Override public String apply(@Nullable String input) {
> > >>>>>> + return ElasticsearchRules.quote(input);
> > >>>>>> + }
> > >>>>>> + });
> > >>>>>> +
> > >>>>>> + final String findString = String.join(", ", newList);
> > >>>>>> + query.append("\"_source\" :
> > [").append(findString).append("]");
> > >>>>>> + } else {
> > >>>>>> + // if scripted fields are present, ES ignores _source
> > attribute
> > >>>>>> + for (String field: fields) {
> > >>>>>> + scriptFields.add(ElasticsearchRules.quote(field) +
> > >>>>> ":{\"script\": "
> > >>>>>> + // _source (ES2) vs params._source (ES5)
> > >>>>>> + + "\"" +
> > >>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> > >>>>>> + + field + "\"}");
> > >>>>>> }
> > >>>>>> + query.append("\"script_fields\": {" + String.join(", ",
> > >>>>> scriptFields) + "}");
> > >>>>>> }
> > >>>>>> - final String findString = Util.toString(findItems, "", ", ",
> > "");
> > >>>>>> - final String scriptFieldString = "\"script_fields\": {"
> > >>>>>> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
> > >>>>>> - final String fieldString = "\"_source\" : [" + findString +
> "]"
> > >>>>>> - + ", " + scriptFieldString;
> > >>>>>>
> > >>>>>> for (String opfield : implementor.list) {
> > >>>>>> if (opfield.startsWith("\"_source\"")) {
> > >>>>>> implementor.list.remove(opfield);
> > >>>>>> }
> > >>>>>> }
> > >>>>>> - implementor.add(fieldString);
> > >>>>>> + implementor.add(query.toString());
> > >>>>>> }
> > >>>>>> }
> > >>>>>>
> > >>>>>>
> > >>>>>>
> > >>>>>
> > >>>
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
> > >>>>>>
> > ----------------------------------------------------------------------
> > >>>>>> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
> > >>>>>> index f24622c..6fbee03 100644
> > >>>>>> --- a/elasticsearch2/pom.xml
> > >>>>>> +++ b/elasticsearch2/pom.xml
> > >>>>>> @@ -73,6 +73,12 @@ limitations under the License.
> > >>>>>> <version>${elasticsearch-java-driver.version}</version>
> > >>>>>> </dependency>
> > >>>>>> <dependency>
> > >>>>>> + <!-- Lang groovy dependency is needed for testing with
> > embedded
> > >>>>> ES (scripted fields like loc[0]) -->
> > >>>>>> + <groupId>org.elasticsearch.module</groupId>
> > >>>>>> + <artifactId>lang-groovy</artifactId>
> > >>>>>> + <scope>test</scope>
> > >>>>>> + </dependency>
> > >>>>>> + <dependency>
> > >>>>>> <groupId>com.carrotsearch</groupId>
> > >>>>>> <artifactId>hppc</artifactId>
> > >>>>>> <version>${hppc.version}</version>
> > >>>>>>
> > >>>>>>
> > >>>>>
> > >>>
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > >>>>>>
> > ----------------------------------------------------------------------
> > >>>>>> diff --git
> > >>>>>
> > >>>
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > >>>>>
> > >>>
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > >>>>>> index 84370ab..c3d2ac0 100644
> > >>>>>> ---
> > >>>>>
> > >>>
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > >>>>>> +++
> > >>>>>
> > >>>
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > >>>>>> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
> > >>>>>> import java.util.Date;
> > >>>>>> import java.util.Iterator;
> > >>>>>> import java.util.List;
> > >>>>>> +import java.util.Locale;
> > >>>>>> import java.util.Map;
> > >>>>>>
> > >>>>>> /**
> > >>>>>> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator
> > implements
> > >>>>> Enumerator<Object> {
> > >>>>>> private static Function1<SearchHit, Object[]> listGetter(
> > >>>>>> final List<Map.Entry<String, Class>> fields) {
> > >>>>>> return new Function1<SearchHit, Object[]>() {
> > >>>>>> - public Object[] apply(SearchHit searchHitFields) {
> > >>>>>> + public Object[] apply(SearchHit hit) {
> > >>>>>> Object[] objects = new Object[fields.size()];
> > >>>>>> for (int i = 0; i < fields.size(); i++) {
> > >>>>>> final Map.Entry<String, Class> field = fields.get(i);
> > >>>>>> final String name = field.getKey();
> > >>>>>> - if (searchHitFields.fields().isEmpty()) {
> > >>>>>> - objects[i] =
> > >>> convert(searchHitFields.getSource().get(name),
> > >>>>> field.getValue());
> > >>>>>> + if (hit.fields().isEmpty()) {
> > >>>>>> + objects[i] = convert(hit.getSource().get(name),
> > >>>>> field.getValue());
> > >>>>>> + } else if (hit.fields().containsKey(name)) {
> > >>>>>> + objects[i] = convert(hit.field(name).getValue(),
> > >>>>> field.getValue());
> > >>>>>> } else {
> > >>>>>> - objects[i] =
> > >>>>> convert(searchHitFields.field(name).getValue(), field.getValue());
> > >>>>>> + throw new IllegalStateException(
> > >>>>>> + String.format(Locale.getDefault(), "No result
> > for
> > >>>>> %s", field));
> > >>>>>> }
> > >>>>>> }
> > >>>>>> return objects;
> > >>>>>>
> > >>>>>>
> > >>>>>
> > >>>
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > >>>>>>
> > ----------------------------------------------------------------------
> > >>>>>> diff --git
> > >>>>>
> > >>>
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > >>>>>
> > >>>
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > >>>>>> index 668402b..46e3fc5 100644
> > >>>>>> ---
> > >>>>>
> > >>>
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > >>>>>> +++
> > >>>>>
> > >>>
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > >>>>>> @@ -22,6 +22,9 @@ import
> > org.apache.calcite.schema.impl.AbstractSchema;
> > >>>>>>
> > >>>>>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
> > >>>>>>
> > >>>>>> +import com.google.common.annotations.VisibleForTesting;
> > >>>>>> +import com.google.common.base.Preconditions;
> > >>>>>> +
> > >>>>>> import com.google.common.collect.ImmutableList;
> > >>>>>> import com.google.common.collect.ImmutableMap;
> > >>>>>>
> > >>>>>> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
> > >>>>> AbstractSchema
> > >>>>>> }
> > >>>>>> }
> > >>>>>>
> > >>>>>> + /**
> > >>>>>> + * Allows schema to be instantiated from existing elastic
> search
> > >>>>> client.
> > >>>>>> + * This constructor is used in tests.
> > >>>>>> + */
> > >>>>>> + @VisibleForTesting
> > >>>>>> + Elasticsearch2Schema(Client client, String index) {
> > >>>>>> + this.client = Preconditions.checkNotNull(client, "client");
> > >>>>>> + this.index = Preconditions.checkNotNull(index, "index");
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> @Override protected Map<String, Table> getTableMap() {
> > >>>>>> final ImmutableMap.Builder<String, Table> builder =
> > >>>>> ImmutableMap.builder();
> > >>>>>>
> > >>>>>> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
> > >>>>> AbstractSchema
> > >>>>>>
> > >>>>>> final List<DiscoveryNode> nodes =
> > >>>>> ImmutableList.copyOf(transportClient.connectedNodes());
> > >>>>>> if (nodes.isEmpty()) {
> > >>>>>> - throw new RuntimeException("Cannot connect to any
> > elasticsearch
> > >>>>> nodes");
> > >>>>>> + throw new IllegalStateException("Cannot connect to any
> > >>>>> elasticsearch node: "
> > >>>>>> + + transportNodes);
> > >>>>>> }
> > >>>>>>
> > >>>>>> client = transportClient;
> > >>>>>>
> > >>>>>>
> > >>>>>
> > >>>
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > >>>>>>
> > ----------------------------------------------------------------------
> > >>>>>> diff --git
> > >>>>>
> > >>>
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > >>>>>
> > >>>
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > >>>>>> index 636aa5f..2928835 100644
> > >>>>>> ---
> > >>>>>
> > >>>
> >
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > >>>>>> +++
> > >>>>>
> > >>>
> >
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > >>>>>> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
> > >>>>> AbstractElasticsearchTable {
> > >>>>>> this.client = client;
> > >>>>>> }
> > >>>>>>
> > >>>>>> + /**
> > >>>>>> + * ES version 2.x. To access document attributes ES2 uses
> {@code
> > >>>>> _source.foo} syntax.
> > >>>>>> + */
> > >>>>>> + @Override protected String scriptedFieldPrefix() {
> > >>>>>> + return "_source";
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> @Override protected Enumerable<Object> find(String index,
> > List<String>
> > >>>>> ops,
> > >>>>>> - List<Map.Entry<String, Class>> fields) {
> > >>>>>> +
> > List<Map.Entry<String,
> > >>>>> Class>> fields) {
> > >>>>>> final String dbName = index;
> > >>>>>>
> > >>>>>> final String queryString = "{" + Util.toString(ops, "", ", ",
> "") +
> > >>>>> "}";
> > >>>>>>
> > >>>>>>
> > >>>>>
> > >>>
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > >>>>>>
> > ----------------------------------------------------------------------
> > >>>>>> diff --git
> > >>>>>
> > >>>
> >
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > >>>>>
> > >>>
> >
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > >>>>>> new file mode 100644
> > >>>>>> index 0000000..287e094
> > >>>>>> --- /dev/null
> > >>>>>> +++
> > >>>>>
> > >>>
> >
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > >>>>>> @@ -0,0 +1,395 @@
> > >>>>>> +/*
> > >>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or
> > more
> > >>>>>> + * contributor license agreements. See the NOTICE file
> distributed
> > >>> with
> > >>>>>> + * this work for additional information regarding copyright
> > ownership.
> > >>>>>> + * The ASF licenses this file to you under the Apache License,
> > Version
> > >>>>> 2.0
> > >>>>>> + * (the "License"); you may not use this file except in
> compliance
> > >>> with
> > >>>>>> + * the License. You may obtain a copy of the License at
> > >>>>>> + *
> > >>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
> > >>>>>> + *
> > >>>>>> + * Unless required by applicable law or agreed to in writing,
> > software
> > >>>>>> + * distributed under the License is distributed on an "AS IS"
> > BASIS,
> > >>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
> or
> > >>>>> implied.
> > >>>>>> + * See the License for the specific language governing
> permissions
> > and
> > >>>>>> + * limitations under the License.
> > >>>>>> + */
> > >>>>>> +package org.apache.calcite.adapter.elasticsearch2;
> > >>>>>> +
> > >>>>>> +import org.apache.calcite.jdbc.CalciteConnection;
> > >>>>>> +import org.apache.calcite.schema.SchemaPlus;
> > >>>>>> +import org.apache.calcite.schema.impl.ViewTable;
> > >>>>>> +import org.apache.calcite.schema.impl.ViewTableMacro;
> > >>>>>> +import org.apache.calcite.test.CalciteAssert;
> > >>>>>> +import org.apache.calcite.test.ElasticChecker;
> > >>>>>> +
> > >>>>>> +import com.google.common.io.LineProcessor;
> > >>>>>> +import com.google.common.io.Resources;
> > >>>>>> +
> > >>>>>> +import org.elasticsearch.action.bulk.BulkItemResponse;
> > >>>>>> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
> > >>>>>> +import org.elasticsearch.action.bulk.BulkResponse;
> > >>>>>> +import org.elasticsearch.common.xcontent.XContentBuilder;
> > >>>>>> +import org.elasticsearch.common.xcontent.XContentFactory;
> > >>>>>> +
> > >>>>>> +import org.junit.BeforeClass;
> > >>>>>> +import org.junit.ClassRule;
> > >>>>>> +import org.junit.Test;
> > >>>>>> +
> > >>>>>> +import java.io.IOException;
> > >>>>>> +import java.nio.charset.StandardCharsets;
> > >>>>>> +import java.sql.Connection;
> > >>>>>> +import java.sql.DriverManager;
> > >>>>>> +import java.sql.SQLException;
> > >>>>>> +import java.util.Arrays;
> > >>>>>> +import java.util.Collections;
> > >>>>>> +import java.util.Locale;
> > >>>>>> +
> > >>>>>> +/**
> > >>>>>> + * Set of tests for ES adapter. Uses real instance via {@link
> > >>>>> EmbeddedElasticRule}. Document
> > >>>>>> + * source is local {@code zips-mini.json} file (located in the
> > >>>>> classpath).
> > >>>>>> + */
> > >>>>>> +public class ElasticSearch2AdapterTest {
> > >>>>>> +
> > >>>>>> + @ClassRule //init once for all tests
> > >>>>>> + public static final EmbeddedElasticRule NODE =
> > >>>>> EmbeddedElasticRule.create();
> > >>>>>> +
> > >>>>>> + private static final String ZIPS = "zips";
> > >>>>>> +
> > >>>>>> + /**
> > >>>>>> + * Used to create {@code zips} index and insert some data
> > >>>>>> + */
> > >>>>>> + @BeforeClass
> > >>>>>> + public static void setupInstance() throws Exception {
> > >>>>>> + // define mapping so fields are searchable (term query)
> > >>>>>> + XContentBuilder mapping =
> > >>>>> XContentFactory.jsonBuilder().startObject()
> > >>>>>> + .startObject("properties")
> > >>>>>> + .startObject("city").field("type", "string")
> > >>>>>> + .field("index", "not_analyzed").endObject()
> > >>>>>> + .startObject("state").field("type", "string")
> > >>>>>> + .field("index", "not_analyzed").endObject()
> > >>>>>> + .startObject("pop").field("type", "long").endObject()
> > >>>>>> + .endObject()
> > >>>>>> + .endObject();
> > >>>>>> +
> > >>>>>> + // create index
> > >>>>>> + NODE.client().admin().indices()
> > >>>>>> + .prepareCreate(ZIPS)
> > >>>>>> + .addMapping(ZIPS, mapping)
> > >>>>>> + .get();
> > >>>>>> +
> > >>>>>> + BulkRequestBuilder bulk =
> > >>>>> NODE.client().prepareBulk().setRefresh(true);
> > >>>>>> +
> > >>>>>> + // load records from file
> > >>>>>> +
> > >>>>>
> > >>>
> >
> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
> > >>>>>> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
> > >>>>>> + @Override public boolean processLine(String line)
> > throws
> > >>>>> IOException {
> > >>>>>> + line = line.replaceAll("_id", "id"); // _id is a
> > >>>>> reserved attribute in ES
> > >>>>>> + bulk.add(NODE.client().prepareIndex(ZIPS,
> > >>>>> ZIPS).setSource(line));
> > >>>>>> + return true;
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Override public Void getResult() {
> > >>>>>> + return null;
> > >>>>>> + }
> > >>>>>> + });
> > >>>>>> +
> > >>>>>> + if (bulk.numberOfActions() == 0) {
> > >>>>>> + throw new IllegalStateException("No records to be
> indexed");
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + BulkResponse response = bulk.execute().get();
> > >>>>>> +
> > >>>>>> + if (response.hasFailures()) {
> > >>>>>> + throw new IllegalStateException(
> > >>>>>> + String.format(Locale.getDefault(), "Failed to
> > populate
> > >>>>> %s:\n%s", NODE.httpAddress(),
> > >>>>>> +
> > >>>>>
> Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
> > >>>>>> +
> > >>>>>
> > >>>
> >
> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + private CalciteAssert.ConnectionFactory newConnectionFactory()
> {
> > >>>>>> + return new CalciteAssert.ConnectionFactory() {
> > >>>>>> + @Override public Connection createConnection() throws
> > >>>>> SQLException {
> > >>>>>> + final Connection connection =
> > >>>>> DriverManager.getConnection("jdbc:calcite:");
> > >>>>>> + final SchemaPlus root =
> > >>>>> connection.unwrap(CalciteConnection.class).getRootSchema();
> > >>>>>> +
> > >>>>>> + root.add("elastic", new
> Elasticsearch2Schema(NODE.client(),
> > >>>>> ZIPS));
> > >>>>>> +
> > >>>>>> + // add calcite view programmatically
> > >>>>>> + final String viewSql = "select cast(_MAP['city'] AS
> > >>>>> varchar(20)) AS \"city\", "
> > >>>>>> + + " cast(_MAP['loc'][0] AS float) AS
> > \"longitude\",\n"
> > >>>>>> + + " cast(_MAP['loc'][1] AS float) AS
> > \"latitude\",\n"
> > >>>>>> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
> > >>>>>> + + " cast(_MAP['state'] AS varchar(2)) AS
> \"state\",
> > "
> > >>>>>> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
> > >>>>>> + + "from \"elastic\".\"zips\"";
> > >>>>>> +
> > >>>>>> + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
> > >>>>>> + Collections.singletonList("elastic"),
> > >>>>> Arrays.asList("elastic", "view"), false);
> > >>>>>> + root.add("ZIPS", macro);
> > >>>>>> +
> > >>>>>> + return connection;
> > >>>>>> + }
> > >>>>>> + };
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + private CalciteAssert.AssertThat calciteAssert() {
> > >>>>>> + return CalciteAssert.that()
> > >>>>>> + .with(newConnectionFactory());
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + /**
> > >>>>>> + * Tests using calcite view
> > >>>>>> + */
> > >>>>>> + @Test
> > >>>>>> + public void view() throws Exception {
> > >>>>>> + calciteAssert()
> > >>>>>> + .query("select * from zips where \"city\" =
> 'BROOKLYN'")
> > >>>>>> + .returns("city=BROOKLYN; longitude=-73.956985;
> > >>>>> latitude=40.646694; "
> > >>>>>> + + "pop=111396; state=NY; id=11226\n")
> > >>>>>> + .returnsCount(1);
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test
> > >>>>>> + public void emptyResult() {
> > >>>>>> + CalciteAssert.that()
> > >>>>>> + .with(newConnectionFactory())
> > >>>>>> + .query("select * from zips limit 0")
> > >>>>>> + .returnsCount(0);
> > >>>>>> +
> > >>>>>> + CalciteAssert.that()
> > >>>>>> + .with(newConnectionFactory())
> > >>>>>> + .query("select * from \"elastic\".\"zips\" where
> > >>>>> _MAP['Foo'] = '_MISSING_'")
> > >>>>>> + .returnsCount(0);
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test
> > >>>>>> + public void basic() throws Exception {
> > >>>>>> + CalciteAssert.that()
> > >>>>>> + .with(newConnectionFactory())
> > >>>>>> + .query("select * from \"elastic\".\"zips\" where
> > >>>>> _MAP['city'] = 'BROOKLYN'")
> > >>>>>> + .returnsCount(1);
> > >>>>>> +
> > >>>>>> + CalciteAssert.that()
> > >>>>>> + .with(newConnectionFactory())
> > >>>>>> + .query("select * from \"elastic\".\"zips\" where"
> > >>>>>> + + " _MAP['city'] in ('BROOKLYN',
> > 'WASHINGTON')")
> > >>>>>> + .returnsCount(2);
> > >>>>>> +
> > >>>>>> + // lower-case
> > >>>>>> + CalciteAssert.that()
> > >>>>>> + .with(newConnectionFactory())
> > >>>>>> + .query("select * from \"elastic\".\"zips\" where "
> > >>>>>> + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
> > >>>>> 'BROOK') ")
> > >>>>>> + .returnsCount(0);
> > >>>>>> +
> > >>>>>> + // missing field
> > >>>>>> + CalciteAssert.that()
> > >>>>>> + .with(newConnectionFactory())
> > >>>>>> + .query("select * from \"elastic\".\"zips\" where
> > >>>>> _MAP['CITY'] = 'BROOKLYN'")
> > >>>>>> + .returnsCount(0);
> > >>>>>> +
> > >>>>>> + // limit works
> > >>>>>> + CalciteAssert.that()
> > >>>>>> + .with(newConnectionFactory())
> > >>>>>> + .query("select * from \"elastic\".\"zips\" limit 42")
> > >>>>>> + .returnsCount(42);
> > >>>>>> +
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testSort() {
> > >>>>>> + final String explain =
> > "PLAN=ElasticsearchToEnumerableConverter\n"
> > >>>>>> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> > >>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> > >>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> > >>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0,
> 'loc'),
> > >>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> > >>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> > >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> > >>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> > >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> > \"ISO-8859-1$en_US$primary\"])\n"
> > >>>>>> + + " ElasticsearchTableScan(table=[[elastic,
> > zips]])";
> > >>>>>> +
> > >>>>>> + calciteAssert()
> > >>>>>> + .query("select * from zips order by \"state\"")
> > >>>>>> + .returnsCount(10)
> > >>>>>> + .explainContains(explain);
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testSortLimit() {
> > >>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> > >>>>>> + + "order by \"state\", \"id\" offset 2 rows fetch
> next
> > 3
> > >>>>> rows only";
> > >>>>>> + calciteAssert()
> > >>>>>> + .query(sql)
> > >>>>>> + .returnsUnordered("state=AK; id=99801",
> > >>>>>> + "state=AL; id=35215",
> > >>>>>> + "state=AL; id=35401")
> > >>>>>> + .queryContains(
> > >>>>>> + ElasticChecker.elasticsearchChecker(
> > >>>>>> + "\"_source\" : [\"state\", \"id\"]",
> > >>>>>> + "\"sort\": [ {\"state\": \"asc\"},
> > >>> {\"id\":
> > >>>>> \"asc\"}]",
> > >>>>>> + "\"from\": 2",
> > >>>>>> + "\"size\": 3"));
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> +
> > >>>>>> +
> > >>>>>> + @Test public void testOffsetLimit() {
> > >>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> > >>>>>> + + "offset 2 fetch next 3 rows only";
> > >>>>>> + calciteAssert()
> > >>>>>> + .query(sql)
> > >>>>>> + .runs()
> > >>>>>> + .queryContains(
> > >>>>>> + ElasticChecker.elasticsearchChecker(
> > >>>>>> + "\"from\": 2",
> > >>>>>> + "\"size\": 3",
> > >>>>>> + "\"_source\" : [\"state\",
> \"id\"]"));
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testLimit() {
> > >>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> > >>>>>> + + "fetch next 3 rows only";
> > >>>>>> +
> > >>>>>> + calciteAssert()
> > >>>>>> + .query(sql)
> > >>>>>> + .runs()
> > >>>>>> + .queryContains(
> > >>>>>> + ElasticChecker.elasticsearchChecker(
> > >>>>>> + "\"size\": 3",
> > >>>>>> + "\"_source\" : [\"state\",
> \"id\"]"));
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testFilterSort() {
> > >>>>>> + final String sql = "select * from zips\n"
> > >>>>>> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
> > >>>>>> + + "order by \"state\", \"id\"";
> > >>>>>> + final String explain =
> > "PLAN=ElasticsearchToEnumerableConverter\n"
> > >>>>>> + + " ElasticsearchSort(sort0=[$4], sort1=[$5],
> > dir0=[ASC],
> > >>>>> dir1=[ASC])\n"
> > >>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> > >>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> > >>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0,
> 'loc'),
> > >>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> > >>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> > >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> > >>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> > >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> > \"ISO-8859-1$en_US$primary\"])\n"
> > >>>>>> + + "
> > >>> ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
> > >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> > >>>>> \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0,
> > 'id')):VARCHAR(5)
> > >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\",
> > >>>>> '70000'))])\n"
> > >>>>>> + + " ElasticsearchTableScan(table=[[elastic,
> > >>> zips]])";
> > >>>>>> + calciteAssert()
> > >>>>>> + .query(sql)
> > >>>>>> + .returnsOrdered("city=LOS ANGELES;
> > longitude=-118.258189;
> > >>>>> latitude=34.007856; "
> > >>>>>> + + "pop=96074; state=CA; id=90011",
> > >>>>>> + "city=BELL GARDENS; longitude=-118.17205;
> > >>>>> latitude=33.969177; "
> > >>>>>> + + "pop=99568; state=CA; id=90201",
> > >>>>>> + "city=NORWALK; longitude=-118.081767;
> > >>>>> latitude=33.90564; "
> > >>>>>> + + "pop=94188; state=CA; id=90650")
> > >>>>>> + .queryContains(
> > >>>>>> +
> ElasticChecker.elasticsearchChecker("\"query\"
> > : "
> > >>>>>> + +
> > >>>>> "{\"constant_score\":{\"filter\":{\"bool\":"
> > >>>>>> + +
> > >>>>> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
> > >>>>>> + +
> > >>>>> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
> > >>>>>> + "\"script_fields\":
> > >>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> > >>>>>> + +
> > >>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> > >>>>>> + + "\"city\":{\"script\":
> > >>>>> \"_source.city\"}, "
> > >>>>>> + + "\"pop\":{\"script\":
> > >>>>> \"_source.pop\"}, "
> > >>>>>> + + "\"state\":{\"script\":
> > >>>>> \"_source.state\"}, "
> > >>>>>> + + "\"id\":{\"script\": \"_
> > >>> source.id
> > >>>>> \"}}",
> > >>>>>> + "\"sort\": [ {\"state\": \"asc\"},
> > >>> {\"id\":
> > >>>>> \"asc\"}]"))
> > >>>>>> + .explainContains(explain);
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testFilterSortDesc() {
> > >>>>>> + final String sql = "select * from zips\n"
> > >>>>>> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
> > >>>>>> + + "order by \"state\" desc, \"pop\"";
> > >>>>>> + calciteAssert()
> > >>>>>> + .query(sql)
> > >>>>>> + .limit(4)
> > >>>>>> + .returnsOrdered(
> > >>>>>> + "city=LOS ANGELES; longitude=-118.258189;
> > >>>>> latitude=34.007856; pop=96074; state=CA; id=90011",
> > >>>>>> + "city=BELL GARDENS; longitude=-118.17205;
> > >>>>> latitude=33.969177; pop=99568; state=CA; id=90201");
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testFilterRedundant() {
> > >>>>>> + final String sql = "select * from zips\n"
> > >>>>>> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and
> > >>>>> \"state\" = 'OK'";
> > >>>>>> + calciteAssert()
> > >>>>>> + .query(sql)
> > >>>>>> + .runs()
> > >>>>>> + .queryContains(
> > >>>>>> + ElasticChecker.elasticsearchChecker(""
> > >>>>>> + + "\"query\" :
> > >>>>> {\"constant_score\":{\"filter\":{\"bool\":"
> > >>>>>> + +
> > >>>>> "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
> > >>>>>> + "\"script_fields\":
> > >>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> > >>>>>> + +
> > >>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> > >>>>>> + + "\"city\":{\"script\":
> > >>>>> \"_source.city\"}, "
> > >>>>>> + + "\"pop\":{\"script\":
> > >>>>> \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
> > >>>>>> + +
> > "\"id\":{\"script\":
> > >>>>> \"_source.id\"}}"
> > >>>>>> + ));
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testInPlan() {
> > >>>>>> + final String[] searches = {
> > >>>>>> + "\"query\" :
> > >>>>> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
> > >>>>>> + +
> > >>>>>
> > >>>
> >
> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
> > >>>>>> + + "{\"pop\":99568}}]}}]}}}}",
> > >>>>>> + "\"script_fields\":
> > >>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> > >>>>>> + +
> "\"latitude\":{\"script\":\"_source.loc[1]\"},
> > "
> > >>>>>> + + "\"city\":{\"script\": \"_source.city\"}, "
> > >>>>>> + + "\"pop\":{\"script\": \"_source.pop\"}, "
> > >>>>>> + + "\"state\":{\"script\": \"_source.state\"}, "
> > >>>>>> + + "\"id\":{\"script\": \"_source.id\"}}"
> > >>>>>> + };
> > >>>>>> +
> > >>>>>> + calciteAssert()
> > >>>>>> + .query("select * from zips where \"pop\" in (96074,
> > >>> 99568)")
> > >>>>>> + .returnsUnordered(
> > >>>>>> + "city=BELL GARDENS; longitude=-118.17205;
> > >>>>> latitude=33.969177; pop=99568; state=CA; id=90201",
> > >>>>>> + "city=LOS ANGELES; longitude=-118.258189;
> > >>>>> latitude=34.007856; pop=96074; state=CA; id=90011"
> > >>>>>> + )
> > >>>>>> +
> > >>>>> .queryContains(ElasticChecker.elasticsearchChecker(searches));
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testZips() {
> > >>>>>> + calciteAssert()
> > >>>>>> + .query("select \"state\", \"city\" from zips")
> > >>>>>> + .returnsCount(10);
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testProject() {
> > >>>>>> + final String sql = "select \"state\", \"city\", 0 as
> > \"zero\"\n"
> > >>>>>> + + "from zips\n"
> > >>>>>> + + "order by \"state\", \"city\"";
> > >>>>>> +
> > >>>>>> + calciteAssert()
> > >>>>>> + .query(sql)
> > >>>>>> + .limit(2)
> > >>>>>> + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0",
> > >>>>>> + "state=AK; city=FAIRBANKS; zero=0")
> > >>>>>> + .queryContains(
> > >>>>>> +
> > >>>>> ElasticChecker.elasticsearchChecker("\"script_fields\": "
> > >>>>>> + + "{\"zero\":{\"script\":
> > \"0\"},
> > >>> "
> > >>>>>> + + "\"state\":{\"script\":
> > >>>>> \"_source.state\"}, "
> > >>>>>> + + "\"city\":{\"script\":
> > >>>>> \"_source.city\"}}",
> > >>>>>> + "\"sort\": [ {\"state\": \"asc\"},
> > >>>>> {\"city\": \"asc\"}]"));
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testFilter() {
> > >>>>>> + final String explain =
> > "PLAN=ElasticsearchToEnumerableConverter\n"
> > >>>>>> + + " ElasticsearchProject(state=[CAST(ITEM($0,
> > >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> > >>>>> \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0,
> > 'city')):VARCHAR(20)
> > >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> > \"ISO-8859-1$en_US$primary\"])\n"
> > >>>>>> + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0,
> > >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> > >>>>> \"ISO-8859-1$en_US$primary\", 'CA')])\n"
> > >>>>>> + + " ElasticsearchTableScan(table=[[elastic,
> > zips]])";
> > >>>>>> + calciteAssert()
> > >>>>>> + .query("select \"state\", \"city\" from zips where
> > >>>>> \"state\" = 'CA'")
> > >>>>>> + .limit(3)
> > >>>>>> + .returnsUnordered("state=CA; city=BELL GARDENS",
> > >>>>>> + "state=CA; city=LOS ANGELES",
> > >>>>>> + "state=CA; city=NORWALK")
> > >>>>>> + .explainContains(explain);
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> + @Test public void testFilterReversed() {
> > >>>>>> + calciteAssert()
> > >>>>>> + .query("select \"state\", \"city\" from zips where
> 'WI' <
> > >>>>> \"state\" order by \"city\"")
> > >>>>>> + .limit(2)
> > >>>>>> + .returnsUnordered("state=WV; city=BECKLEY",
> > >>>>>> + "state=WY; city=CHEYENNE");
> > >>>>>> + calciteAssert()
> > >>>>>> + .query("select \"state\", \"city\" from zips where
> > \"state\"
> > >>>>>> 'WI' order by \"city\"")
> > >>>>>> + .limit(2)
> > >>>>>> + .returnsUnordered("state=WV; city=BECKLEY",
> > >>>>>> + "state=WY; city=CHEYENNE");
> > >>>>>> + }
> > >>>>>> +
> > >>>>>> +}
> > >>>>>> +
> > >>>>>> +// End ElasticSearch2AdapterTest.java
> > >>>>>>
> > >>>>>>
> > >>>>>
> > >>>
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> > >>>>>>
> > ----------------------------------------------------------------------
> > >>>>>> diff --git
> > >>>>>
> > >>>
> >
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> > >>>>>
> > >>>
> >
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> > >>>>>> new file mode 100644
> > >>>>>> index 0000000..4474add
> > >>>>>> --- /dev/null
> > >>>>>> +++
> > >>>>>
> > >>>
> >
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> > >>>>>> @@ -0,0 +1,147 @@
> > >>>>>> +/*
> > >>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or
> > more
> > >>>>>> + * contributor license agreements. See the NOTICE file
> distributed
> > >>> with
> > >>>>>> + * this work for additional information regarding copyright
> > ownership.
> > >>>>>> + * The ASF licenses this file to you under the Apache License,
> > Version
> > >>>>> 2.0
> > >>>>>> + * (the "License"); you may not use this file except in
> compliance
> > >>> with
> > >>>>>> + * the License. You may obtain a copy of the License at
> > >>>>>> + *
> > >>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
> > >>>>>> + *
> > >>>>>> + * Unless required by applicable law or agreed to in writing,
> > software
> > >>>>>> + * distributed under the License is distributed on an "AS IS"
> > BASIS,
> > >>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
> or
> > >>>>> implied.
> > >>>>>> + * See the License for the specific language governing
> permissions
> > and
> > >>>>>> + * limitations under the License.
> > >>>>>> + */
> > >>>>>> +package org.apache.calcite.adapter.elasticsearch2;
> > >>>>>> +
> > >>>>>> +import com.google.common.base.Preconditions;
> > >>>>>>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Volodymyr Vysotskyi <vo...@apache.org>.
I have also checked "mvn javadoc:javadoc javadoc:test-javadoc" on my
machine (Ubuntu, JDK 8, Maven 3.3.9) and it fails on elasticsearch2 module
(didn't check this problem for the next modules).
"mvn clean -DskipTests site" fails for MongoDB module.
Kind regards,
Volodymyr Vysotskyi
сб, 23 черв. 2018 о 03:12 Julian Hyde <jh...@apache.org> пише:
> Same failure for me on JDK 8/macOS. The most likely other variable is
> maven: I have 3.5.2 on macOS, 3.5.3 on ubuntu.
>
> I’m mystified why no one else is seeing javadoc failures on macOS.
>
> Julian
>
>
> > On Jun 22, 2018, at 4:35 PM, Julian Hyde <jh...@apache.org> wrote:
> >
> > On JDK 8/ubuntu, “mvn clean -DskipTests site” gives the following
> failure:
> >
> > [ERROR] Failed to execute goal
> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> project calcite-mongodb: Error generating
> maven-javadoc-plugin:3.0.1:test-javadoc report:
> > [ERROR] Exit code: 1 -
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:22:
> error: cannot find symbol
> > [ERROR] import org.apache.calcite.test.CalciteAssert;
> > [ERROR] ^
> > [ERROR] symbol: class CalciteAssert
> > [ERROR] location: package org.apache.calcite.test
> > [ERROR]
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:137:
> error: package CalciteAssert does not exist
> > [ERROR] private CalciteAssert.AssertThat assertModel(String model) {
> > [ERROR] ^
> > [ERROR]
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:145:
> error: package CalciteAssert does not exist
> > [ERROR] private CalciteAssert.AssertThat assertModel(URL url) {
> > [ERROR] ^
> > [ERROR]
> /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:43:
> error: reference not found
> > [ERROR] /** Similar to {@link CalciteAssert#checkResultUnordered}, but
> filters strings
> > [ERROR] ^
> >
> >
> >
> >> On Jun 22, 2018, at 3:46 PM, Andrei Sereda <an...@sereda.cc> wrote:
> >>
> >> Hmm. Both master and c12cb4b0de work for me. Perhaps the issue is JDK 8
> vs
> >> 9 ?
> >>
> >> # JDK8 (oracle) / macOS
> >> $ mvn clean -DskipTests site
> >>
> >> On Fri, Jun 22, 2018 at 5:56 PM Julian Hyde <jh...@apache.org> wrote:
> >>
> >>> I just tried it again. The following fails for me (jdk9, ubuntu):
> >>>
> >>> $ git checkout c12cb4b0de
> >>> $ mvn clean -DskipTests site
> >>>
> >>> Constructing Javadoc information...
> >>> 1 error
> >>> [INFO]
> >>>
> ------------------------------------------------------------------------
> >>> [INFO] Reactor Summary:
> >>> [INFO]
> >>> [INFO] Calcite 1.17.0-SNAPSHOT ............................ FAILURE
> [02:29
> >>> min]
> >>> [INFO] Calcite Linq4j ..................................... SKIPPED
> >>> [INFO] Calcite Core ....................................... SKIPPED
> >>> [INFO] Calcite Cassandra .................................. SKIPPED
> >>> [INFO] Calcite Druid ...................................... SKIPPED
> >>> [INFO] Calcite Elasticsearch .............................. SKIPPED
> >>> [INFO] Calcite Elasticsearch5 ............................. SKIPPED
> >>> [INFO] Calcite Examples ................................... SKIPPED
> >>> [INFO] Calcite Example CSV ................................ SKIPPED
> >>> [INFO] Calcite Example Function ........................... SKIPPED
> >>> [INFO] Calcite File ....................................... SKIPPED
> >>> [INFO] Calcite Geode ...................................... SKIPPED
> >>> [INFO] Calcite MongoDB .................................... SKIPPED
> >>> [INFO] Calcite Pig ........................................ SKIPPED
> >>> [INFO] Calcite Piglet ..................................... SKIPPED
> >>> [INFO] Calcite Plus ....................................... SKIPPED
> >>> [INFO] Calcite Server ..................................... SKIPPED
> >>> [INFO] Calcite Spark ...................................... SKIPPED
> >>> [INFO] Calcite Splunk ..................................... SKIPPED
> >>> [INFO] Calcite Ubenchmark 1.17.0-SNAPSHOT ................. SKIPPED
> >>> [INFO]
> >>>
> ------------------------------------------------------------------------
> >>> [INFO] BUILD FAILURE
> >>> [INFO]
> >>>
> ------------------------------------------------------------------------
> >>> [INFO] Total time: 02:29 min
> >>> [INFO] Finished at: 2018-06-22T14:53:13-07:00
> >>> [INFO]
> >>>
> ------------------------------------------------------------------------
> >>> [ERROR] Failed to execute goal
> >>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> >>> project calcite: Error generating
> maven-javadoc-plugin:3.0.1:test-aggregate
> >>> report:
> >>> [ERROR] Exit code: 1 -
> >>>
> /home/jhyde/open1/calcite.4/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> >>> error: package org.elasticsearch.node.internal does not exist
> >>> [ERROR] import
> org.elasticsearch.node.internal.InternalSettingsPreparer;
> >>> [ERROR] ^
> >>> [ERROR]
> >>> [ERROR] Command line was: /usr/lib/jvm/jdk9/bin/javadoc @options
> @packages
> >>> @argfile
> >>> [ERROR]
> >>> [ERROR] Refer to the generated Javadoc files in
> >>> '/home/jhyde/open1/calcite.4/target/site/testapidocs' dir.
> >>> [ERROR] -> [Help 1]
> >>> [ERROR]
> >>> [ERROR] To see the full stack trace of the errors, re-run Maven with
> the
> >>> -e switch.
> >>> [ERROR] Re-run Maven using the -X switch to enable full debug logging.
> >>> [ERROR]
> >>> [ERROR] For more information about the errors and possible solutions,
> >>> please read the following articles:
> >>> [ERROR] [Help 1]
> >>>
> http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException
> >>>
> >>>
> >>>> On Jun 22, 2018, at 2:41 PM, Michael Mior <mm...@apache.org> wrote:
> >>>>
> >>>> Odd, mvn clean site still works fine for me.
> >>>> --
> >>>> Michael Mior
> >>>> mmior@apache.org
> >>>>
> >>>>
> >>>>
> >>>> Le ven. 22 juin 2018 à 17:12, Julian Hyde <jh...@apache.org> a écrit
> :
> >>>>
> >>>>> Looks like this change broke “mvn site” (perhaps also “mvn
> >>>>> javadoc:test-javadoc”).
> >>>>>
> >>>>> [ERROR] Failed to execute goal
> >>>>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> >>>>> project calcite: Error generating
> >>> maven-javadoc-plugin:3.0.1:test-aggregate
> >>>>> report:
> >>>>> [ERROR] Exit code: 1 -
> >>>>>
> >>>
> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> >>>>> error: package org.elasticsearch.node.internal does not exist
> >>>>> [ERROR] import
> org.elasticsearch.node.internal.InternalSettingsPreparer;
> >>>>> [ERROR] ^
> >>>>> [ERROR]
> >>>>>
> >>>>>> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
> >>>>>>
> >>>>>> [CALCITE-2347] running ElasticSearch in embedded mode for unit
> tests of
> >>>>> ES adapter (Andrei Sereda)
> >>>>>>
> >>>>>> After discussion on dev-list Integration tests (for ES) have been
> >>>>> removed. They're now
> >>>>>> superseded by unit tests (which execute queries against a real
> elastic
> >>>>> instance)
> >>>>>>
> >>>>>> Added local file (zips-mini.json) which contains a small subset of
> >>>>> original zips.json
> >>>>>> (allows to bootstrap tests faster)
> >>>>>>
> >>>>>> Created separate ES JUnit rule which can be re-used across different
> >>>>> tests.
> >>>>>>
> >>>>>> Both v2 and v5 of ES adapters are supported.
> >>>>>>
> >>>>>> Close apache/calcite#716
> >>>>>>
> >>>>>>
> >>>>>> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
> >>>>>> Commit:
> http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
> >>>>>> Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
> >>>>>> Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
> >>>>>>
> >>>>>> Branch: refs/heads/master
> >>>>>> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
> >>>>>> Parents: 37944bb
> >>>>>> Author: Andrei Sereda <an...@nospam.com>
> >>>>>> Authored: Thu May 31 18:19:10 2018 -0400
> >>>>>> Committer: Michael Mior <mm...@uwaterloo.ca>
> >>>>>> Committed: Thu Jun 21 06:38:50 2018 -0400
> >>>>>>
> >>>>>>
> ----------------------------------------------------------------------
> >>>>>> .../AbstractElasticsearchTable.java | 12 +
> >>>>>> .../elasticsearch/ElasticsearchProject.java | 61 ++-
> >>>>>> elasticsearch2/pom.xml | 6 +
> >>>>>> .../Elasticsearch2Enumerator.java | 12 +-
> >>>>>> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
> >>>>>> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
> >>>>>> .../ElasticSearch2AdapterTest.java | 395
> >>> ++++++++++++++++++
> >>>>>> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
> >>>>>> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
> >>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> >>>>>> .../calcite/test/Elasticsearch2AdapterIT.java | 270 -------------
> >>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
> >>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
> >>>>>> elasticsearch5/pom.xml | 31 ++
> >>>>>> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
> >>>>>> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
> >>>>>> .../ElasticSearch5AdapterTest.java | 399
> >>> +++++++++++++++++++
> >>>>>> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
> >>>>>> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
> >>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> >>>>>> .../calcite/test/Elasticsearch5AdapterIT.java | 270 -------------
> >>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
> >>>>>> elasticsearch5/src/test/resources/log4j2.xml | 16 +
> >>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
> >>>>>> pom.xml | 20 +-
> >>>>>> 25 files changed, 1866 insertions(+), 670 deletions(-)
> >>>>>>
> ----------------------------------------------------------------------
> >>>>>>
> >>>>>>
> >>>>>>
> >>>>>
> >>>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>>>>>
> ----------------------------------------------------------------------
> >>>>>> diff --git
> >>>>>
> >>>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>>>>
> >>>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>>>>> index 0980469..8cc5933 100644
> >>>>>> ---
> >>>>>
> >>>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>>>>> +++
> >>>>>
> >>>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>>>>> @@ -75,6 +75,18 @@ public abstract class AbstractElasticsearchTable
> >>>>> extends AbstractQueryableTable
> >>>>>> relOptTable, this, null);
> >>>>>> }
> >>>>>>
> >>>>>> + /**
> >>>>>> + * In ES 5.x scripted fields start with {@code
> params._source.foo}
> >>>>> while in ES2.x
> >>>>>> + * {@code _source.foo}. Helper method to build correct query
> based
> >>> on
> >>>>> runtime version of elastic.
> >>>>>> + *
> >>>>>> + * @see <a href="
> >>>>> https://github.com/elastic/elasticsearch/issues/20068">_source
> >>>>> variable</a>
> >>>>>> + * @see <a href="
> >>>>>
> >>>
> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html
> >>> ">Scripted
> >>>>> Fields</a>
> >>>>>> + */
> >>>>>> + protected String scriptedFieldPrefix() {
> >>>>>> + // this is default pattern starting 5.x
> >>>>>> + return "params._source";
> >>>>>> + }
> >>>>>> +
> >>>>>> /** Executes a "find" operation on the underlying type.
> >>>>>> *
> >>>>>> * <p>For example,
> >>>>>>
> >>>>>>
> >>>>>
> >>>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>>>>>
> ----------------------------------------------------------------------
> >>>>>> diff --git
> >>>>>
> >>>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>>>>
> >>>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>>>>> index b42abd7..961c8b0 100644
> >>>>>> ---
> >>>>>
> >>>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>>>>> +++
> >>>>>
> >>>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>>>>> @@ -27,11 +27,15 @@ import
> >>>>> org.apache.calcite.rel.metadata.RelMetadataQuery;
> >>>>>> import org.apache.calcite.rel.type.RelDataType;
> >>>>>> import org.apache.calcite.rex.RexNode;
> >>>>>> import org.apache.calcite.util.Pair;
> >>>>>> -import org.apache.calcite.util.Util;
> >>>>>> +
> >>>>>> +import com.google.common.base.Function;
> >>>>>> +import com.google.common.collect.Lists;
> >>>>>>
> >>>>>> import java.util.ArrayList;
> >>>>>> import java.util.List;
> >>>>>>
> >>>>>> +import javax.annotation.Nullable;
> >>>>>> +
> >>>>>> /**
> >>>>>> * Implementation of {@link org.apache.calcite.rel.core.Project}
> >>>>>> * relational expression in Elasticsearch.
> >>>>>> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends
> Project
> >>>>> implements ElasticsearchRel {
> >>>>>> implementor.visitChild(0, getInput());
> >>>>>>
> >>>>>> final List<String> inFields =
> >>>>>> -
> >>>>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> >>>>>> +
> >>>>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> >>>>>> final ElasticsearchRules.RexToElasticsearchTranslator translator =
> >>>>>> - new ElasticsearchRules.RexToElasticsearchTranslator(
> >>>>>> - (JavaTypeFactory) getCluster().getTypeFactory(),
> >>> inFields);
> >>>>>> + new ElasticsearchRules.RexToElasticsearchTranslator(
> >>>>>> + (JavaTypeFactory)
> getCluster().getTypeFactory(),
> >>>>> inFields);
> >>>>>>
> >>>>>> - final List<String> findItems = new ArrayList<>();
> >>>>>> - final List<String> scriptFieldItems = new ArrayList<>();
> >>>>>> + final List<String> fields = new ArrayList<>();
> >>>>>> + final List<String> scriptFields = new ArrayList<>();
> >>>>>> for (Pair<RexNode, String> pair: getNamedProjects()) {
> >>>>>> final String name = pair.right;
> >>>>>> final String expr = pair.left.accept(translator);
> >>>>>>
> >>>>>> if (expr.equals("\"" + name + "\"")) {
> >>>>>> - findItems.add(ElasticsearchRules.quote(name));
> >>>>>> + fields.add(name);
> >>>>>> } else if (expr.matches("\"literal\":.+")) {
> >>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> >>>>>> - + ":{\"script\": "
> >>>>>> - + expr.split(":")[1] + "}");
> >>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
> >>>>>> + + ":{\"script\": "
> >>>>>> + + expr.split(":")[1] + "}");
> >>>>>> } else {
> >>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> >>>>>> - + ":{\"script\":\"params._source."
> >>>>>> - + expr.replaceAll("\"", "") + "\"}");
> >>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
> >>>>>> + + ":{\"script\":"
> >>>>>> + // _source (ES2) vs params._source (ES5)
> >>>>>> + + "\"" +
> >>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> >>>>>> + + expr.replaceAll("\"", "") + "\"}");
> >>>>>> + }
> >>>>>> + }
> >>>>>> +
> >>>>>> + StringBuilder query = new StringBuilder();
> >>>>>> + if (scriptFields.isEmpty()) {
> >>>>>> + List<String> newList = Lists.transform(fields, new
> >>>>> Function<String, String>() {
> >>>>>> + @Nullable
> >>>>>> + @Override public String apply(@Nullable String input) {
> >>>>>> + return ElasticsearchRules.quote(input);
> >>>>>> + }
> >>>>>> + });
> >>>>>> +
> >>>>>> + final String findString = String.join(", ", newList);
> >>>>>> + query.append("\"_source\" :
> [").append(findString).append("]");
> >>>>>> + } else {
> >>>>>> + // if scripted fields are present, ES ignores _source
> attribute
> >>>>>> + for (String field: fields) {
> >>>>>> + scriptFields.add(ElasticsearchRules.quote(field) +
> >>>>> ":{\"script\": "
> >>>>>> + // _source (ES2) vs params._source (ES5)
> >>>>>> + + "\"" +
> >>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> >>>>>> + + field + "\"}");
> >>>>>> }
> >>>>>> + query.append("\"script_fields\": {" + String.join(", ",
> >>>>> scriptFields) + "}");
> >>>>>> }
> >>>>>> - final String findString = Util.toString(findItems, "", ", ",
> "");
> >>>>>> - final String scriptFieldString = "\"script_fields\": {"
> >>>>>> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
> >>>>>> - final String fieldString = "\"_source\" : [" + findString + "]"
> >>>>>> - + ", " + scriptFieldString;
> >>>>>>
> >>>>>> for (String opfield : implementor.list) {
> >>>>>> if (opfield.startsWith("\"_source\"")) {
> >>>>>> implementor.list.remove(opfield);
> >>>>>> }
> >>>>>> }
> >>>>>> - implementor.add(fieldString);
> >>>>>> + implementor.add(query.toString());
> >>>>>> }
> >>>>>> }
> >>>>>>
> >>>>>>
> >>>>>>
> >>>>>
> >>>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
> >>>>>>
> ----------------------------------------------------------------------
> >>>>>> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
> >>>>>> index f24622c..6fbee03 100644
> >>>>>> --- a/elasticsearch2/pom.xml
> >>>>>> +++ b/elasticsearch2/pom.xml
> >>>>>> @@ -73,6 +73,12 @@ limitations under the License.
> >>>>>> <version>${elasticsearch-java-driver.version}</version>
> >>>>>> </dependency>
> >>>>>> <dependency>
> >>>>>> + <!-- Lang groovy dependency is needed for testing with
> embedded
> >>>>> ES (scripted fields like loc[0]) -->
> >>>>>> + <groupId>org.elasticsearch.module</groupId>
> >>>>>> + <artifactId>lang-groovy</artifactId>
> >>>>>> + <scope>test</scope>
> >>>>>> + </dependency>
> >>>>>> + <dependency>
> >>>>>> <groupId>com.carrotsearch</groupId>
> >>>>>> <artifactId>hppc</artifactId>
> >>>>>> <version>${hppc.version}</version>
> >>>>>>
> >>>>>>
> >>>>>
> >>>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>>>>>
> ----------------------------------------------------------------------
> >>>>>> diff --git
> >>>>>
> >>>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>>>>
> >>>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>>>>> index 84370ab..c3d2ac0 100644
> >>>>>> ---
> >>>>>
> >>>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>>>>> +++
> >>>>>
> >>>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>>>>> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
> >>>>>> import java.util.Date;
> >>>>>> import java.util.Iterator;
> >>>>>> import java.util.List;
> >>>>>> +import java.util.Locale;
> >>>>>> import java.util.Map;
> >>>>>>
> >>>>>> /**
> >>>>>> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator
> implements
> >>>>> Enumerator<Object> {
> >>>>>> private static Function1<SearchHit, Object[]> listGetter(
> >>>>>> final List<Map.Entry<String, Class>> fields) {
> >>>>>> return new Function1<SearchHit, Object[]>() {
> >>>>>> - public Object[] apply(SearchHit searchHitFields) {
> >>>>>> + public Object[] apply(SearchHit hit) {
> >>>>>> Object[] objects = new Object[fields.size()];
> >>>>>> for (int i = 0; i < fields.size(); i++) {
> >>>>>> final Map.Entry<String, Class> field = fields.get(i);
> >>>>>> final String name = field.getKey();
> >>>>>> - if (searchHitFields.fields().isEmpty()) {
> >>>>>> - objects[i] =
> >>> convert(searchHitFields.getSource().get(name),
> >>>>> field.getValue());
> >>>>>> + if (hit.fields().isEmpty()) {
> >>>>>> + objects[i] = convert(hit.getSource().get(name),
> >>>>> field.getValue());
> >>>>>> + } else if (hit.fields().containsKey(name)) {
> >>>>>> + objects[i] = convert(hit.field(name).getValue(),
> >>>>> field.getValue());
> >>>>>> } else {
> >>>>>> - objects[i] =
> >>>>> convert(searchHitFields.field(name).getValue(), field.getValue());
> >>>>>> + throw new IllegalStateException(
> >>>>>> + String.format(Locale.getDefault(), "No result
> for
> >>>>> %s", field));
> >>>>>> }
> >>>>>> }
> >>>>>> return objects;
> >>>>>>
> >>>>>>
> >>>>>
> >>>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>>>>>
> ----------------------------------------------------------------------
> >>>>>> diff --git
> >>>>>
> >>>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>>>>
> >>>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>>>>> index 668402b..46e3fc5 100644
> >>>>>> ---
> >>>>>
> >>>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>>>>> +++
> >>>>>
> >>>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>>>>> @@ -22,6 +22,9 @@ import
> org.apache.calcite.schema.impl.AbstractSchema;
> >>>>>>
> >>>>>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
> >>>>>>
> >>>>>> +import com.google.common.annotations.VisibleForTesting;
> >>>>>> +import com.google.common.base.Preconditions;
> >>>>>> +
> >>>>>> import com.google.common.collect.ImmutableList;
> >>>>>> import com.google.common.collect.ImmutableMap;
> >>>>>>
> >>>>>> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
> >>>>> AbstractSchema
> >>>>>> }
> >>>>>> }
> >>>>>>
> >>>>>> + /**
> >>>>>> + * Allows schema to be instantiated from existing elastic search
> >>>>> client.
> >>>>>> + * This constructor is used in tests.
> >>>>>> + */
> >>>>>> + @VisibleForTesting
> >>>>>> + Elasticsearch2Schema(Client client, String index) {
> >>>>>> + this.client = Preconditions.checkNotNull(client, "client");
> >>>>>> + this.index = Preconditions.checkNotNull(index, "index");
> >>>>>> + }
> >>>>>> +
> >>>>>> @Override protected Map<String, Table> getTableMap() {
> >>>>>> final ImmutableMap.Builder<String, Table> builder =
> >>>>> ImmutableMap.builder();
> >>>>>>
> >>>>>> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
> >>>>> AbstractSchema
> >>>>>>
> >>>>>> final List<DiscoveryNode> nodes =
> >>>>> ImmutableList.copyOf(transportClient.connectedNodes());
> >>>>>> if (nodes.isEmpty()) {
> >>>>>> - throw new RuntimeException("Cannot connect to any
> elasticsearch
> >>>>> nodes");
> >>>>>> + throw new IllegalStateException("Cannot connect to any
> >>>>> elasticsearch node: "
> >>>>>> + + transportNodes);
> >>>>>> }
> >>>>>>
> >>>>>> client = transportClient;
> >>>>>>
> >>>>>>
> >>>>>
> >>>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>>>>>
> ----------------------------------------------------------------------
> >>>>>> diff --git
> >>>>>
> >>>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>>>>
> >>>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>>>>> index 636aa5f..2928835 100644
> >>>>>> ---
> >>>>>
> >>>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>>>>> +++
> >>>>>
> >>>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>>>>> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
> >>>>> AbstractElasticsearchTable {
> >>>>>> this.client = client;
> >>>>>> }
> >>>>>>
> >>>>>> + /**
> >>>>>> + * ES version 2.x. To access document attributes ES2 uses {@code
> >>>>> _source.foo} syntax.
> >>>>>> + */
> >>>>>> + @Override protected String scriptedFieldPrefix() {
> >>>>>> + return "_source";
> >>>>>> + }
> >>>>>> +
> >>>>>> @Override protected Enumerable<Object> find(String index,
> List<String>
> >>>>> ops,
> >>>>>> - List<Map.Entry<String, Class>> fields) {
> >>>>>> +
> List<Map.Entry<String,
> >>>>> Class>> fields) {
> >>>>>> final String dbName = index;
> >>>>>>
> >>>>>> final String queryString = "{" + Util.toString(ops, "", ", ", "") +
> >>>>> "}";
> >>>>>>
> >>>>>>
> >>>>>
> >>>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>>>>>
> ----------------------------------------------------------------------
> >>>>>> diff --git
> >>>>>
> >>>
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>>>>
> >>>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>>>>> new file mode 100644
> >>>>>> index 0000000..287e094
> >>>>>> --- /dev/null
> >>>>>> +++
> >>>>>
> >>>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>>>>> @@ -0,0 +1,395 @@
> >>>>>> +/*
> >>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or
> more
> >>>>>> + * contributor license agreements. See the NOTICE file distributed
> >>> with
> >>>>>> + * this work for additional information regarding copyright
> ownership.
> >>>>>> + * The ASF licenses this file to you under the Apache License,
> Version
> >>>>> 2.0
> >>>>>> + * (the "License"); you may not use this file except in compliance
> >>> with
> >>>>>> + * the License. You may obtain a copy of the License at
> >>>>>> + *
> >>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
> >>>>>> + *
> >>>>>> + * Unless required by applicable law or agreed to in writing,
> software
> >>>>>> + * distributed under the License is distributed on an "AS IS"
> BASIS,
> >>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> >>>>> implied.
> >>>>>> + * See the License for the specific language governing permissions
> and
> >>>>>> + * limitations under the License.
> >>>>>> + */
> >>>>>> +package org.apache.calcite.adapter.elasticsearch2;
> >>>>>> +
> >>>>>> +import org.apache.calcite.jdbc.CalciteConnection;
> >>>>>> +import org.apache.calcite.schema.SchemaPlus;
> >>>>>> +import org.apache.calcite.schema.impl.ViewTable;
> >>>>>> +import org.apache.calcite.schema.impl.ViewTableMacro;
> >>>>>> +import org.apache.calcite.test.CalciteAssert;
> >>>>>> +import org.apache.calcite.test.ElasticChecker;
> >>>>>> +
> >>>>>> +import com.google.common.io.LineProcessor;
> >>>>>> +import com.google.common.io.Resources;
> >>>>>> +
> >>>>>> +import org.elasticsearch.action.bulk.BulkItemResponse;
> >>>>>> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
> >>>>>> +import org.elasticsearch.action.bulk.BulkResponse;
> >>>>>> +import org.elasticsearch.common.xcontent.XContentBuilder;
> >>>>>> +import org.elasticsearch.common.xcontent.XContentFactory;
> >>>>>> +
> >>>>>> +import org.junit.BeforeClass;
> >>>>>> +import org.junit.ClassRule;
> >>>>>> +import org.junit.Test;
> >>>>>> +
> >>>>>> +import java.io.IOException;
> >>>>>> +import java.nio.charset.StandardCharsets;
> >>>>>> +import java.sql.Connection;
> >>>>>> +import java.sql.DriverManager;
> >>>>>> +import java.sql.SQLException;
> >>>>>> +import java.util.Arrays;
> >>>>>> +import java.util.Collections;
> >>>>>> +import java.util.Locale;
> >>>>>> +
> >>>>>> +/**
> >>>>>> + * Set of tests for ES adapter. Uses real instance via {@link
> >>>>> EmbeddedElasticRule}. Document
> >>>>>> + * source is local {@code zips-mini.json} file (located in the
> >>>>> classpath).
> >>>>>> + */
> >>>>>> +public class ElasticSearch2AdapterTest {
> >>>>>> +
> >>>>>> + @ClassRule //init once for all tests
> >>>>>> + public static final EmbeddedElasticRule NODE =
> >>>>> EmbeddedElasticRule.create();
> >>>>>> +
> >>>>>> + private static final String ZIPS = "zips";
> >>>>>> +
> >>>>>> + /**
> >>>>>> + * Used to create {@code zips} index and insert some data
> >>>>>> + */
> >>>>>> + @BeforeClass
> >>>>>> + public static void setupInstance() throws Exception {
> >>>>>> + // define mapping so fields are searchable (term query)
> >>>>>> + XContentBuilder mapping =
> >>>>> XContentFactory.jsonBuilder().startObject()
> >>>>>> + .startObject("properties")
> >>>>>> + .startObject("city").field("type", "string")
> >>>>>> + .field("index", "not_analyzed").endObject()
> >>>>>> + .startObject("state").field("type", "string")
> >>>>>> + .field("index", "not_analyzed").endObject()
> >>>>>> + .startObject("pop").field("type", "long").endObject()
> >>>>>> + .endObject()
> >>>>>> + .endObject();
> >>>>>> +
> >>>>>> + // create index
> >>>>>> + NODE.client().admin().indices()
> >>>>>> + .prepareCreate(ZIPS)
> >>>>>> + .addMapping(ZIPS, mapping)
> >>>>>> + .get();
> >>>>>> +
> >>>>>> + BulkRequestBuilder bulk =
> >>>>> NODE.client().prepareBulk().setRefresh(true);
> >>>>>> +
> >>>>>> + // load records from file
> >>>>>> +
> >>>>>
> >>>
> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
> >>>>>> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
> >>>>>> + @Override public boolean processLine(String line)
> throws
> >>>>> IOException {
> >>>>>> + line = line.replaceAll("_id", "id"); // _id is a
> >>>>> reserved attribute in ES
> >>>>>> + bulk.add(NODE.client().prepareIndex(ZIPS,
> >>>>> ZIPS).setSource(line));
> >>>>>> + return true;
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Override public Void getResult() {
> >>>>>> + return null;
> >>>>>> + }
> >>>>>> + });
> >>>>>> +
> >>>>>> + if (bulk.numberOfActions() == 0) {
> >>>>>> + throw new IllegalStateException("No records to be indexed");
> >>>>>> + }
> >>>>>> +
> >>>>>> + BulkResponse response = bulk.execute().get();
> >>>>>> +
> >>>>>> + if (response.hasFailures()) {
> >>>>>> + throw new IllegalStateException(
> >>>>>> + String.format(Locale.getDefault(), "Failed to
> populate
> >>>>> %s:\n%s", NODE.httpAddress(),
> >>>>>> +
> >>>>> Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
> >>>>>> +
> >>>>>
> >>>
> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
> >>>>>> + }
> >>>>>> +
> >>>>>> + }
> >>>>>> +
> >>>>>> + private CalciteAssert.ConnectionFactory newConnectionFactory() {
> >>>>>> + return new CalciteAssert.ConnectionFactory() {
> >>>>>> + @Override public Connection createConnection() throws
> >>>>> SQLException {
> >>>>>> + final Connection connection =
> >>>>> DriverManager.getConnection("jdbc:calcite:");
> >>>>>> + final SchemaPlus root =
> >>>>> connection.unwrap(CalciteConnection.class).getRootSchema();
> >>>>>> +
> >>>>>> + root.add("elastic", new Elasticsearch2Schema(NODE.client(),
> >>>>> ZIPS));
> >>>>>> +
> >>>>>> + // add calcite view programmatically
> >>>>>> + final String viewSql = "select cast(_MAP['city'] AS
> >>>>> varchar(20)) AS \"city\", "
> >>>>>> + + " cast(_MAP['loc'][0] AS float) AS
> \"longitude\",\n"
> >>>>>> + + " cast(_MAP['loc'][1] AS float) AS
> \"latitude\",\n"
> >>>>>> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
> >>>>>> + + " cast(_MAP['state'] AS varchar(2)) AS \"state\",
> "
> >>>>>> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
> >>>>>> + + "from \"elastic\".\"zips\"";
> >>>>>> +
> >>>>>> + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
> >>>>>> + Collections.singletonList("elastic"),
> >>>>> Arrays.asList("elastic", "view"), false);
> >>>>>> + root.add("ZIPS", macro);
> >>>>>> +
> >>>>>> + return connection;
> >>>>>> + }
> >>>>>> + };
> >>>>>> + }
> >>>>>> +
> >>>>>> + private CalciteAssert.AssertThat calciteAssert() {
> >>>>>> + return CalciteAssert.that()
> >>>>>> + .with(newConnectionFactory());
> >>>>>> + }
> >>>>>> +
> >>>>>> + /**
> >>>>>> + * Tests using calcite view
> >>>>>> + */
> >>>>>> + @Test
> >>>>>> + public void view() throws Exception {
> >>>>>> + calciteAssert()
> >>>>>> + .query("select * from zips where \"city\" = 'BROOKLYN'")
> >>>>>> + .returns("city=BROOKLYN; longitude=-73.956985;
> >>>>> latitude=40.646694; "
> >>>>>> + + "pop=111396; state=NY; id=11226\n")
> >>>>>> + .returnsCount(1);
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test
> >>>>>> + public void emptyResult() {
> >>>>>> + CalciteAssert.that()
> >>>>>> + .with(newConnectionFactory())
> >>>>>> + .query("select * from zips limit 0")
> >>>>>> + .returnsCount(0);
> >>>>>> +
> >>>>>> + CalciteAssert.that()
> >>>>>> + .with(newConnectionFactory())
> >>>>>> + .query("select * from \"elastic\".\"zips\" where
> >>>>> _MAP['Foo'] = '_MISSING_'")
> >>>>>> + .returnsCount(0);
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test
> >>>>>> + public void basic() throws Exception {
> >>>>>> + CalciteAssert.that()
> >>>>>> + .with(newConnectionFactory())
> >>>>>> + .query("select * from \"elastic\".\"zips\" where
> >>>>> _MAP['city'] = 'BROOKLYN'")
> >>>>>> + .returnsCount(1);
> >>>>>> +
> >>>>>> + CalciteAssert.that()
> >>>>>> + .with(newConnectionFactory())
> >>>>>> + .query("select * from \"elastic\".\"zips\" where"
> >>>>>> + + " _MAP['city'] in ('BROOKLYN',
> 'WASHINGTON')")
> >>>>>> + .returnsCount(2);
> >>>>>> +
> >>>>>> + // lower-case
> >>>>>> + CalciteAssert.that()
> >>>>>> + .with(newConnectionFactory())
> >>>>>> + .query("select * from \"elastic\".\"zips\" where "
> >>>>>> + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
> >>>>> 'BROOK') ")
> >>>>>> + .returnsCount(0);
> >>>>>> +
> >>>>>> + // missing field
> >>>>>> + CalciteAssert.that()
> >>>>>> + .with(newConnectionFactory())
> >>>>>> + .query("select * from \"elastic\".\"zips\" where
> >>>>> _MAP['CITY'] = 'BROOKLYN'")
> >>>>>> + .returnsCount(0);
> >>>>>> +
> >>>>>> + // limit works
> >>>>>> + CalciteAssert.that()
> >>>>>> + .with(newConnectionFactory())
> >>>>>> + .query("select * from \"elastic\".\"zips\" limit 42")
> >>>>>> + .returnsCount(42);
> >>>>>> +
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testSort() {
> >>>>>> + final String explain =
> "PLAN=ElasticsearchToEnumerableConverter\n"
> >>>>>> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> >>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> >>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> >>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> >>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"])\n"
> >>>>>> + + " ElasticsearchTableScan(table=[[elastic,
> zips]])";
> >>>>>> +
> >>>>>> + calciteAssert()
> >>>>>> + .query("select * from zips order by \"state\"")
> >>>>>> + .returnsCount(10)
> >>>>>> + .explainContains(explain);
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testSortLimit() {
> >>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> >>>>>> + + "order by \"state\", \"id\" offset 2 rows fetch next
> 3
> >>>>> rows only";
> >>>>>> + calciteAssert()
> >>>>>> + .query(sql)
> >>>>>> + .returnsUnordered("state=AK; id=99801",
> >>>>>> + "state=AL; id=35215",
> >>>>>> + "state=AL; id=35401")
> >>>>>> + .queryContains(
> >>>>>> + ElasticChecker.elasticsearchChecker(
> >>>>>> + "\"_source\" : [\"state\", \"id\"]",
> >>>>>> + "\"sort\": [ {\"state\": \"asc\"},
> >>> {\"id\":
> >>>>> \"asc\"}]",
> >>>>>> + "\"from\": 2",
> >>>>>> + "\"size\": 3"));
> >>>>>> + }
> >>>>>> +
> >>>>>> +
> >>>>>> +
> >>>>>> + @Test public void testOffsetLimit() {
> >>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> >>>>>> + + "offset 2 fetch next 3 rows only";
> >>>>>> + calciteAssert()
> >>>>>> + .query(sql)
> >>>>>> + .runs()
> >>>>>> + .queryContains(
> >>>>>> + ElasticChecker.elasticsearchChecker(
> >>>>>> + "\"from\": 2",
> >>>>>> + "\"size\": 3",
> >>>>>> + "\"_source\" : [\"state\", \"id\"]"));
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testLimit() {
> >>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
> >>>>>> + + "fetch next 3 rows only";
> >>>>>> +
> >>>>>> + calciteAssert()
> >>>>>> + .query(sql)
> >>>>>> + .runs()
> >>>>>> + .queryContains(
> >>>>>> + ElasticChecker.elasticsearchChecker(
> >>>>>> + "\"size\": 3",
> >>>>>> + "\"_source\" : [\"state\", \"id\"]"));
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testFilterSort() {
> >>>>>> + final String sql = "select * from zips\n"
> >>>>>> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
> >>>>>> + + "order by \"state\", \"id\"";
> >>>>>> + final String explain =
> "PLAN=ElasticsearchToEnumerableConverter\n"
> >>>>>> + + " ElasticsearchSort(sort0=[$4], sort1=[$5],
> dir0=[ASC],
> >>>>> dir1=[ASC])\n"
> >>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> >>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> >>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> >>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"])\n"
> >>>>>> + + "
> >>> ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
> >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>> \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0,
> 'id')):VARCHAR(5)
> >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\",
> >>>>> '70000'))])\n"
> >>>>>> + + " ElasticsearchTableScan(table=[[elastic,
> >>> zips]])";
> >>>>>> + calciteAssert()
> >>>>>> + .query(sql)
> >>>>>> + .returnsOrdered("city=LOS ANGELES;
> longitude=-118.258189;
> >>>>> latitude=34.007856; "
> >>>>>> + + "pop=96074; state=CA; id=90011",
> >>>>>> + "city=BELL GARDENS; longitude=-118.17205;
> >>>>> latitude=33.969177; "
> >>>>>> + + "pop=99568; state=CA; id=90201",
> >>>>>> + "city=NORWALK; longitude=-118.081767;
> >>>>> latitude=33.90564; "
> >>>>>> + + "pop=94188; state=CA; id=90650")
> >>>>>> + .queryContains(
> >>>>>> + ElasticChecker.elasticsearchChecker("\"query\"
> : "
> >>>>>> + +
> >>>>> "{\"constant_score\":{\"filter\":{\"bool\":"
> >>>>>> + +
> >>>>> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
> >>>>>> + +
> >>>>> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
> >>>>>> + "\"script_fields\":
> >>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> >>>>>> + +
> >>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> >>>>>> + + "\"city\":{\"script\":
> >>>>> \"_source.city\"}, "
> >>>>>> + + "\"pop\":{\"script\":
> >>>>> \"_source.pop\"}, "
> >>>>>> + + "\"state\":{\"script\":
> >>>>> \"_source.state\"}, "
> >>>>>> + + "\"id\":{\"script\": \"_
> >>> source.id
> >>>>> \"}}",
> >>>>>> + "\"sort\": [ {\"state\": \"asc\"},
> >>> {\"id\":
> >>>>> \"asc\"}]"))
> >>>>>> + .explainContains(explain);
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testFilterSortDesc() {
> >>>>>> + final String sql = "select * from zips\n"
> >>>>>> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
> >>>>>> + + "order by \"state\" desc, \"pop\"";
> >>>>>> + calciteAssert()
> >>>>>> + .query(sql)
> >>>>>> + .limit(4)
> >>>>>> + .returnsOrdered(
> >>>>>> + "city=LOS ANGELES; longitude=-118.258189;
> >>>>> latitude=34.007856; pop=96074; state=CA; id=90011",
> >>>>>> + "city=BELL GARDENS; longitude=-118.17205;
> >>>>> latitude=33.969177; pop=99568; state=CA; id=90201");
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testFilterRedundant() {
> >>>>>> + final String sql = "select * from zips\n"
> >>>>>> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and
> >>>>> \"state\" = 'OK'";
> >>>>>> + calciteAssert()
> >>>>>> + .query(sql)
> >>>>>> + .runs()
> >>>>>> + .queryContains(
> >>>>>> + ElasticChecker.elasticsearchChecker(""
> >>>>>> + + "\"query\" :
> >>>>> {\"constant_score\":{\"filter\":{\"bool\":"
> >>>>>> + +
> >>>>> "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
> >>>>>> + "\"script_fields\":
> >>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> >>>>>> + +
> >>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> >>>>>> + + "\"city\":{\"script\":
> >>>>> \"_source.city\"}, "
> >>>>>> + + "\"pop\":{\"script\":
> >>>>> \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
> >>>>>> + +
> "\"id\":{\"script\":
> >>>>> \"_source.id\"}}"
> >>>>>> + ));
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testInPlan() {
> >>>>>> + final String[] searches = {
> >>>>>> + "\"query\" :
> >>>>> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
> >>>>>> + +
> >>>>>
> >>>
> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
> >>>>>> + + "{\"pop\":99568}}]}}]}}}}",
> >>>>>> + "\"script_fields\":
> >>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> >>>>>> + + "\"latitude\":{\"script\":\"_source.loc[1]\"},
> "
> >>>>>> + + "\"city\":{\"script\": \"_source.city\"}, "
> >>>>>> + + "\"pop\":{\"script\": \"_source.pop\"}, "
> >>>>>> + + "\"state\":{\"script\": \"_source.state\"}, "
> >>>>>> + + "\"id\":{\"script\": \"_source.id\"}}"
> >>>>>> + };
> >>>>>> +
> >>>>>> + calciteAssert()
> >>>>>> + .query("select * from zips where \"pop\" in (96074,
> >>> 99568)")
> >>>>>> + .returnsUnordered(
> >>>>>> + "city=BELL GARDENS; longitude=-118.17205;
> >>>>> latitude=33.969177; pop=99568; state=CA; id=90201",
> >>>>>> + "city=LOS ANGELES; longitude=-118.258189;
> >>>>> latitude=34.007856; pop=96074; state=CA; id=90011"
> >>>>>> + )
> >>>>>> +
> >>>>> .queryContains(ElasticChecker.elasticsearchChecker(searches));
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testZips() {
> >>>>>> + calciteAssert()
> >>>>>> + .query("select \"state\", \"city\" from zips")
> >>>>>> + .returnsCount(10);
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testProject() {
> >>>>>> + final String sql = "select \"state\", \"city\", 0 as
> \"zero\"\n"
> >>>>>> + + "from zips\n"
> >>>>>> + + "order by \"state\", \"city\"";
> >>>>>> +
> >>>>>> + calciteAssert()
> >>>>>> + .query(sql)
> >>>>>> + .limit(2)
> >>>>>> + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0",
> >>>>>> + "state=AK; city=FAIRBANKS; zero=0")
> >>>>>> + .queryContains(
> >>>>>> +
> >>>>> ElasticChecker.elasticsearchChecker("\"script_fields\": "
> >>>>>> + + "{\"zero\":{\"script\":
> \"0\"},
> >>> "
> >>>>>> + + "\"state\":{\"script\":
> >>>>> \"_source.state\"}, "
> >>>>>> + + "\"city\":{\"script\":
> >>>>> \"_source.city\"}}",
> >>>>>> + "\"sort\": [ {\"state\": \"asc\"},
> >>>>> {\"city\": \"asc\"}]"));
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testFilter() {
> >>>>>> + final String explain =
> "PLAN=ElasticsearchToEnumerableConverter\n"
> >>>>>> + + " ElasticsearchProject(state=[CAST(ITEM($0,
> >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>> \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0,
> 'city')):VARCHAR(20)
> >>>>> CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"])\n"
> >>>>>> + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0,
> >>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >>>>> \"ISO-8859-1$en_US$primary\", 'CA')])\n"
> >>>>>> + + " ElasticsearchTableScan(table=[[elastic,
> zips]])";
> >>>>>> + calciteAssert()
> >>>>>> + .query("select \"state\", \"city\" from zips where
> >>>>> \"state\" = 'CA'")
> >>>>>> + .limit(3)
> >>>>>> + .returnsUnordered("state=CA; city=BELL GARDENS",
> >>>>>> + "state=CA; city=LOS ANGELES",
> >>>>>> + "state=CA; city=NORWALK")
> >>>>>> + .explainContains(explain);
> >>>>>> + }
> >>>>>> +
> >>>>>> + @Test public void testFilterReversed() {
> >>>>>> + calciteAssert()
> >>>>>> + .query("select \"state\", \"city\" from zips where 'WI' <
> >>>>> \"state\" order by \"city\"")
> >>>>>> + .limit(2)
> >>>>>> + .returnsUnordered("state=WV; city=BECKLEY",
> >>>>>> + "state=WY; city=CHEYENNE");
> >>>>>> + calciteAssert()
> >>>>>> + .query("select \"state\", \"city\" from zips where
> \"state\"
> >>>>>> 'WI' order by \"city\"")
> >>>>>> + .limit(2)
> >>>>>> + .returnsUnordered("state=WV; city=BECKLEY",
> >>>>>> + "state=WY; city=CHEYENNE");
> >>>>>> + }
> >>>>>> +
> >>>>>> +}
> >>>>>> +
> >>>>>> +// End ElasticSearch2AdapterTest.java
> >>>>>>
> >>>>>>
> >>>>>
> >>>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >>>>>>
> ----------------------------------------------------------------------
> >>>>>> diff --git
> >>>>>
> >>>
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >>>>>
> >>>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >>>>>> new file mode 100644
> >>>>>> index 0000000..4474add
> >>>>>> --- /dev/null
> >>>>>> +++
> >>>>>
> >>>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >>>>>> @@ -0,0 +1,147 @@
> >>>>>> +/*
> >>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or
> more
> >>>>>> + * contributor license agreements. See the NOTICE file distributed
> >>> with
> >>>>>> + * this work for additional information regarding copyright
> ownership.
> >>>>>> + * The ASF licenses this file to you under the Apache License,
> Version
> >>>>> 2.0
> >>>>>> + * (the "License"); you may not use this file except in compliance
> >>> with
> >>>>>> + * the License. You may obtain a copy of the License at
> >>>>>> + *
> >>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
> >>>>>> + *
> >>>>>> + * Unless required by applicable law or agreed to in writing,
> software
> >>>>>> + * distributed under the License is distributed on an "AS IS"
> BASIS,
> >>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> >>>>> implied.
> >>>>>> + * See the License for the specific language governing permissions
> and
> >>>>>> + * limitations under the License.
> >>>>>> + */
> >>>>>> +package org.apache.calcite.adapter.elasticsearch2;
> >>>>>> +
> >>>>>> +import com.google.common.base.Preconditions;
> >>>>>> +import com.google.common.io.Files;
> >>>>>> +
> >>>>>> +import org.elasticsearch.Version;
> >>>>>> +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
> >>>>>> +import org.elasticsearch.action.admin.cluster.node.info
> >>>>> .NodesInfoResponse;
> >>>>>> +import org.elasticsearch.client.Client;
> >>>>>> +import org.elasticsearch.common.settings.Settings;
> >>>>>> +import org.elasticsearch.common.transport.TransportAddress;
> >>>>>> +import org.elasticsearch.node.Node;
> >>>>>> +import org.elasticsearch.node.internal.InternalSettingsPreparer;
> >>>>>> +import org.elasticsearch.plugins.Plugin;
> >>>>>> +import org.elasticsearch.script.groovy.GroovyPlugin;
> >>>>>> +
> >>>>>> +import java.io.File;
> >>>>>> +import java.util.Arrays;
> >>>>>> +import java.util.Collection;
> >>>>>> +import java.util.Collections;
> >>>>>> +
> >>>>>> +/**
> >>>>>> + * Represents a single elastic search node which can run embedded
> in a
> >>>>> java application.
> >>>>>> + * Intended for unit and integration tests. Settings and plugins
> are
> >>>>> crafted for Calcite.
> >>>>>> + */
> >>>>>> +class EmbeddedElasticNode implements AutoCloseable {
> >>>>>> +
> >>>>>> + private final LocalNode node;
> >>>>>> + private volatile boolean isStarted;
> >>>>>> +
> >>>>>> + private EmbeddedElasticNode(LocalNode node) {
> >>>>>> + this.node = Preconditions.checkNotNull(node, "node");
> >>>>>> + }
> >>>>>> +
> >>>>>> + /**
> >>>>>> + * Having separate class to expose (protected) constructor which
> >>>>> allows to install
> >>>>>> + * different plugins. In our case it is {@code GroovyPlugin} for
> >>>>> scripted fields like
> >>>>>> + * {@code loc[0]} or {@code loc[1]['foo']}.
> >>>>>> + */
> >>>>>> + private static class LocalNode extends Node {
> >>>>>> + private LocalNode(Settings settings, Collection<Class<? extends
> >>>>> Plugin>> classpathPlugins) {
> >>>>>> + super(InternalSettingsPreparer.prepareEnvironment(settings,
> >>> null),
> >>>>>> + Version.CURRENT,
> >>>>>> + classpathPlugins);
> >>>>>> + }
> >>>>>> + }
> >>>>>> +
> >>>>>> + /**
> >>>>>> + * Creates an instance with existing settings
> >>>>>> + */
> >>>>>> + private static EmbeddedElasticNode create(Settings settings) {
> >>>>>> + // ensure GroovyPlugin is installed or otherwise scripted
> fields
> >>>>> would not work
> >>>>>> + LocalNode node = new LocalNode(settings,
> >>>>> Collections.singleton(GroovyPlugin.class));
> >>>>>> + return new EmbeddedElasticNode(node);
> >>>>>> + }
> >>>>>> +
> >>>>>> + /**
> >>>>>> + * Creates elastic node as single member of a cluster. Node will
> not
> >>>>> be started
> >>>>>> + * unless {@link #start()} is explicitly called.
> >>>>>> + */
> >>>>>> + public static EmbeddedElasticNode create() {
> >>>>>> + File data = Files.createTempDir();
> >>>>>> + data.deleteOnExit();
> >>>>>> + File home = Files.createTempDir();
> >>>>>> + home.deleteOnExit();
> >>>>>> +
> >>>>>> + Settings settings = Settings.builder()
> >>>>>> + .put("node.name", "fake-elastic")
> >>>>>> + .put("path.home", home.getAbsolutePath())
> >>>>>> + .put("path.data", data.getAbsolutePath())
> >>>>>> + .put("script.inline", true) // requires GroovyPlugin
> >>>>>> + .put("script.indexed", true) // requires GroovyPlugin
> >>>>>> +
> .put("cluster.routing.allocation.disk.threshold_enabled",
> >>>>> false)
> >>>>>> + .put("node.local", true)
> >>>>>> + .put("node.data", true)
> >>>>>> + .put("network.host", "localhost")
> >>>>>> + .build();
> >>>>>> +
> >>>>>> + return create(settings);
> >>>>>> + }
> >>>>>> +
> >>>>>> + /**
> >>>>>> + * Starts current node
> >>>>>> + */
> >>>>>> + public void start() {
> >>>>>> + Preconditions.checkState(!isStarted, "already started");
> >>>>>> + node.start();
> >>>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Julian Hyde <jh...@apache.org>.
Same failure for me on JDK 8/macOS. The most likely other variable is maven: I have 3.5.2 on macOS, 3.5.3 on ubuntu.
I’m mystified why no one else is seeing javadoc failures on macOS.
Julian
> On Jun 22, 2018, at 4:35 PM, Julian Hyde <jh...@apache.org> wrote:
>
> On JDK 8/ubuntu, “mvn clean -DskipTests site” gives the following failure:
>
> [ERROR] Failed to execute goal org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on project calcite-mongodb: Error generating maven-javadoc-plugin:3.0.1:test-javadoc report:
> [ERROR] Exit code: 1 - /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:22: error: cannot find symbol
> [ERROR] import org.apache.calcite.test.CalciteAssert;
> [ERROR] ^
> [ERROR] symbol: class CalciteAssert
> [ERROR] location: package org.apache.calcite.test
> [ERROR] /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:137: error: package CalciteAssert does not exist
> [ERROR] private CalciteAssert.AssertThat assertModel(String model) {
> [ERROR] ^
> [ERROR] /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:145: error: package CalciteAssert does not exist
> [ERROR] private CalciteAssert.AssertThat assertModel(URL url) {
> [ERROR] ^
> [ERROR] /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:43: error: reference not found
> [ERROR] /** Similar to {@link CalciteAssert#checkResultUnordered}, but filters strings
> [ERROR] ^
>
>
>
>> On Jun 22, 2018, at 3:46 PM, Andrei Sereda <an...@sereda.cc> wrote:
>>
>> Hmm. Both master and c12cb4b0de work for me. Perhaps the issue is JDK 8 vs
>> 9 ?
>>
>> # JDK8 (oracle) / macOS
>> $ mvn clean -DskipTests site
>>
>> On Fri, Jun 22, 2018 at 5:56 PM Julian Hyde <jh...@apache.org> wrote:
>>
>>> I just tried it again. The following fails for me (jdk9, ubuntu):
>>>
>>> $ git checkout c12cb4b0de
>>> $ mvn clean -DskipTests site
>>>
>>> Constructing Javadoc information...
>>> 1 error
>>> [INFO]
>>> ------------------------------------------------------------------------
>>> [INFO] Reactor Summary:
>>> [INFO]
>>> [INFO] Calcite 1.17.0-SNAPSHOT ............................ FAILURE [02:29
>>> min]
>>> [INFO] Calcite Linq4j ..................................... SKIPPED
>>> [INFO] Calcite Core ....................................... SKIPPED
>>> [INFO] Calcite Cassandra .................................. SKIPPED
>>> [INFO] Calcite Druid ...................................... SKIPPED
>>> [INFO] Calcite Elasticsearch .............................. SKIPPED
>>> [INFO] Calcite Elasticsearch5 ............................. SKIPPED
>>> [INFO] Calcite Examples ................................... SKIPPED
>>> [INFO] Calcite Example CSV ................................ SKIPPED
>>> [INFO] Calcite Example Function ........................... SKIPPED
>>> [INFO] Calcite File ....................................... SKIPPED
>>> [INFO] Calcite Geode ...................................... SKIPPED
>>> [INFO] Calcite MongoDB .................................... SKIPPED
>>> [INFO] Calcite Pig ........................................ SKIPPED
>>> [INFO] Calcite Piglet ..................................... SKIPPED
>>> [INFO] Calcite Plus ....................................... SKIPPED
>>> [INFO] Calcite Server ..................................... SKIPPED
>>> [INFO] Calcite Spark ...................................... SKIPPED
>>> [INFO] Calcite Splunk ..................................... SKIPPED
>>> [INFO] Calcite Ubenchmark 1.17.0-SNAPSHOT ................. SKIPPED
>>> [INFO]
>>> ------------------------------------------------------------------------
>>> [INFO] BUILD FAILURE
>>> [INFO]
>>> ------------------------------------------------------------------------
>>> [INFO] Total time: 02:29 min
>>> [INFO] Finished at: 2018-06-22T14:53:13-07:00
>>> [INFO]
>>> ------------------------------------------------------------------------
>>> [ERROR] Failed to execute goal
>>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
>>> project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate
>>> report:
>>> [ERROR] Exit code: 1 -
>>> /home/jhyde/open1/calcite.4/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
>>> error: package org.elasticsearch.node.internal does not exist
>>> [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
>>> [ERROR] ^
>>> [ERROR]
>>> [ERROR] Command line was: /usr/lib/jvm/jdk9/bin/javadoc @options @packages
>>> @argfile
>>> [ERROR]
>>> [ERROR] Refer to the generated Javadoc files in
>>> '/home/jhyde/open1/calcite.4/target/site/testapidocs' dir.
>>> [ERROR] -> [Help 1]
>>> [ERROR]
>>> [ERROR] To see the full stack trace of the errors, re-run Maven with the
>>> -e switch.
>>> [ERROR] Re-run Maven using the -X switch to enable full debug logging.
>>> [ERROR]
>>> [ERROR] For more information about the errors and possible solutions,
>>> please read the following articles:
>>> [ERROR] [Help 1]
>>> http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException
>>>
>>>
>>>> On Jun 22, 2018, at 2:41 PM, Michael Mior <mm...@apache.org> wrote:
>>>>
>>>> Odd, mvn clean site still works fine for me.
>>>> --
>>>> Michael Mior
>>>> mmior@apache.org
>>>>
>>>>
>>>>
>>>> Le ven. 22 juin 2018 à 17:12, Julian Hyde <jh...@apache.org> a écrit :
>>>>
>>>>> Looks like this change broke “mvn site” (perhaps also “mvn
>>>>> javadoc:test-javadoc”).
>>>>>
>>>>> [ERROR] Failed to execute goal
>>>>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
>>>>> project calcite: Error generating
>>> maven-javadoc-plugin:3.0.1:test-aggregate
>>>>> report:
>>>>> [ERROR] Exit code: 1 -
>>>>>
>>> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
>>>>> error: package org.elasticsearch.node.internal does not exist
>>>>> [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
>>>>> [ERROR] ^
>>>>> [ERROR]
>>>>>
>>>>>> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
>>>>>>
>>>>>> [CALCITE-2347] running ElasticSearch in embedded mode for unit tests of
>>>>> ES adapter (Andrei Sereda)
>>>>>>
>>>>>> After discussion on dev-list Integration tests (for ES) have been
>>>>> removed. They're now
>>>>>> superseded by unit tests (which execute queries against a real elastic
>>>>> instance)
>>>>>>
>>>>>> Added local file (zips-mini.json) which contains a small subset of
>>>>> original zips.json
>>>>>> (allows to bootstrap tests faster)
>>>>>>
>>>>>> Created separate ES JUnit rule which can be re-used across different
>>>>> tests.
>>>>>>
>>>>>> Both v2 and v5 of ES adapters are supported.
>>>>>>
>>>>>> Close apache/calcite#716
>>>>>>
>>>>>>
>>>>>> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
>>>>>> Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
>>>>>> Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
>>>>>> Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
>>>>>>
>>>>>> Branch: refs/heads/master
>>>>>> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
>>>>>> Parents: 37944bb
>>>>>> Author: Andrei Sereda <an...@nospam.com>
>>>>>> Authored: Thu May 31 18:19:10 2018 -0400
>>>>>> Committer: Michael Mior <mm...@uwaterloo.ca>
>>>>>> Committed: Thu Jun 21 06:38:50 2018 -0400
>>>>>>
>>>>>> ----------------------------------------------------------------------
>>>>>> .../AbstractElasticsearchTable.java | 12 +
>>>>>> .../elasticsearch/ElasticsearchProject.java | 61 ++-
>>>>>> elasticsearch2/pom.xml | 6 +
>>>>>> .../Elasticsearch2Enumerator.java | 12 +-
>>>>>> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
>>>>>> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
>>>>>> .../ElasticSearch2AdapterTest.java | 395
>>> ++++++++++++++++++
>>>>>> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
>>>>>> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
>>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
>>>>>> .../calcite/test/Elasticsearch2AdapterIT.java | 270 -------------
>>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
>>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
>>>>>> elasticsearch5/pom.xml | 31 ++
>>>>>> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
>>>>>> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
>>>>>> .../ElasticSearch5AdapterTest.java | 399
>>> +++++++++++++++++++
>>>>>> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
>>>>>> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
>>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
>>>>>> .../calcite/test/Elasticsearch5AdapterIT.java | 270 -------------
>>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
>>>>>> elasticsearch5/src/test/resources/log4j2.xml | 16 +
>>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
>>>>>> pom.xml | 20 +-
>>>>>> 25 files changed, 1866 insertions(+), 670 deletions(-)
>>>>>> ----------------------------------------------------------------------
>>>>>>
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git
>>>>>
>>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>>
>>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>>> index 0980469..8cc5933 100644
>>>>>> ---
>>>>>
>>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>>> +++
>>>>>
>>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>>> @@ -75,6 +75,18 @@ public abstract class AbstractElasticsearchTable
>>>>> extends AbstractQueryableTable
>>>>>> relOptTable, this, null);
>>>>>> }
>>>>>>
>>>>>> + /**
>>>>>> + * In ES 5.x scripted fields start with {@code params._source.foo}
>>>>> while in ES2.x
>>>>>> + * {@code _source.foo}. Helper method to build correct query based
>>> on
>>>>> runtime version of elastic.
>>>>>> + *
>>>>>> + * @see <a href="
>>>>> https://github.com/elastic/elasticsearch/issues/20068">_source
>>>>> variable</a>
>>>>>> + * @see <a href="
>>>>>
>>> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html
>>> ">Scripted
>>>>> Fields</a>
>>>>>> + */
>>>>>> + protected String scriptedFieldPrefix() {
>>>>>> + // this is default pattern starting 5.x
>>>>>> + return "params._source";
>>>>>> + }
>>>>>> +
>>>>>> /** Executes a "find" operation on the underlying type.
>>>>>> *
>>>>>> * <p>For example,
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git
>>>>>
>>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>>
>>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>>> index b42abd7..961c8b0 100644
>>>>>> ---
>>>>>
>>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>>> +++
>>>>>
>>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>>> @@ -27,11 +27,15 @@ import
>>>>> org.apache.calcite.rel.metadata.RelMetadataQuery;
>>>>>> import org.apache.calcite.rel.type.RelDataType;
>>>>>> import org.apache.calcite.rex.RexNode;
>>>>>> import org.apache.calcite.util.Pair;
>>>>>> -import org.apache.calcite.util.Util;
>>>>>> +
>>>>>> +import com.google.common.base.Function;
>>>>>> +import com.google.common.collect.Lists;
>>>>>>
>>>>>> import java.util.ArrayList;
>>>>>> import java.util.List;
>>>>>>
>>>>>> +import javax.annotation.Nullable;
>>>>>> +
>>>>>> /**
>>>>>> * Implementation of {@link org.apache.calcite.rel.core.Project}
>>>>>> * relational expression in Elasticsearch.
>>>>>> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends Project
>>>>> implements ElasticsearchRel {
>>>>>> implementor.visitChild(0, getInput());
>>>>>>
>>>>>> final List<String> inFields =
>>>>>> -
>>>>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
>>>>>> +
>>>>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
>>>>>> final ElasticsearchRules.RexToElasticsearchTranslator translator =
>>>>>> - new ElasticsearchRules.RexToElasticsearchTranslator(
>>>>>> - (JavaTypeFactory) getCluster().getTypeFactory(),
>>> inFields);
>>>>>> + new ElasticsearchRules.RexToElasticsearchTranslator(
>>>>>> + (JavaTypeFactory) getCluster().getTypeFactory(),
>>>>> inFields);
>>>>>>
>>>>>> - final List<String> findItems = new ArrayList<>();
>>>>>> - final List<String> scriptFieldItems = new ArrayList<>();
>>>>>> + final List<String> fields = new ArrayList<>();
>>>>>> + final List<String> scriptFields = new ArrayList<>();
>>>>>> for (Pair<RexNode, String> pair: getNamedProjects()) {
>>>>>> final String name = pair.right;
>>>>>> final String expr = pair.left.accept(translator);
>>>>>>
>>>>>> if (expr.equals("\"" + name + "\"")) {
>>>>>> - findItems.add(ElasticsearchRules.quote(name));
>>>>>> + fields.add(name);
>>>>>> } else if (expr.matches("\"literal\":.+")) {
>>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
>>>>>> - + ":{\"script\": "
>>>>>> - + expr.split(":")[1] + "}");
>>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
>>>>>> + + ":{\"script\": "
>>>>>> + + expr.split(":")[1] + "}");
>>>>>> } else {
>>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
>>>>>> - + ":{\"script\":\"params._source."
>>>>>> - + expr.replaceAll("\"", "") + "\"}");
>>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
>>>>>> + + ":{\"script\":"
>>>>>> + // _source (ES2) vs params._source (ES5)
>>>>>> + + "\"" +
>>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
>>>>>> + + expr.replaceAll("\"", "") + "\"}");
>>>>>> + }
>>>>>> + }
>>>>>> +
>>>>>> + StringBuilder query = new StringBuilder();
>>>>>> + if (scriptFields.isEmpty()) {
>>>>>> + List<String> newList = Lists.transform(fields, new
>>>>> Function<String, String>() {
>>>>>> + @Nullable
>>>>>> + @Override public String apply(@Nullable String input) {
>>>>>> + return ElasticsearchRules.quote(input);
>>>>>> + }
>>>>>> + });
>>>>>> +
>>>>>> + final String findString = String.join(", ", newList);
>>>>>> + query.append("\"_source\" : [").append(findString).append("]");
>>>>>> + } else {
>>>>>> + // if scripted fields are present, ES ignores _source attribute
>>>>>> + for (String field: fields) {
>>>>>> + scriptFields.add(ElasticsearchRules.quote(field) +
>>>>> ":{\"script\": "
>>>>>> + // _source (ES2) vs params._source (ES5)
>>>>>> + + "\"" +
>>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
>>>>>> + + field + "\"}");
>>>>>> }
>>>>>> + query.append("\"script_fields\": {" + String.join(", ",
>>>>> scriptFields) + "}");
>>>>>> }
>>>>>> - final String findString = Util.toString(findItems, "", ", ", "");
>>>>>> - final String scriptFieldString = "\"script_fields\": {"
>>>>>> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
>>>>>> - final String fieldString = "\"_source\" : [" + findString + "]"
>>>>>> - + ", " + scriptFieldString;
>>>>>>
>>>>>> for (String opfield : implementor.list) {
>>>>>> if (opfield.startsWith("\"_source\"")) {
>>>>>> implementor.list.remove(opfield);
>>>>>> }
>>>>>> }
>>>>>> - implementor.add(fieldString);
>>>>>> + implementor.add(query.toString());
>>>>>> }
>>>>>> }
>>>>>>
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
>>>>>> index f24622c..6fbee03 100644
>>>>>> --- a/elasticsearch2/pom.xml
>>>>>> +++ b/elasticsearch2/pom.xml
>>>>>> @@ -73,6 +73,12 @@ limitations under the License.
>>>>>> <version>${elasticsearch-java-driver.version}</version>
>>>>>> </dependency>
>>>>>> <dependency>
>>>>>> + <!-- Lang groovy dependency is needed for testing with embedded
>>>>> ES (scripted fields like loc[0]) -->
>>>>>> + <groupId>org.elasticsearch.module</groupId>
>>>>>> + <artifactId>lang-groovy</artifactId>
>>>>>> + <scope>test</scope>
>>>>>> + </dependency>
>>>>>> + <dependency>
>>>>>> <groupId>com.carrotsearch</groupId>
>>>>>> <artifactId>hppc</artifactId>
>>>>>> <version>${hppc.version}</version>
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git
>>>>>
>>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>>
>>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>>> index 84370ab..c3d2ac0 100644
>>>>>> ---
>>>>>
>>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>>> +++
>>>>>
>>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>>> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
>>>>>> import java.util.Date;
>>>>>> import java.util.Iterator;
>>>>>> import java.util.List;
>>>>>> +import java.util.Locale;
>>>>>> import java.util.Map;
>>>>>>
>>>>>> /**
>>>>>> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator implements
>>>>> Enumerator<Object> {
>>>>>> private static Function1<SearchHit, Object[]> listGetter(
>>>>>> final List<Map.Entry<String, Class>> fields) {
>>>>>> return new Function1<SearchHit, Object[]>() {
>>>>>> - public Object[] apply(SearchHit searchHitFields) {
>>>>>> + public Object[] apply(SearchHit hit) {
>>>>>> Object[] objects = new Object[fields.size()];
>>>>>> for (int i = 0; i < fields.size(); i++) {
>>>>>> final Map.Entry<String, Class> field = fields.get(i);
>>>>>> final String name = field.getKey();
>>>>>> - if (searchHitFields.fields().isEmpty()) {
>>>>>> - objects[i] =
>>> convert(searchHitFields.getSource().get(name),
>>>>> field.getValue());
>>>>>> + if (hit.fields().isEmpty()) {
>>>>>> + objects[i] = convert(hit.getSource().get(name),
>>>>> field.getValue());
>>>>>> + } else if (hit.fields().containsKey(name)) {
>>>>>> + objects[i] = convert(hit.field(name).getValue(),
>>>>> field.getValue());
>>>>>> } else {
>>>>>> - objects[i] =
>>>>> convert(searchHitFields.field(name).getValue(), field.getValue());
>>>>>> + throw new IllegalStateException(
>>>>>> + String.format(Locale.getDefault(), "No result for
>>>>> %s", field));
>>>>>> }
>>>>>> }
>>>>>> return objects;
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git
>>>>>
>>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>>
>>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>>> index 668402b..46e3fc5 100644
>>>>>> ---
>>>>>
>>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>>> +++
>>>>>
>>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>>> @@ -22,6 +22,9 @@ import org.apache.calcite.schema.impl.AbstractSchema;
>>>>>>
>>>>>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
>>>>>>
>>>>>> +import com.google.common.annotations.VisibleForTesting;
>>>>>> +import com.google.common.base.Preconditions;
>>>>>> +
>>>>>> import com.google.common.collect.ImmutableList;
>>>>>> import com.google.common.collect.ImmutableMap;
>>>>>>
>>>>>> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
>>>>> AbstractSchema
>>>>>> }
>>>>>> }
>>>>>>
>>>>>> + /**
>>>>>> + * Allows schema to be instantiated from existing elastic search
>>>>> client.
>>>>>> + * This constructor is used in tests.
>>>>>> + */
>>>>>> + @VisibleForTesting
>>>>>> + Elasticsearch2Schema(Client client, String index) {
>>>>>> + this.client = Preconditions.checkNotNull(client, "client");
>>>>>> + this.index = Preconditions.checkNotNull(index, "index");
>>>>>> + }
>>>>>> +
>>>>>> @Override protected Map<String, Table> getTableMap() {
>>>>>> final ImmutableMap.Builder<String, Table> builder =
>>>>> ImmutableMap.builder();
>>>>>>
>>>>>> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
>>>>> AbstractSchema
>>>>>>
>>>>>> final List<DiscoveryNode> nodes =
>>>>> ImmutableList.copyOf(transportClient.connectedNodes());
>>>>>> if (nodes.isEmpty()) {
>>>>>> - throw new RuntimeException("Cannot connect to any elasticsearch
>>>>> nodes");
>>>>>> + throw new IllegalStateException("Cannot connect to any
>>>>> elasticsearch node: "
>>>>>> + + transportNodes);
>>>>>> }
>>>>>>
>>>>>> client = transportClient;
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git
>>>>>
>>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>>
>>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>>> index 636aa5f..2928835 100644
>>>>>> ---
>>>>>
>>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>>> +++
>>>>>
>>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>>> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
>>>>> AbstractElasticsearchTable {
>>>>>> this.client = client;
>>>>>> }
>>>>>>
>>>>>> + /**
>>>>>> + * ES version 2.x. To access document attributes ES2 uses {@code
>>>>> _source.foo} syntax.
>>>>>> + */
>>>>>> + @Override protected String scriptedFieldPrefix() {
>>>>>> + return "_source";
>>>>>> + }
>>>>>> +
>>>>>> @Override protected Enumerable<Object> find(String index, List<String>
>>>>> ops,
>>>>>> - List<Map.Entry<String, Class>> fields) {
>>>>>> + List<Map.Entry<String,
>>>>> Class>> fields) {
>>>>>> final String dbName = index;
>>>>>>
>>>>>> final String queryString = "{" + Util.toString(ops, "", ", ", "") +
>>>>> "}";
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git
>>>>>
>>> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>>
>>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>>> new file mode 100644
>>>>>> index 0000000..287e094
>>>>>> --- /dev/null
>>>>>> +++
>>>>>
>>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>>> @@ -0,0 +1,395 @@
>>>>>> +/*
>>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>>>>> + * contributor license agreements. See the NOTICE file distributed
>>> with
>>>>>> + * this work for additional information regarding copyright ownership.
>>>>>> + * The ASF licenses this file to you under the Apache License, Version
>>>>> 2.0
>>>>>> + * (the "License"); you may not use this file except in compliance
>>> with
>>>>>> + * the License. You may obtain a copy of the License at
>>>>>> + *
>>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>>>>> + *
>>>>>> + * Unless required by applicable law or agreed to in writing, software
>>>>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>>>>> implied.
>>>>>> + * See the License for the specific language governing permissions and
>>>>>> + * limitations under the License.
>>>>>> + */
>>>>>> +package org.apache.calcite.adapter.elasticsearch2;
>>>>>> +
>>>>>> +import org.apache.calcite.jdbc.CalciteConnection;
>>>>>> +import org.apache.calcite.schema.SchemaPlus;
>>>>>> +import org.apache.calcite.schema.impl.ViewTable;
>>>>>> +import org.apache.calcite.schema.impl.ViewTableMacro;
>>>>>> +import org.apache.calcite.test.CalciteAssert;
>>>>>> +import org.apache.calcite.test.ElasticChecker;
>>>>>> +
>>>>>> +import com.google.common.io.LineProcessor;
>>>>>> +import com.google.common.io.Resources;
>>>>>> +
>>>>>> +import org.elasticsearch.action.bulk.BulkItemResponse;
>>>>>> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
>>>>>> +import org.elasticsearch.action.bulk.BulkResponse;
>>>>>> +import org.elasticsearch.common.xcontent.XContentBuilder;
>>>>>> +import org.elasticsearch.common.xcontent.XContentFactory;
>>>>>> +
>>>>>> +import org.junit.BeforeClass;
>>>>>> +import org.junit.ClassRule;
>>>>>> +import org.junit.Test;
>>>>>> +
>>>>>> +import java.io.IOException;
>>>>>> +import java.nio.charset.StandardCharsets;
>>>>>> +import java.sql.Connection;
>>>>>> +import java.sql.DriverManager;
>>>>>> +import java.sql.SQLException;
>>>>>> +import java.util.Arrays;
>>>>>> +import java.util.Collections;
>>>>>> +import java.util.Locale;
>>>>>> +
>>>>>> +/**
>>>>>> + * Set of tests for ES adapter. Uses real instance via {@link
>>>>> EmbeddedElasticRule}. Document
>>>>>> + * source is local {@code zips-mini.json} file (located in the
>>>>> classpath).
>>>>>> + */
>>>>>> +public class ElasticSearch2AdapterTest {
>>>>>> +
>>>>>> + @ClassRule //init once for all tests
>>>>>> + public static final EmbeddedElasticRule NODE =
>>>>> EmbeddedElasticRule.create();
>>>>>> +
>>>>>> + private static final String ZIPS = "zips";
>>>>>> +
>>>>>> + /**
>>>>>> + * Used to create {@code zips} index and insert some data
>>>>>> + */
>>>>>> + @BeforeClass
>>>>>> + public static void setupInstance() throws Exception {
>>>>>> + // define mapping so fields are searchable (term query)
>>>>>> + XContentBuilder mapping =
>>>>> XContentFactory.jsonBuilder().startObject()
>>>>>> + .startObject("properties")
>>>>>> + .startObject("city").field("type", "string")
>>>>>> + .field("index", "not_analyzed").endObject()
>>>>>> + .startObject("state").field("type", "string")
>>>>>> + .field("index", "not_analyzed").endObject()
>>>>>> + .startObject("pop").field("type", "long").endObject()
>>>>>> + .endObject()
>>>>>> + .endObject();
>>>>>> +
>>>>>> + // create index
>>>>>> + NODE.client().admin().indices()
>>>>>> + .prepareCreate(ZIPS)
>>>>>> + .addMapping(ZIPS, mapping)
>>>>>> + .get();
>>>>>> +
>>>>>> + BulkRequestBuilder bulk =
>>>>> NODE.client().prepareBulk().setRefresh(true);
>>>>>> +
>>>>>> + // load records from file
>>>>>> +
>>>>>
>>> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
>>>>>> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
>>>>>> + @Override public boolean processLine(String line) throws
>>>>> IOException {
>>>>>> + line = line.replaceAll("_id", "id"); // _id is a
>>>>> reserved attribute in ES
>>>>>> + bulk.add(NODE.client().prepareIndex(ZIPS,
>>>>> ZIPS).setSource(line));
>>>>>> + return true;
>>>>>> + }
>>>>>> +
>>>>>> + @Override public Void getResult() {
>>>>>> + return null;
>>>>>> + }
>>>>>> + });
>>>>>> +
>>>>>> + if (bulk.numberOfActions() == 0) {
>>>>>> + throw new IllegalStateException("No records to be indexed");
>>>>>> + }
>>>>>> +
>>>>>> + BulkResponse response = bulk.execute().get();
>>>>>> +
>>>>>> + if (response.hasFailures()) {
>>>>>> + throw new IllegalStateException(
>>>>>> + String.format(Locale.getDefault(), "Failed to populate
>>>>> %s:\n%s", NODE.httpAddress(),
>>>>>> +
>>>>> Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
>>>>>> +
>>>>>
>>> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
>>>>>> + }
>>>>>> +
>>>>>> + }
>>>>>> +
>>>>>> + private CalciteAssert.ConnectionFactory newConnectionFactory() {
>>>>>> + return new CalciteAssert.ConnectionFactory() {
>>>>>> + @Override public Connection createConnection() throws
>>>>> SQLException {
>>>>>> + final Connection connection =
>>>>> DriverManager.getConnection("jdbc:calcite:");
>>>>>> + final SchemaPlus root =
>>>>> connection.unwrap(CalciteConnection.class).getRootSchema();
>>>>>> +
>>>>>> + root.add("elastic", new Elasticsearch2Schema(NODE.client(),
>>>>> ZIPS));
>>>>>> +
>>>>>> + // add calcite view programmatically
>>>>>> + final String viewSql = "select cast(_MAP['city'] AS
>>>>> varchar(20)) AS \"city\", "
>>>>>> + + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n"
>>>>>> + + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n"
>>>>>> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
>>>>>> + + " cast(_MAP['state'] AS varchar(2)) AS \"state\", "
>>>>>> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
>>>>>> + + "from \"elastic\".\"zips\"";
>>>>>> +
>>>>>> + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
>>>>>> + Collections.singletonList("elastic"),
>>>>> Arrays.asList("elastic", "view"), false);
>>>>>> + root.add("ZIPS", macro);
>>>>>> +
>>>>>> + return connection;
>>>>>> + }
>>>>>> + };
>>>>>> + }
>>>>>> +
>>>>>> + private CalciteAssert.AssertThat calciteAssert() {
>>>>>> + return CalciteAssert.that()
>>>>>> + .with(newConnectionFactory());
>>>>>> + }
>>>>>> +
>>>>>> + /**
>>>>>> + * Tests using calcite view
>>>>>> + */
>>>>>> + @Test
>>>>>> + public void view() throws Exception {
>>>>>> + calciteAssert()
>>>>>> + .query("select * from zips where \"city\" = 'BROOKLYN'")
>>>>>> + .returns("city=BROOKLYN; longitude=-73.956985;
>>>>> latitude=40.646694; "
>>>>>> + + "pop=111396; state=NY; id=11226\n")
>>>>>> + .returnsCount(1);
>>>>>> + }
>>>>>> +
>>>>>> + @Test
>>>>>> + public void emptyResult() {
>>>>>> + CalciteAssert.that()
>>>>>> + .with(newConnectionFactory())
>>>>>> + .query("select * from zips limit 0")
>>>>>> + .returnsCount(0);
>>>>>> +
>>>>>> + CalciteAssert.that()
>>>>>> + .with(newConnectionFactory())
>>>>>> + .query("select * from \"elastic\".\"zips\" where
>>>>> _MAP['Foo'] = '_MISSING_'")
>>>>>> + .returnsCount(0);
>>>>>> + }
>>>>>> +
>>>>>> + @Test
>>>>>> + public void basic() throws Exception {
>>>>>> + CalciteAssert.that()
>>>>>> + .with(newConnectionFactory())
>>>>>> + .query("select * from \"elastic\".\"zips\" where
>>>>> _MAP['city'] = 'BROOKLYN'")
>>>>>> + .returnsCount(1);
>>>>>> +
>>>>>> + CalciteAssert.that()
>>>>>> + .with(newConnectionFactory())
>>>>>> + .query("select * from \"elastic\".\"zips\" where"
>>>>>> + + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')")
>>>>>> + .returnsCount(2);
>>>>>> +
>>>>>> + // lower-case
>>>>>> + CalciteAssert.that()
>>>>>> + .with(newConnectionFactory())
>>>>>> + .query("select * from \"elastic\".\"zips\" where "
>>>>>> + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
>>>>> 'BROOK') ")
>>>>>> + .returnsCount(0);
>>>>>> +
>>>>>> + // missing field
>>>>>> + CalciteAssert.that()
>>>>>> + .with(newConnectionFactory())
>>>>>> + .query("select * from \"elastic\".\"zips\" where
>>>>> _MAP['CITY'] = 'BROOKLYN'")
>>>>>> + .returnsCount(0);
>>>>>> +
>>>>>> + // limit works
>>>>>> + CalciteAssert.that()
>>>>>> + .with(newConnectionFactory())
>>>>>> + .query("select * from \"elastic\".\"zips\" limit 42")
>>>>>> + .returnsCount(42);
>>>>>> +
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testSort() {
>>>>>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>>>>> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
>>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
>>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
>>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
>>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
>>>>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>>>>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
>>>>>> +
>>>>>> + calciteAssert()
>>>>>> + .query("select * from zips order by \"state\"")
>>>>>> + .returnsCount(10)
>>>>>> + .explainContains(explain);
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testSortLimit() {
>>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>>>>> + + "order by \"state\", \"id\" offset 2 rows fetch next 3
>>>>> rows only";
>>>>>> + calciteAssert()
>>>>>> + .query(sql)
>>>>>> + .returnsUnordered("state=AK; id=99801",
>>>>>> + "state=AL; id=35215",
>>>>>> + "state=AL; id=35401")
>>>>>> + .queryContains(
>>>>>> + ElasticChecker.elasticsearchChecker(
>>>>>> + "\"_source\" : [\"state\", \"id\"]",
>>>>>> + "\"sort\": [ {\"state\": \"asc\"},
>>> {\"id\":
>>>>> \"asc\"}]",
>>>>>> + "\"from\": 2",
>>>>>> + "\"size\": 3"));
>>>>>> + }
>>>>>> +
>>>>>> +
>>>>>> +
>>>>>> + @Test public void testOffsetLimit() {
>>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>>>>> + + "offset 2 fetch next 3 rows only";
>>>>>> + calciteAssert()
>>>>>> + .query(sql)
>>>>>> + .runs()
>>>>>> + .queryContains(
>>>>>> + ElasticChecker.elasticsearchChecker(
>>>>>> + "\"from\": 2",
>>>>>> + "\"size\": 3",
>>>>>> + "\"_source\" : [\"state\", \"id\"]"));
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testLimit() {
>>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>>>>> + + "fetch next 3 rows only";
>>>>>> +
>>>>>> + calciteAssert()
>>>>>> + .query(sql)
>>>>>> + .runs()
>>>>>> + .queryContains(
>>>>>> + ElasticChecker.elasticsearchChecker(
>>>>>> + "\"size\": 3",
>>>>>> + "\"_source\" : [\"state\", \"id\"]"));
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testFilterSort() {
>>>>>> + final String sql = "select * from zips\n"
>>>>>> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
>>>>>> + + "order by \"state\", \"id\"";
>>>>>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>>>>> + + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC],
>>>>> dir1=[ASC])\n"
>>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
>>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
>>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
>>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
>>>>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>>>>> + + "
>>> ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>> \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0, 'id')):VARCHAR(5)
>>>>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\",
>>>>> '70000'))])\n"
>>>>>> + + " ElasticsearchTableScan(table=[[elastic,
>>> zips]])";
>>>>>> + calciteAssert()
>>>>>> + .query(sql)
>>>>>> + .returnsOrdered("city=LOS ANGELES; longitude=-118.258189;
>>>>> latitude=34.007856; "
>>>>>> + + "pop=96074; state=CA; id=90011",
>>>>>> + "city=BELL GARDENS; longitude=-118.17205;
>>>>> latitude=33.969177; "
>>>>>> + + "pop=99568; state=CA; id=90201",
>>>>>> + "city=NORWALK; longitude=-118.081767;
>>>>> latitude=33.90564; "
>>>>>> + + "pop=94188; state=CA; id=90650")
>>>>>> + .queryContains(
>>>>>> + ElasticChecker.elasticsearchChecker("\"query\" : "
>>>>>> + +
>>>>> "{\"constant_score\":{\"filter\":{\"bool\":"
>>>>>> + +
>>>>> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
>>>>>> + +
>>>>> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
>>>>>> + "\"script_fields\":
>>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>>>>> + +
>>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>>>>> + + "\"city\":{\"script\":
>>>>> \"_source.city\"}, "
>>>>>> + + "\"pop\":{\"script\":
>>>>> \"_source.pop\"}, "
>>>>>> + + "\"state\":{\"script\":
>>>>> \"_source.state\"}, "
>>>>>> + + "\"id\":{\"script\": \"_
>>> source.id
>>>>> \"}}",
>>>>>> + "\"sort\": [ {\"state\": \"asc\"},
>>> {\"id\":
>>>>> \"asc\"}]"))
>>>>>> + .explainContains(explain);
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testFilterSortDesc() {
>>>>>> + final String sql = "select * from zips\n"
>>>>>> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
>>>>>> + + "order by \"state\" desc, \"pop\"";
>>>>>> + calciteAssert()
>>>>>> + .query(sql)
>>>>>> + .limit(4)
>>>>>> + .returnsOrdered(
>>>>>> + "city=LOS ANGELES; longitude=-118.258189;
>>>>> latitude=34.007856; pop=96074; state=CA; id=90011",
>>>>>> + "city=BELL GARDENS; longitude=-118.17205;
>>>>> latitude=33.969177; pop=99568; state=CA; id=90201");
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testFilterRedundant() {
>>>>>> + final String sql = "select * from zips\n"
>>>>>> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and
>>>>> \"state\" = 'OK'";
>>>>>> + calciteAssert()
>>>>>> + .query(sql)
>>>>>> + .runs()
>>>>>> + .queryContains(
>>>>>> + ElasticChecker.elasticsearchChecker(""
>>>>>> + + "\"query\" :
>>>>> {\"constant_score\":{\"filter\":{\"bool\":"
>>>>>> + +
>>>>> "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
>>>>>> + "\"script_fields\":
>>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>>>>> + +
>>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>>>>> + + "\"city\":{\"script\":
>>>>> \"_source.city\"}, "
>>>>>> + + "\"pop\":{\"script\":
>>>>> \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
>>>>>> + + "\"id\":{\"script\":
>>>>> \"_source.id\"}}"
>>>>>> + ));
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testInPlan() {
>>>>>> + final String[] searches = {
>>>>>> + "\"query\" :
>>>>> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
>>>>>> + +
>>>>>
>>> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
>>>>>> + + "{\"pop\":99568}}]}}]}}}}",
>>>>>> + "\"script_fields\":
>>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>>>>> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>>>>> + + "\"city\":{\"script\": \"_source.city\"}, "
>>>>>> + + "\"pop\":{\"script\": \"_source.pop\"}, "
>>>>>> + + "\"state\":{\"script\": \"_source.state\"}, "
>>>>>> + + "\"id\":{\"script\": \"_source.id\"}}"
>>>>>> + };
>>>>>> +
>>>>>> + calciteAssert()
>>>>>> + .query("select * from zips where \"pop\" in (96074,
>>> 99568)")
>>>>>> + .returnsUnordered(
>>>>>> + "city=BELL GARDENS; longitude=-118.17205;
>>>>> latitude=33.969177; pop=99568; state=CA; id=90201",
>>>>>> + "city=LOS ANGELES; longitude=-118.258189;
>>>>> latitude=34.007856; pop=96074; state=CA; id=90011"
>>>>>> + )
>>>>>> +
>>>>> .queryContains(ElasticChecker.elasticsearchChecker(searches));
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testZips() {
>>>>>> + calciteAssert()
>>>>>> + .query("select \"state\", \"city\" from zips")
>>>>>> + .returnsCount(10);
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testProject() {
>>>>>> + final String sql = "select \"state\", \"city\", 0 as \"zero\"\n"
>>>>>> + + "from zips\n"
>>>>>> + + "order by \"state\", \"city\"";
>>>>>> +
>>>>>> + calciteAssert()
>>>>>> + .query(sql)
>>>>>> + .limit(2)
>>>>>> + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0",
>>>>>> + "state=AK; city=FAIRBANKS; zero=0")
>>>>>> + .queryContains(
>>>>>> +
>>>>> ElasticChecker.elasticsearchChecker("\"script_fields\": "
>>>>>> + + "{\"zero\":{\"script\": \"0\"},
>>> "
>>>>>> + + "\"state\":{\"script\":
>>>>> \"_source.state\"}, "
>>>>>> + + "\"city\":{\"script\":
>>>>> \"_source.city\"}}",
>>>>>> + "\"sort\": [ {\"state\": \"asc\"},
>>>>> {\"city\": \"asc\"}]"));
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testFilter() {
>>>>>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>>>>> + + " ElasticsearchProject(state=[CAST(ITEM($0,
>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>> \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20)
>>>>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>>>>> + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0,
>>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>>> \"ISO-8859-1$en_US$primary\", 'CA')])\n"
>>>>>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
>>>>>> + calciteAssert()
>>>>>> + .query("select \"state\", \"city\" from zips where
>>>>> \"state\" = 'CA'")
>>>>>> + .limit(3)
>>>>>> + .returnsUnordered("state=CA; city=BELL GARDENS",
>>>>>> + "state=CA; city=LOS ANGELES",
>>>>>> + "state=CA; city=NORWALK")
>>>>>> + .explainContains(explain);
>>>>>> + }
>>>>>> +
>>>>>> + @Test public void testFilterReversed() {
>>>>>> + calciteAssert()
>>>>>> + .query("select \"state\", \"city\" from zips where 'WI' <
>>>>> \"state\" order by \"city\"")
>>>>>> + .limit(2)
>>>>>> + .returnsUnordered("state=WV; city=BECKLEY",
>>>>>> + "state=WY; city=CHEYENNE");
>>>>>> + calciteAssert()
>>>>>> + .query("select \"state\", \"city\" from zips where \"state\"
>>>>>> 'WI' order by \"city\"")
>>>>>> + .limit(2)
>>>>>> + .returnsUnordered("state=WV; city=BECKLEY",
>>>>>> + "state=WY; city=CHEYENNE");
>>>>>> + }
>>>>>> +
>>>>>> +}
>>>>>> +
>>>>>> +// End ElasticSearch2AdapterTest.java
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git
>>>>>
>>> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>>>>
>>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>>>>> new file mode 100644
>>>>>> index 0000000..4474add
>>>>>> --- /dev/null
>>>>>> +++
>>>>>
>>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>>>>> @@ -0,0 +1,147 @@
>>>>>> +/*
>>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>>>>> + * contributor license agreements. See the NOTICE file distributed
>>> with
>>>>>> + * this work for additional information regarding copyright ownership.
>>>>>> + * The ASF licenses this file to you under the Apache License, Version
>>>>> 2.0
>>>>>> + * (the "License"); you may not use this file except in compliance
>>> with
>>>>>> + * the License. You may obtain a copy of the License at
>>>>>> + *
>>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>>>>> + *
>>>>>> + * Unless required by applicable law or agreed to in writing, software
>>>>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>>>>> implied.
>>>>>> + * See the License for the specific language governing permissions and
>>>>>> + * limitations under the License.
>>>>>> + */
>>>>>> +package org.apache.calcite.adapter.elasticsearch2;
>>>>>> +
>>>>>> +import com.google.common.base.Preconditions;
>>>>>> +import com.google.common.io.Files;
>>>>>> +
>>>>>> +import org.elasticsearch.Version;
>>>>>> +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
>>>>>> +import org.elasticsearch.action.admin.cluster.node.info
>>>>> .NodesInfoResponse;
>>>>>> +import org.elasticsearch.client.Client;
>>>>>> +import org.elasticsearch.common.settings.Settings;
>>>>>> +import org.elasticsearch.common.transport.TransportAddress;
>>>>>> +import org.elasticsearch.node.Node;
>>>>>> +import org.elasticsearch.node.internal.InternalSettingsPreparer;
>>>>>> +import org.elasticsearch.plugins.Plugin;
>>>>>> +import org.elasticsearch.script.groovy.GroovyPlugin;
>>>>>> +
>>>>>> +import java.io.File;
>>>>>> +import java.util.Arrays;
>>>>>> +import java.util.Collection;
>>>>>> +import java.util.Collections;
>>>>>> +
>>>>>> +/**
>>>>>> + * Represents a single elastic search node which can run embedded in a
>>>>> java application.
>>>>>> + * Intended for unit and integration tests. Settings and plugins are
>>>>> crafted for Calcite.
>>>>>> + */
>>>>>> +class EmbeddedElasticNode implements AutoCloseable {
>>>>>> +
>>>>>> + private final LocalNode node;
>>>>>> + private volatile boolean isStarted;
>>>>>> +
>>>>>> + private EmbeddedElasticNode(LocalNode node) {
>>>>>> + this.node = Preconditions.checkNotNull(node, "node");
>>>>>> + }
>>>>>> +
>>>>>> + /**
>>>>>> + * Having separate class to expose (protected) constructor which
>>>>> allows to install
>>>>>> + * different plugins. In our case it is {@code GroovyPlugin} for
>>>>> scripted fields like
>>>>>> + * {@code loc[0]} or {@code loc[1]['foo']}.
>>>>>> + */
>>>>>> + private static class LocalNode extends Node {
>>>>>> + private LocalNode(Settings settings, Collection<Class<? extends
>>>>> Plugin>> classpathPlugins) {
>>>>>> + super(InternalSettingsPreparer.prepareEnvironment(settings,
>>> null),
>>>>>> + Version.CURRENT,
>>>>>> + classpathPlugins);
>>>>>> + }
>>>>>> + }
>>>>>> +
>>>>>> + /**
>>>>>> + * Creates an instance with existing settings
>>>>>> + */
>>>>>> + private static EmbeddedElasticNode create(Settings settings) {
>>>>>> + // ensure GroovyPlugin is installed or otherwise scripted fields
>>>>> would not work
>>>>>> + LocalNode node = new LocalNode(settings,
>>>>> Collections.singleton(GroovyPlugin.class));
>>>>>> + return new EmbeddedElasticNode(node);
>>>>>> + }
>>>>>> +
>>>>>> + /**
>>>>>> + * Creates elastic node as single member of a cluster. Node will not
>>>>> be started
>>>>>> + * unless {@link #start()} is explicitly called.
>>>>>> + */
>>>>>> + public static EmbeddedElasticNode create() {
>>>>>> + File data = Files.createTempDir();
>>>>>> + data.deleteOnExit();
>>>>>> + File home = Files.createTempDir();
>>>>>> + home.deleteOnExit();
>>>>>> +
>>>>>> + Settings settings = Settings.builder()
>>>>>> + .put("node.name", "fake-elastic")
>>>>>> + .put("path.home", home.getAbsolutePath())
>>>>>> + .put("path.data", data.getAbsolutePath())
>>>>>> + .put("script.inline", true) // requires GroovyPlugin
>>>>>> + .put("script.indexed", true) // requires GroovyPlugin
>>>>>> + .put("cluster.routing.allocation.disk.threshold_enabled",
>>>>> false)
>>>>>> + .put("node.local", true)
>>>>>> + .put("node.data", true)
>>>>>> + .put("network.host", "localhost")
>>>>>> + .build();
>>>>>> +
>>>>>> + return create(settings);
>>>>>> + }
>>>>>> +
>>>>>> + /**
>>>>>> + * Starts current node
>>>>>> + */
>>>>>> + public void start() {
>>>>>> + Preconditions.checkState(!isStarted, "already started");
>>>>>> + node.start();
>>>>>> + this.isStarted = true;
>>>>>> + }
>>>>>> +
>>>>>> + /**
>>>>>> + * Returns current address to connect to with HTTP client.
>>>>>> + */
>>>>>> + public TransportAddress httpAddress() {
>>>>>> + Preconditions.checkState(isStarted, "node is not started");
>>>>>> +
>>>>>> + NodesInfoResponse response =
>>>>> client().admin().cluster().prepareNodesInfo()
>>>>>> + .execute().actionGet();
>>>>>> + if (response.getNodes().length != 1) {
>>>>>> + throw new IllegalStateException("Expected single node but got "
>>>>>> + + response.getNodes().length);
>>>>>> + }
>>>>>> + NodeInfo node = response.getNodes()[0];
>>>>>> + return node.getHttp().address().boundAddresses()[0];
>>>>>> + }
>>>>>> +
>>>>>> + /**
>>>>>> + * Exposes elastic
>>>>>> + * <a href="
>>>>>
>>> https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html
>>> ">transport
>>>>> client</a>
>>>>>> + *
>>>>>> + * (use of HTTP client is preferred).
>>>>>> + */
>>>>>> + public Client client() {
>>>>>> + Preconditions.checkState(isStarted, "node is not started");
>>>>>> + return node.client();
>>>>>> + }
>>>>>> +
>>>>>> + @Override public void close() throws Exception {
>>>>>> + node.close();
>>>>>> + // cleanup data dirs
>>>>>> + File data = new File(node.settings().get("path.data"));
>>>>>> + File home = new File(node.settings().get("path.home"));
>>>>>> + for (File file: Arrays.asList(data, home)) {
>>>>>> + if (file.exists()) {
>>>>>> + file.delete();
>>>>>> + }
>>>>>> + }
>>>>>> + }
>>>>>> +}
>>>>>> +
>>>>>> +// End EmbeddedElasticNode.java
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git
>>>>>
>>> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>>>>
>>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>>>>> new file mode 100644
>>>>>> index 0000000..a633078
>>>>>> --- /dev/null
>>>>>> +++
>>>>>
>>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>>>>> @@ -0,0 +1,97 @@
>>>>>> +/*
>>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>>>>> + * contributor license agreements. See the NOTICE file distributed
>>> with
>>>>>> + * this work for additional information regarding copyright ownership.
>>>>>> + * The ASF licenses this file to you under the Apache License, Version
>>>>> 2.0
>>>>>> + * (the "License"); you may not use this file except in compliance
>>> with
>>>>>> + * the License. You may obtain a copy of the License at
>>>>>> + *
>>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>>>>> + *
>>>>>> + * Unless required by applicable law or agreed to in writing, software
>>>>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>>>>> implied.
>>>>>> + * See the License for the specific language governing permissions and
>>>>>> + * limitations under the License.
>>>>>> + */
>>>>>> +package org.apache.calcite.adapter.elasticsearch2;
>>>>>> +
>>>>>> +import com.google.common.base.Preconditions;
>>>>>> +
>>>>>> +import org.elasticsearch.client.Client;
>>>>>> +import org.elasticsearch.common.transport.TransportAddress;
>>>>>> +import org.junit.rules.ExternalResource;
>>>>>> +
>>>>>> +/**
>>>>>> + * Used to initialize a single elastic node. For performance reasons
>>>>> (node startup costs),
>>>>>> + * same instance is usually shared across multiple tests.
>>>>>> + *
>>>>>> + * This rule should be used as follows:
>>>>>> + * <pre>
>>>>>> + * {@code
>>>>>> + *
>>>>>> + * public class MyTest {
>>>>>> + * @literal @ClassRule
>>>>>> + * public static final ElasticSearchRule RULE =
>>>>> ElasticSearchRule.create();
>>>>>> + *
>>>>>> + * @literal @BeforeClass
>>>>>> + * public void setup() {
>>>>>> + * // ... populate instance
>>>>>> + * }
>>>>>> + *
>>>>>> + * @literal @Test
>>>>>> + * public void myTest() {
>>>>>> + * TransportAddress address = RULE.httpAddress();
>>>>>> + * // ....
>>>>>> + * }
>>>>>> + * }
>>>>>> + * }
>>>>>> + * </pre>
>>>>>> + *
>>>>>> + * @see ExternalResource
>>>>>> + */
>>>>>> +class EmbeddedElasticRule extends ExternalResource {
>>>>>> +
>>>>>> + private final EmbeddedElasticNode node;
>>>>>> +
>>>>>> + private EmbeddedElasticRule(EmbeddedElasticNode resource) {
>>>>>> + this.node = Preconditions.checkNotNull(resource, "resource");
>>>>>> + }
>>>>>> +
>>>>>> + @Override protected void before() throws Throwable {
>>>>>> + node.start();
>>>>>> + }
>>>>>> +
>>>>>> + @Override protected void after() {
>>>>>> + try {
>>>>>> + node.close();
>>>>>> + } catch (Exception e) {
>>>>>> + throw new RuntimeException(e);
>>>>>> + }
>>>>>> + }
>>>>>> +
>>>>>> + /**
>>>>>> + * Factory method to create this rule.
>>>>>> + */
>>>>>> + public static EmbeddedElasticRule create() {
>>>>>> + return new EmbeddedElasticRule(EmbeddedElasticNode.create());
>>>>>> + }
>>>>>> +
>>>>>> + /**
>>>>>> + * Exposes current ES transport client.
>>>>>> + */
>>>>>> + Client client() {
>>>>>> + return node.client();
>>>>>> + }
>>>>>> +
>>>>>> + /**
>>>>>> + * HTTP address for rest clients (can be ES native or any other).
>>>>>> + */
>>>>>> + TransportAddress httpAddress() {
>>>>>> + return node.httpAddress();
>>>>>> + }
>>>>>> +
>>>>>> +
>>>>>> +}
>>>>>> +
>>>>>> +// End EmbeddedElasticRule.java
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git
>>>>>
>>> a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>>>>
>>> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>>>>> new file mode 100644
>>>>>> index 0000000..21fc491
>>>>>> --- /dev/null
>>>>>> +++
>>>>>
>>> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>>>>> @@ -0,0 +1,49 @@
>>>>>> +/*
>>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>>>>> + * contributor license agreements. See the NOTICE file distributed
>>> with
>>>>>> + * this work for additional information regarding copyright ownership.
>>>>>> + * The ASF licenses this file to you under the Apache License, Version
>>>>> 2.0
>>>>>> + * (the "License"); you may not use this file except in compliance
>>> with
>>>>>> + * the License. You may obtain a copy of the License at
>>>>>> + *
>>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>>>>> + *
>>>>>> + * Unless required by applicable law or agreed to in writing, software
>>>>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>>>>> implied.
>>>>>> + * See the License for the specific language governing permissions and
>>>>>> + * limitations under the License.
>>>>>> + */
>>>>>> +package org.apache.calcite.test;
>>>>>> +
>>>>>> +import com.google.common.base.Function;
>>>>>> +
>>>>>> +import java.util.List;
>>>>>> +
>>>>>> +import javax.annotation.Nullable;
>>>>>> +
>>>>>> +/**
>>>>>> + * Internal util methods for ElasticSearch tests
>>>>>> + */
>>>>>> +public class ElasticChecker {
>>>>>> +
>>>>>> + private ElasticChecker() {}
>>>>>> +
>>>>>> +
>>>>>> + /** Returns a function that checks that a particular Elasticsearch
>>>>> pipeline is
>>>>>> + * generated to implement a query. */
>>>>>> + public static Function<List, Void> elasticsearchChecker(final
>>>>> String... strings) {
>>>>>> + return new Function<List, Void>() {
>>>>>> + @Nullable
>>>>>> + @Override public Void apply(@Nullable List actual) {
>>>>>> + Object[] actualArray = actual == null || actual.isEmpty() ?
>>> null
>>>>>> + : ((List) actual.get(0)).toArray();
>>>>>> + CalciteAssert.assertArrayEqual("expected Elasticsearch query
>>>>> not found", strings,
>>>>>> + actualArray);
>>>>>> + return null;
>>>>>> + }
>>>>>> + };
>>>>>> + }
>>>>>> +}
>>>>>> +
>>>>>> +// End ElasticChecker.java
>>>>>>
>>>>>>
>>>>>
>>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>>>>> ----------------------------------------------------------------------
>>>>>> diff --git
>>>>>
>>> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>>>>
>>> b/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>>>>> deleted file mode 100644
>>>>>> index 4e0c2b6..0000000
>>>>>> ---
>>>>>
>>> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>>>>> +++ /dev/null
>>>>>> @@ -1,270 +0,0 @@
>>>>>> -/*
>>>>>> - * Licensed to the Apache Software Foundation (ASF) under one or more
>>>>>> - * contributor license agreements. See the NOTICE file distributed
>>> with
>>>>>> - * this work for additional information regarding copyright ownership.
>>>>>> - * The ASF licenses this file to you under the Apache License, Version
>>>>> 2.0
>>>>>> - * (the "License"); you may not use this file except in compliance
>>> with
>>>>>> - * the License. You may obtain a copy of the License at
>>>>>> - *
>>>>>> - * http://www.apache.org/licenses/LICENSE-2.0
>>>>>> - *
>>>>>> - * Unless required by applicable law or agreed to in writing, software
>>>>>> - * distributed under the License is distributed on an "AS IS" BASIS,
>>>>>> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>>>>> implied.
>>>>>> - * See the License for the specific language governing permissions and
>>>>>> - * limitations under the License.
>>>>>> - */
>>>>>> -package org.apache.calcite.test;
>>>>>> -
>>>>>> -import org.apache.calcite.util.Util;
>>>>>> -
>>>>>> -import com.google.common.base.Function;
>>>>>> -import com.google.common.collect.ImmutableMap;
>>>>>> -
>>>>>> -import org.junit.Test;
>>>>>> -
>>>>>> -import java.util.List;
>>>>>> -import javax.annotation.Nullable;
>>>>>> -
>>>>>> -/**
>>>>>> - * Tests for the {@code org.apache.calcite.adapter.elasticsearch2}
>>>>> package.
>>>>>> - *
>>>>>> - * <p>Before calling this test, you need to populate Elasticsearch, as
>>>>> follows:
>>>>>> - *
>>>>>> - * <blockquote><code>
>>>>>> - * git clone https://github.com/vlsi/calcite-test-dataset<br>
>>>>>> - * cd calcite-test-dataset<br>
>>>>>> - * mvn install
>>>>>> - * </code></blockquote>
>>>>>> - *
>>>>>> - * <p>This will create a virtual machine with Elastic
>
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Julian Hyde <jh...@apache.org>.
On JDK 8/ubuntu, “mvn clean -DskipTests site” gives the following failure:
[ERROR] Failed to execute goal org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on project calcite-mongodb: Error generating maven-javadoc-plugin:3.0.1:test-javadoc report:
[ERROR] Exit code: 1 - /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:22: error: cannot find symbol
[ERROR] import org.apache.calcite.test.CalciteAssert;
[ERROR] ^
[ERROR] symbol: class CalciteAssert
[ERROR] location: package org.apache.calcite.test
[ERROR] /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:137: error: package CalciteAssert does not exist
[ERROR] private CalciteAssert.AssertThat assertModel(String model) {
[ERROR] ^
[ERROR] /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/adapter/mongodb/MongoAdapterTest.java:145: error: package CalciteAssert does not exist
[ERROR] private CalciteAssert.AssertThat assertModel(URL url) {
[ERROR] ^
[ERROR] /home/jhyde/open1/calcite.4/mongodb/src/test/java/org/apache/calcite/test/MongoAssertions.java:43: error: reference not found
[ERROR] /** Similar to {@link CalciteAssert#checkResultUnordered}, but filters strings
[ERROR] ^
> On Jun 22, 2018, at 3:46 PM, Andrei Sereda <an...@sereda.cc> wrote:
>
> Hmm. Both master and c12cb4b0de work for me. Perhaps the issue is JDK 8 vs
> 9 ?
>
> # JDK8 (oracle) / macOS
> $ mvn clean -DskipTests site
>
> On Fri, Jun 22, 2018 at 5:56 PM Julian Hyde <jh...@apache.org> wrote:
>
>> I just tried it again. The following fails for me (jdk9, ubuntu):
>>
>> $ git checkout c12cb4b0de
>> $ mvn clean -DskipTests site
>>
>> Constructing Javadoc information...
>> 1 error
>> [INFO]
>> ------------------------------------------------------------------------
>> [INFO] Reactor Summary:
>> [INFO]
>> [INFO] Calcite 1.17.0-SNAPSHOT ............................ FAILURE [02:29
>> min]
>> [INFO] Calcite Linq4j ..................................... SKIPPED
>> [INFO] Calcite Core ....................................... SKIPPED
>> [INFO] Calcite Cassandra .................................. SKIPPED
>> [INFO] Calcite Druid ...................................... SKIPPED
>> [INFO] Calcite Elasticsearch .............................. SKIPPED
>> [INFO] Calcite Elasticsearch5 ............................. SKIPPED
>> [INFO] Calcite Examples ................................... SKIPPED
>> [INFO] Calcite Example CSV ................................ SKIPPED
>> [INFO] Calcite Example Function ........................... SKIPPED
>> [INFO] Calcite File ....................................... SKIPPED
>> [INFO] Calcite Geode ...................................... SKIPPED
>> [INFO] Calcite MongoDB .................................... SKIPPED
>> [INFO] Calcite Pig ........................................ SKIPPED
>> [INFO] Calcite Piglet ..................................... SKIPPED
>> [INFO] Calcite Plus ....................................... SKIPPED
>> [INFO] Calcite Server ..................................... SKIPPED
>> [INFO] Calcite Spark ...................................... SKIPPED
>> [INFO] Calcite Splunk ..................................... SKIPPED
>> [INFO] Calcite Ubenchmark 1.17.0-SNAPSHOT ................. SKIPPED
>> [INFO]
>> ------------------------------------------------------------------------
>> [INFO] BUILD FAILURE
>> [INFO]
>> ------------------------------------------------------------------------
>> [INFO] Total time: 02:29 min
>> [INFO] Finished at: 2018-06-22T14:53:13-07:00
>> [INFO]
>> ------------------------------------------------------------------------
>> [ERROR] Failed to execute goal
>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
>> project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate
>> report:
>> [ERROR] Exit code: 1 -
>> /home/jhyde/open1/calcite.4/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
>> error: package org.elasticsearch.node.internal does not exist
>> [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
>> [ERROR] ^
>> [ERROR]
>> [ERROR] Command line was: /usr/lib/jvm/jdk9/bin/javadoc @options @packages
>> @argfile
>> [ERROR]
>> [ERROR] Refer to the generated Javadoc files in
>> '/home/jhyde/open1/calcite.4/target/site/testapidocs' dir.
>> [ERROR] -> [Help 1]
>> [ERROR]
>> [ERROR] To see the full stack trace of the errors, re-run Maven with the
>> -e switch.
>> [ERROR] Re-run Maven using the -X switch to enable full debug logging.
>> [ERROR]
>> [ERROR] For more information about the errors and possible solutions,
>> please read the following articles:
>> [ERROR] [Help 1]
>> http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException
>>
>>
>>> On Jun 22, 2018, at 2:41 PM, Michael Mior <mm...@apache.org> wrote:
>>>
>>> Odd, mvn clean site still works fine for me.
>>> --
>>> Michael Mior
>>> mmior@apache.org
>>>
>>>
>>>
>>> Le ven. 22 juin 2018 à 17:12, Julian Hyde <jh...@apache.org> a écrit :
>>>
>>>> Looks like this change broke “mvn site” (perhaps also “mvn
>>>> javadoc:test-javadoc”).
>>>>
>>>> [ERROR] Failed to execute goal
>>>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
>>>> project calcite: Error generating
>> maven-javadoc-plugin:3.0.1:test-aggregate
>>>> report:
>>>> [ERROR] Exit code: 1 -
>>>>
>> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
>>>> error: package org.elasticsearch.node.internal does not exist
>>>> [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
>>>> [ERROR] ^
>>>> [ERROR]
>>>>
>>>>> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
>>>>>
>>>>> [CALCITE-2347] running ElasticSearch in embedded mode for unit tests of
>>>> ES adapter (Andrei Sereda)
>>>>>
>>>>> After discussion on dev-list Integration tests (for ES) have been
>>>> removed. They're now
>>>>> superseded by unit tests (which execute queries against a real elastic
>>>> instance)
>>>>>
>>>>> Added local file (zips-mini.json) which contains a small subset of
>>>> original zips.json
>>>>> (allows to bootstrap tests faster)
>>>>>
>>>>> Created separate ES JUnit rule which can be re-used across different
>>>> tests.
>>>>>
>>>>> Both v2 and v5 of ES adapters are supported.
>>>>>
>>>>> Close apache/calcite#716
>>>>>
>>>>>
>>>>> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
>>>>> Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
>>>>> Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
>>>>> Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
>>>>>
>>>>> Branch: refs/heads/master
>>>>> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
>>>>> Parents: 37944bb
>>>>> Author: Andrei Sereda <an...@nospam.com>
>>>>> Authored: Thu May 31 18:19:10 2018 -0400
>>>>> Committer: Michael Mior <mm...@uwaterloo.ca>
>>>>> Committed: Thu Jun 21 06:38:50 2018 -0400
>>>>>
>>>>> ----------------------------------------------------------------------
>>>>> .../AbstractElasticsearchTable.java | 12 +
>>>>> .../elasticsearch/ElasticsearchProject.java | 61 ++-
>>>>> elasticsearch2/pom.xml | 6 +
>>>>> .../Elasticsearch2Enumerator.java | 12 +-
>>>>> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
>>>>> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
>>>>> .../ElasticSearch2AdapterTest.java | 395
>> ++++++++++++++++++
>>>>> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
>>>>> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
>>>>> .../calcite/test/Elasticsearch2AdapterIT.java | 270 -------------
>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
>>>>> elasticsearch5/pom.xml | 31 ++
>>>>> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
>>>>> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
>>>>> .../ElasticSearch5AdapterTest.java | 399
>> +++++++++++++++++++
>>>>> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
>>>>> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
>>>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
>>>>> .../calcite/test/Elasticsearch5AdapterIT.java | 270 -------------
>>>>> .../resources/elasticsearch-zips-model.json | 50 ---
>>>>> elasticsearch5/src/test/resources/log4j2.xml | 16 +
>>>>> .../src/test/resources/zips-mini.json | 149 +++++++
>>>>> pom.xml | 20 +-
>>>>> 25 files changed, 1866 insertions(+), 670 deletions(-)
>>>>> ----------------------------------------------------------------------
>>>>>
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>> ----------------------------------------------------------------------
>>>>> diff --git
>>>>
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>> index 0980469..8cc5933 100644
>>>>> ---
>>>>
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>> +++
>>>>
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>>>> @@ -75,6 +75,18 @@ public abstract class AbstractElasticsearchTable
>>>> extends AbstractQueryableTable
>>>>> relOptTable, this, null);
>>>>> }
>>>>>
>>>>> + /**
>>>>> + * In ES 5.x scripted fields start with {@code params._source.foo}
>>>> while in ES2.x
>>>>> + * {@code _source.foo}. Helper method to build correct query based
>> on
>>>> runtime version of elastic.
>>>>> + *
>>>>> + * @see <a href="
>>>> https://github.com/elastic/elasticsearch/issues/20068">_source
>>>> variable</a>
>>>>> + * @see <a href="
>>>>
>> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html
>> ">Scripted
>>>> Fields</a>
>>>>> + */
>>>>> + protected String scriptedFieldPrefix() {
>>>>> + // this is default pattern starting 5.x
>>>>> + return "params._source";
>>>>> + }
>>>>> +
>>>>> /** Executes a "find" operation on the underlying type.
>>>>> *
>>>>> * <p>For example,
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>> ----------------------------------------------------------------------
>>>>> diff --git
>>>>
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>> index b42abd7..961c8b0 100644
>>>>> ---
>>>>
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>> +++
>>>>
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>>>> @@ -27,11 +27,15 @@ import
>>>> org.apache.calcite.rel.metadata.RelMetadataQuery;
>>>>> import org.apache.calcite.rel.type.RelDataType;
>>>>> import org.apache.calcite.rex.RexNode;
>>>>> import org.apache.calcite.util.Pair;
>>>>> -import org.apache.calcite.util.Util;
>>>>> +
>>>>> +import com.google.common.base.Function;
>>>>> +import com.google.common.collect.Lists;
>>>>>
>>>>> import java.util.ArrayList;
>>>>> import java.util.List;
>>>>>
>>>>> +import javax.annotation.Nullable;
>>>>> +
>>>>> /**
>>>>> * Implementation of {@link org.apache.calcite.rel.core.Project}
>>>>> * relational expression in Elasticsearch.
>>>>> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends Project
>>>> implements ElasticsearchRel {
>>>>> implementor.visitChild(0, getInput());
>>>>>
>>>>> final List<String> inFields =
>>>>> -
>>>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
>>>>> +
>>>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
>>>>> final ElasticsearchRules.RexToElasticsearchTranslator translator =
>>>>> - new ElasticsearchRules.RexToElasticsearchTranslator(
>>>>> - (JavaTypeFactory) getCluster().getTypeFactory(),
>> inFields);
>>>>> + new ElasticsearchRules.RexToElasticsearchTranslator(
>>>>> + (JavaTypeFactory) getCluster().getTypeFactory(),
>>>> inFields);
>>>>>
>>>>> - final List<String> findItems = new ArrayList<>();
>>>>> - final List<String> scriptFieldItems = new ArrayList<>();
>>>>> + final List<String> fields = new ArrayList<>();
>>>>> + final List<String> scriptFields = new ArrayList<>();
>>>>> for (Pair<RexNode, String> pair: getNamedProjects()) {
>>>>> final String name = pair.right;
>>>>> final String expr = pair.left.accept(translator);
>>>>>
>>>>> if (expr.equals("\"" + name + "\"")) {
>>>>> - findItems.add(ElasticsearchRules.quote(name));
>>>>> + fields.add(name);
>>>>> } else if (expr.matches("\"literal\":.+")) {
>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
>>>>> - + ":{\"script\": "
>>>>> - + expr.split(":")[1] + "}");
>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
>>>>> + + ":{\"script\": "
>>>>> + + expr.split(":")[1] + "}");
>>>>> } else {
>>>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
>>>>> - + ":{\"script\":\"params._source."
>>>>> - + expr.replaceAll("\"", "") + "\"}");
>>>>> + scriptFields.add(ElasticsearchRules.quote(name)
>>>>> + + ":{\"script\":"
>>>>> + // _source (ES2) vs params._source (ES5)
>>>>> + + "\"" +
>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
>>>>> + + expr.replaceAll("\"", "") + "\"}");
>>>>> + }
>>>>> + }
>>>>> +
>>>>> + StringBuilder query = new StringBuilder();
>>>>> + if (scriptFields.isEmpty()) {
>>>>> + List<String> newList = Lists.transform(fields, new
>>>> Function<String, String>() {
>>>>> + @Nullable
>>>>> + @Override public String apply(@Nullable String input) {
>>>>> + return ElasticsearchRules.quote(input);
>>>>> + }
>>>>> + });
>>>>> +
>>>>> + final String findString = String.join(", ", newList);
>>>>> + query.append("\"_source\" : [").append(findString).append("]");
>>>>> + } else {
>>>>> + // if scripted fields are present, ES ignores _source attribute
>>>>> + for (String field: fields) {
>>>>> + scriptFields.add(ElasticsearchRules.quote(field) +
>>>> ":{\"script\": "
>>>>> + // _source (ES2) vs params._source (ES5)
>>>>> + + "\"" +
>>>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
>>>>> + + field + "\"}");
>>>>> }
>>>>> + query.append("\"script_fields\": {" + String.join(", ",
>>>> scriptFields) + "}");
>>>>> }
>>>>> - final String findString = Util.toString(findItems, "", ", ", "");
>>>>> - final String scriptFieldString = "\"script_fields\": {"
>>>>> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
>>>>> - final String fieldString = "\"_source\" : [" + findString + "]"
>>>>> - + ", " + scriptFieldString;
>>>>>
>>>>> for (String opfield : implementor.list) {
>>>>> if (opfield.startsWith("\"_source\"")) {
>>>>> implementor.list.remove(opfield);
>>>>> }
>>>>> }
>>>>> - implementor.add(fieldString);
>>>>> + implementor.add(query.toString());
>>>>> }
>>>>> }
>>>>>
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
>>>>> ----------------------------------------------------------------------
>>>>> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
>>>>> index f24622c..6fbee03 100644
>>>>> --- a/elasticsearch2/pom.xml
>>>>> +++ b/elasticsearch2/pom.xml
>>>>> @@ -73,6 +73,12 @@ limitations under the License.
>>>>> <version>${elasticsearch-java-driver.version}</version>
>>>>> </dependency>
>>>>> <dependency>
>>>>> + <!-- Lang groovy dependency is needed for testing with embedded
>>>> ES (scripted fields like loc[0]) -->
>>>>> + <groupId>org.elasticsearch.module</groupId>
>>>>> + <artifactId>lang-groovy</artifactId>
>>>>> + <scope>test</scope>
>>>>> + </dependency>
>>>>> + <dependency>
>>>>> <groupId>com.carrotsearch</groupId>
>>>>> <artifactId>hppc</artifactId>
>>>>> <version>${hppc.version}</version>
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>> ----------------------------------------------------------------------
>>>>> diff --git
>>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>> index 84370ab..c3d2ac0 100644
>>>>> ---
>>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>> +++
>>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>>>> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
>>>>> import java.util.Date;
>>>>> import java.util.Iterator;
>>>>> import java.util.List;
>>>>> +import java.util.Locale;
>>>>> import java.util.Map;
>>>>>
>>>>> /**
>>>>> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator implements
>>>> Enumerator<Object> {
>>>>> private static Function1<SearchHit, Object[]> listGetter(
>>>>> final List<Map.Entry<String, Class>> fields) {
>>>>> return new Function1<SearchHit, Object[]>() {
>>>>> - public Object[] apply(SearchHit searchHitFields) {
>>>>> + public Object[] apply(SearchHit hit) {
>>>>> Object[] objects = new Object[fields.size()];
>>>>> for (int i = 0; i < fields.size(); i++) {
>>>>> final Map.Entry<String, Class> field = fields.get(i);
>>>>> final String name = field.getKey();
>>>>> - if (searchHitFields.fields().isEmpty()) {
>>>>> - objects[i] =
>> convert(searchHitFields.getSource().get(name),
>>>> field.getValue());
>>>>> + if (hit.fields().isEmpty()) {
>>>>> + objects[i] = convert(hit.getSource().get(name),
>>>> field.getValue());
>>>>> + } else if (hit.fields().containsKey(name)) {
>>>>> + objects[i] = convert(hit.field(name).getValue(),
>>>> field.getValue());
>>>>> } else {
>>>>> - objects[i] =
>>>> convert(searchHitFields.field(name).getValue(), field.getValue());
>>>>> + throw new IllegalStateException(
>>>>> + String.format(Locale.getDefault(), "No result for
>>>> %s", field));
>>>>> }
>>>>> }
>>>>> return objects;
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>> ----------------------------------------------------------------------
>>>>> diff --git
>>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>> index 668402b..46e3fc5 100644
>>>>> ---
>>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>> +++
>>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>>>> @@ -22,6 +22,9 @@ import org.apache.calcite.schema.impl.AbstractSchema;
>>>>>
>>>>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
>>>>>
>>>>> +import com.google.common.annotations.VisibleForTesting;
>>>>> +import com.google.common.base.Preconditions;
>>>>> +
>>>>> import com.google.common.collect.ImmutableList;
>>>>> import com.google.common.collect.ImmutableMap;
>>>>>
>>>>> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
>>>> AbstractSchema
>>>>> }
>>>>> }
>>>>>
>>>>> + /**
>>>>> + * Allows schema to be instantiated from existing elastic search
>>>> client.
>>>>> + * This constructor is used in tests.
>>>>> + */
>>>>> + @VisibleForTesting
>>>>> + Elasticsearch2Schema(Client client, String index) {
>>>>> + this.client = Preconditions.checkNotNull(client, "client");
>>>>> + this.index = Preconditions.checkNotNull(index, "index");
>>>>> + }
>>>>> +
>>>>> @Override protected Map<String, Table> getTableMap() {
>>>>> final ImmutableMap.Builder<String, Table> builder =
>>>> ImmutableMap.builder();
>>>>>
>>>>> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
>>>> AbstractSchema
>>>>>
>>>>> final List<DiscoveryNode> nodes =
>>>> ImmutableList.copyOf(transportClient.connectedNodes());
>>>>> if (nodes.isEmpty()) {
>>>>> - throw new RuntimeException("Cannot connect to any elasticsearch
>>>> nodes");
>>>>> + throw new IllegalStateException("Cannot connect to any
>>>> elasticsearch node: "
>>>>> + + transportNodes);
>>>>> }
>>>>>
>>>>> client = transportClient;
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>> ----------------------------------------------------------------------
>>>>> diff --git
>>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>> index 636aa5f..2928835 100644
>>>>> ---
>>>>
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>> +++
>>>>
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>>>> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
>>>> AbstractElasticsearchTable {
>>>>> this.client = client;
>>>>> }
>>>>>
>>>>> + /**
>>>>> + * ES version 2.x. To access document attributes ES2 uses {@code
>>>> _source.foo} syntax.
>>>>> + */
>>>>> + @Override protected String scriptedFieldPrefix() {
>>>>> + return "_source";
>>>>> + }
>>>>> +
>>>>> @Override protected Enumerable<Object> find(String index, List<String>
>>>> ops,
>>>>> - List<Map.Entry<String, Class>> fields) {
>>>>> + List<Map.Entry<String,
>>>> Class>> fields) {
>>>>> final String dbName = index;
>>>>>
>>>>> final String queryString = "{" + Util.toString(ops, "", ", ", "") +
>>>> "}";
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>> ----------------------------------------------------------------------
>>>>> diff --git
>>>>
>> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>> new file mode 100644
>>>>> index 0000000..287e094
>>>>> --- /dev/null
>>>>> +++
>>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>>>> @@ -0,0 +1,395 @@
>>>>> +/*
>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>>>> + * contributor license agreements. See the NOTICE file distributed
>> with
>>>>> + * this work for additional information regarding copyright ownership.
>>>>> + * The ASF licenses this file to you under the Apache License, Version
>>>> 2.0
>>>>> + * (the "License"); you may not use this file except in compliance
>> with
>>>>> + * the License. You may obtain a copy of the License at
>>>>> + *
>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>>>> + *
>>>>> + * Unless required by applicable law or agreed to in writing, software
>>>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>>>> implied.
>>>>> + * See the License for the specific language governing permissions and
>>>>> + * limitations under the License.
>>>>> + */
>>>>> +package org.apache.calcite.adapter.elasticsearch2;
>>>>> +
>>>>> +import org.apache.calcite.jdbc.CalciteConnection;
>>>>> +import org.apache.calcite.schema.SchemaPlus;
>>>>> +import org.apache.calcite.schema.impl.ViewTable;
>>>>> +import org.apache.calcite.schema.impl.ViewTableMacro;
>>>>> +import org.apache.calcite.test.CalciteAssert;
>>>>> +import org.apache.calcite.test.ElasticChecker;
>>>>> +
>>>>> +import com.google.common.io.LineProcessor;
>>>>> +import com.google.common.io.Resources;
>>>>> +
>>>>> +import org.elasticsearch.action.bulk.BulkItemResponse;
>>>>> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
>>>>> +import org.elasticsearch.action.bulk.BulkResponse;
>>>>> +import org.elasticsearch.common.xcontent.XContentBuilder;
>>>>> +import org.elasticsearch.common.xcontent.XContentFactory;
>>>>> +
>>>>> +import org.junit.BeforeClass;
>>>>> +import org.junit.ClassRule;
>>>>> +import org.junit.Test;
>>>>> +
>>>>> +import java.io.IOException;
>>>>> +import java.nio.charset.StandardCharsets;
>>>>> +import java.sql.Connection;
>>>>> +import java.sql.DriverManager;
>>>>> +import java.sql.SQLException;
>>>>> +import java.util.Arrays;
>>>>> +import java.util.Collections;
>>>>> +import java.util.Locale;
>>>>> +
>>>>> +/**
>>>>> + * Set of tests for ES adapter. Uses real instance via {@link
>>>> EmbeddedElasticRule}. Document
>>>>> + * source is local {@code zips-mini.json} file (located in the
>>>> classpath).
>>>>> + */
>>>>> +public class ElasticSearch2AdapterTest {
>>>>> +
>>>>> + @ClassRule //init once for all tests
>>>>> + public static final EmbeddedElasticRule NODE =
>>>> EmbeddedElasticRule.create();
>>>>> +
>>>>> + private static final String ZIPS = "zips";
>>>>> +
>>>>> + /**
>>>>> + * Used to create {@code zips} index and insert some data
>>>>> + */
>>>>> + @BeforeClass
>>>>> + public static void setupInstance() throws Exception {
>>>>> + // define mapping so fields are searchable (term query)
>>>>> + XContentBuilder mapping =
>>>> XContentFactory.jsonBuilder().startObject()
>>>>> + .startObject("properties")
>>>>> + .startObject("city").field("type", "string")
>>>>> + .field("index", "not_analyzed").endObject()
>>>>> + .startObject("state").field("type", "string")
>>>>> + .field("index", "not_analyzed").endObject()
>>>>> + .startObject("pop").field("type", "long").endObject()
>>>>> + .endObject()
>>>>> + .endObject();
>>>>> +
>>>>> + // create index
>>>>> + NODE.client().admin().indices()
>>>>> + .prepareCreate(ZIPS)
>>>>> + .addMapping(ZIPS, mapping)
>>>>> + .get();
>>>>> +
>>>>> + BulkRequestBuilder bulk =
>>>> NODE.client().prepareBulk().setRefresh(true);
>>>>> +
>>>>> + // load records from file
>>>>> +
>>>>
>> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
>>>>> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
>>>>> + @Override public boolean processLine(String line) throws
>>>> IOException {
>>>>> + line = line.replaceAll("_id", "id"); // _id is a
>>>> reserved attribute in ES
>>>>> + bulk.add(NODE.client().prepareIndex(ZIPS,
>>>> ZIPS).setSource(line));
>>>>> + return true;
>>>>> + }
>>>>> +
>>>>> + @Override public Void getResult() {
>>>>> + return null;
>>>>> + }
>>>>> + });
>>>>> +
>>>>> + if (bulk.numberOfActions() == 0) {
>>>>> + throw new IllegalStateException("No records to be indexed");
>>>>> + }
>>>>> +
>>>>> + BulkResponse response = bulk.execute().get();
>>>>> +
>>>>> + if (response.hasFailures()) {
>>>>> + throw new IllegalStateException(
>>>>> + String.format(Locale.getDefault(), "Failed to populate
>>>> %s:\n%s", NODE.httpAddress(),
>>>>> +
>>>> Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
>>>>> +
>>>>
>> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
>>>>> + }
>>>>> +
>>>>> + }
>>>>> +
>>>>> + private CalciteAssert.ConnectionFactory newConnectionFactory() {
>>>>> + return new CalciteAssert.ConnectionFactory() {
>>>>> + @Override public Connection createConnection() throws
>>>> SQLException {
>>>>> + final Connection connection =
>>>> DriverManager.getConnection("jdbc:calcite:");
>>>>> + final SchemaPlus root =
>>>> connection.unwrap(CalciteConnection.class).getRootSchema();
>>>>> +
>>>>> + root.add("elastic", new Elasticsearch2Schema(NODE.client(),
>>>> ZIPS));
>>>>> +
>>>>> + // add calcite view programmatically
>>>>> + final String viewSql = "select cast(_MAP['city'] AS
>>>> varchar(20)) AS \"city\", "
>>>>> + + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n"
>>>>> + + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n"
>>>>> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
>>>>> + + " cast(_MAP['state'] AS varchar(2)) AS \"state\", "
>>>>> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
>>>>> + + "from \"elastic\".\"zips\"";
>>>>> +
>>>>> + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
>>>>> + Collections.singletonList("elastic"),
>>>> Arrays.asList("elastic", "view"), false);
>>>>> + root.add("ZIPS", macro);
>>>>> +
>>>>> + return connection;
>>>>> + }
>>>>> + };
>>>>> + }
>>>>> +
>>>>> + private CalciteAssert.AssertThat calciteAssert() {
>>>>> + return CalciteAssert.that()
>>>>> + .with(newConnectionFactory());
>>>>> + }
>>>>> +
>>>>> + /**
>>>>> + * Tests using calcite view
>>>>> + */
>>>>> + @Test
>>>>> + public void view() throws Exception {
>>>>> + calciteAssert()
>>>>> + .query("select * from zips where \"city\" = 'BROOKLYN'")
>>>>> + .returns("city=BROOKLYN; longitude=-73.956985;
>>>> latitude=40.646694; "
>>>>> + + "pop=111396; state=NY; id=11226\n")
>>>>> + .returnsCount(1);
>>>>> + }
>>>>> +
>>>>> + @Test
>>>>> + public void emptyResult() {
>>>>> + CalciteAssert.that()
>>>>> + .with(newConnectionFactory())
>>>>> + .query("select * from zips limit 0")
>>>>> + .returnsCount(0);
>>>>> +
>>>>> + CalciteAssert.that()
>>>>> + .with(newConnectionFactory())
>>>>> + .query("select * from \"elastic\".\"zips\" where
>>>> _MAP['Foo'] = '_MISSING_'")
>>>>> + .returnsCount(0);
>>>>> + }
>>>>> +
>>>>> + @Test
>>>>> + public void basic() throws Exception {
>>>>> + CalciteAssert.that()
>>>>> + .with(newConnectionFactory())
>>>>> + .query("select * from \"elastic\".\"zips\" where
>>>> _MAP['city'] = 'BROOKLYN'")
>>>>> + .returnsCount(1);
>>>>> +
>>>>> + CalciteAssert.that()
>>>>> + .with(newConnectionFactory())
>>>>> + .query("select * from \"elastic\".\"zips\" where"
>>>>> + + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')")
>>>>> + .returnsCount(2);
>>>>> +
>>>>> + // lower-case
>>>>> + CalciteAssert.that()
>>>>> + .with(newConnectionFactory())
>>>>> + .query("select * from \"elastic\".\"zips\" where "
>>>>> + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
>>>> 'BROOK') ")
>>>>> + .returnsCount(0);
>>>>> +
>>>>> + // missing field
>>>>> + CalciteAssert.that()
>>>>> + .with(newConnectionFactory())
>>>>> + .query("select * from \"elastic\".\"zips\" where
>>>> _MAP['CITY'] = 'BROOKLYN'")
>>>>> + .returnsCount(0);
>>>>> +
>>>>> + // limit works
>>>>> + CalciteAssert.that()
>>>>> + .with(newConnectionFactory())
>>>>> + .query("select * from \"elastic\".\"zips\" limit 42")
>>>>> + .returnsCount(42);
>>>>> +
>>>>> + }
>>>>> +
>>>>> + @Test public void testSort() {
>>>>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>>>> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
>>>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>>>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
>>>>> +
>>>>> + calciteAssert()
>>>>> + .query("select * from zips order by \"state\"")
>>>>> + .returnsCount(10)
>>>>> + .explainContains(explain);
>>>>> + }
>>>>> +
>>>>> + @Test public void testSortLimit() {
>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>>>> + + "order by \"state\", \"id\" offset 2 rows fetch next 3
>>>> rows only";
>>>>> + calciteAssert()
>>>>> + .query(sql)
>>>>> + .returnsUnordered("state=AK; id=99801",
>>>>> + "state=AL; id=35215",
>>>>> + "state=AL; id=35401")
>>>>> + .queryContains(
>>>>> + ElasticChecker.elasticsearchChecker(
>>>>> + "\"_source\" : [\"state\", \"id\"]",
>>>>> + "\"sort\": [ {\"state\": \"asc\"},
>> {\"id\":
>>>> \"asc\"}]",
>>>>> + "\"from\": 2",
>>>>> + "\"size\": 3"));
>>>>> + }
>>>>> +
>>>>> +
>>>>> +
>>>>> + @Test public void testOffsetLimit() {
>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>>>> + + "offset 2 fetch next 3 rows only";
>>>>> + calciteAssert()
>>>>> + .query(sql)
>>>>> + .runs()
>>>>> + .queryContains(
>>>>> + ElasticChecker.elasticsearchChecker(
>>>>> + "\"from\": 2",
>>>>> + "\"size\": 3",
>>>>> + "\"_source\" : [\"state\", \"id\"]"));
>>>>> + }
>>>>> +
>>>>> + @Test public void testLimit() {
>>>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>>>> + + "fetch next 3 rows only";
>>>>> +
>>>>> + calciteAssert()
>>>>> + .query(sql)
>>>>> + .runs()
>>>>> + .queryContains(
>>>>> + ElasticChecker.elasticsearchChecker(
>>>>> + "\"size\": 3",
>>>>> + "\"_source\" : [\"state\", \"id\"]"));
>>>>> + }
>>>>> +
>>>>> + @Test public void testFilterSort() {
>>>>> + final String sql = "select * from zips\n"
>>>>> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
>>>>> + + "order by \"state\", \"id\"";
>>>>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>>>> + + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC],
>>>> dir1=[ASC])\n"
>>>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
>>>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
>>>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
>>>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
>>>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>>>> + + "
>> ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>> \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0, 'id')):VARCHAR(5)
>>>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\",
>>>> '70000'))])\n"
>>>>> + + " ElasticsearchTableScan(table=[[elastic,
>> zips]])";
>>>>> + calciteAssert()
>>>>> + .query(sql)
>>>>> + .returnsOrdered("city=LOS ANGELES; longitude=-118.258189;
>>>> latitude=34.007856; "
>>>>> + + "pop=96074; state=CA; id=90011",
>>>>> + "city=BELL GARDENS; longitude=-118.17205;
>>>> latitude=33.969177; "
>>>>> + + "pop=99568; state=CA; id=90201",
>>>>> + "city=NORWALK; longitude=-118.081767;
>>>> latitude=33.90564; "
>>>>> + + "pop=94188; state=CA; id=90650")
>>>>> + .queryContains(
>>>>> + ElasticChecker.elasticsearchChecker("\"query\" : "
>>>>> + +
>>>> "{\"constant_score\":{\"filter\":{\"bool\":"
>>>>> + +
>>>> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
>>>>> + +
>>>> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
>>>>> + "\"script_fields\":
>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>>>> + +
>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>>>> + + "\"city\":{\"script\":
>>>> \"_source.city\"}, "
>>>>> + + "\"pop\":{\"script\":
>>>> \"_source.pop\"}, "
>>>>> + + "\"state\":{\"script\":
>>>> \"_source.state\"}, "
>>>>> + + "\"id\":{\"script\": \"_
>> source.id
>>>> \"}}",
>>>>> + "\"sort\": [ {\"state\": \"asc\"},
>> {\"id\":
>>>> \"asc\"}]"))
>>>>> + .explainContains(explain);
>>>>> + }
>>>>> +
>>>>> + @Test public void testFilterSortDesc() {
>>>>> + final String sql = "select * from zips\n"
>>>>> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
>>>>> + + "order by \"state\" desc, \"pop\"";
>>>>> + calciteAssert()
>>>>> + .query(sql)
>>>>> + .limit(4)
>>>>> + .returnsOrdered(
>>>>> + "city=LOS ANGELES; longitude=-118.258189;
>>>> latitude=34.007856; pop=96074; state=CA; id=90011",
>>>>> + "city=BELL GARDENS; longitude=-118.17205;
>>>> latitude=33.969177; pop=99568; state=CA; id=90201");
>>>>> + }
>>>>> +
>>>>> + @Test public void testFilterRedundant() {
>>>>> + final String sql = "select * from zips\n"
>>>>> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and
>>>> \"state\" = 'OK'";
>>>>> + calciteAssert()
>>>>> + .query(sql)
>>>>> + .runs()
>>>>> + .queryContains(
>>>>> + ElasticChecker.elasticsearchChecker(""
>>>>> + + "\"query\" :
>>>> {\"constant_score\":{\"filter\":{\"bool\":"
>>>>> + +
>>>> "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
>>>>> + "\"script_fields\":
>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>>>> + +
>>>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>>>> + + "\"city\":{\"script\":
>>>> \"_source.city\"}, "
>>>>> + + "\"pop\":{\"script\":
>>>> \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
>>>>> + + "\"id\":{\"script\":
>>>> \"_source.id\"}}"
>>>>> + ));
>>>>> + }
>>>>> +
>>>>> + @Test public void testInPlan() {
>>>>> + final String[] searches = {
>>>>> + "\"query\" :
>>>> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
>>>>> + +
>>>>
>> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
>>>>> + + "{\"pop\":99568}}]}}]}}}}",
>>>>> + "\"script_fields\":
>>>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>>>> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>>>> + + "\"city\":{\"script\": \"_source.city\"}, "
>>>>> + + "\"pop\":{\"script\": \"_source.pop\"}, "
>>>>> + + "\"state\":{\"script\": \"_source.state\"}, "
>>>>> + + "\"id\":{\"script\": \"_source.id\"}}"
>>>>> + };
>>>>> +
>>>>> + calciteAssert()
>>>>> + .query("select * from zips where \"pop\" in (96074,
>> 99568)")
>>>>> + .returnsUnordered(
>>>>> + "city=BELL GARDENS; longitude=-118.17205;
>>>> latitude=33.969177; pop=99568; state=CA; id=90201",
>>>>> + "city=LOS ANGELES; longitude=-118.258189;
>>>> latitude=34.007856; pop=96074; state=CA; id=90011"
>>>>> + )
>>>>> +
>>>> .queryContains(ElasticChecker.elasticsearchChecker(searches));
>>>>> + }
>>>>> +
>>>>> + @Test public void testZips() {
>>>>> + calciteAssert()
>>>>> + .query("select \"state\", \"city\" from zips")
>>>>> + .returnsCount(10);
>>>>> + }
>>>>> +
>>>>> + @Test public void testProject() {
>>>>> + final String sql = "select \"state\", \"city\", 0 as \"zero\"\n"
>>>>> + + "from zips\n"
>>>>> + + "order by \"state\", \"city\"";
>>>>> +
>>>>> + calciteAssert()
>>>>> + .query(sql)
>>>>> + .limit(2)
>>>>> + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0",
>>>>> + "state=AK; city=FAIRBANKS; zero=0")
>>>>> + .queryContains(
>>>>> +
>>>> ElasticChecker.elasticsearchChecker("\"script_fields\": "
>>>>> + + "{\"zero\":{\"script\": \"0\"},
>> "
>>>>> + + "\"state\":{\"script\":
>>>> \"_source.state\"}, "
>>>>> + + "\"city\":{\"script\":
>>>> \"_source.city\"}}",
>>>>> + "\"sort\": [ {\"state\": \"asc\"},
>>>> {\"city\": \"asc\"}]"));
>>>>> + }
>>>>> +
>>>>> + @Test public void testFilter() {
>>>>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>>>> + + " ElasticsearchProject(state=[CAST(ITEM($0,
>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>> \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20)
>>>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>>>> + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0,
>>>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>>>> \"ISO-8859-1$en_US$primary\", 'CA')])\n"
>>>>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
>>>>> + calciteAssert()
>>>>> + .query("select \"state\", \"city\" from zips where
>>>> \"state\" = 'CA'")
>>>>> + .limit(3)
>>>>> + .returnsUnordered("state=CA; city=BELL GARDENS",
>>>>> + "state=CA; city=LOS ANGELES",
>>>>> + "state=CA; city=NORWALK")
>>>>> + .explainContains(explain);
>>>>> + }
>>>>> +
>>>>> + @Test public void testFilterReversed() {
>>>>> + calciteAssert()
>>>>> + .query("select \"state\", \"city\" from zips where 'WI' <
>>>> \"state\" order by \"city\"")
>>>>> + .limit(2)
>>>>> + .returnsUnordered("state=WV; city=BECKLEY",
>>>>> + "state=WY; city=CHEYENNE");
>>>>> + calciteAssert()
>>>>> + .query("select \"state\", \"city\" from zips where \"state\"
>>>>> 'WI' order by \"city\"")
>>>>> + .limit(2)
>>>>> + .returnsUnordered("state=WV; city=BECKLEY",
>>>>> + "state=WY; city=CHEYENNE");
>>>>> + }
>>>>> +
>>>>> +}
>>>>> +
>>>>> +// End ElasticSearch2AdapterTest.java
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>>>> ----------------------------------------------------------------------
>>>>> diff --git
>>>>
>> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>>>> new file mode 100644
>>>>> index 0000000..4474add
>>>>> --- /dev/null
>>>>> +++
>>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>>>> @@ -0,0 +1,147 @@
>>>>> +/*
>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>>>> + * contributor license agreements. See the NOTICE file distributed
>> with
>>>>> + * this work for additional information regarding copyright ownership.
>>>>> + * The ASF licenses this file to you under the Apache License, Version
>>>> 2.0
>>>>> + * (the "License"); you may not use this file except in compliance
>> with
>>>>> + * the License. You may obtain a copy of the License at
>>>>> + *
>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>>>> + *
>>>>> + * Unless required by applicable law or agreed to in writing, software
>>>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>>>> implied.
>>>>> + * See the License for the specific language governing permissions and
>>>>> + * limitations under the License.
>>>>> + */
>>>>> +package org.apache.calcite.adapter.elasticsearch2;
>>>>> +
>>>>> +import com.google.common.base.Preconditions;
>>>>> +import com.google.common.io.Files;
>>>>> +
>>>>> +import org.elasticsearch.Version;
>>>>> +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
>>>>> +import org.elasticsearch.action.admin.cluster.node.info
>>>> .NodesInfoResponse;
>>>>> +import org.elasticsearch.client.Client;
>>>>> +import org.elasticsearch.common.settings.Settings;
>>>>> +import org.elasticsearch.common.transport.TransportAddress;
>>>>> +import org.elasticsearch.node.Node;
>>>>> +import org.elasticsearch.node.internal.InternalSettingsPreparer;
>>>>> +import org.elasticsearch.plugins.Plugin;
>>>>> +import org.elasticsearch.script.groovy.GroovyPlugin;
>>>>> +
>>>>> +import java.io.File;
>>>>> +import java.util.Arrays;
>>>>> +import java.util.Collection;
>>>>> +import java.util.Collections;
>>>>> +
>>>>> +/**
>>>>> + * Represents a single elastic search node which can run embedded in a
>>>> java application.
>>>>> + * Intended for unit and integration tests. Settings and plugins are
>>>> crafted for Calcite.
>>>>> + */
>>>>> +class EmbeddedElasticNode implements AutoCloseable {
>>>>> +
>>>>> + private final LocalNode node;
>>>>> + private volatile boolean isStarted;
>>>>> +
>>>>> + private EmbeddedElasticNode(LocalNode node) {
>>>>> + this.node = Preconditions.checkNotNull(node, "node");
>>>>> + }
>>>>> +
>>>>> + /**
>>>>> + * Having separate class to expose (protected) constructor which
>>>> allows to install
>>>>> + * different plugins. In our case it is {@code GroovyPlugin} for
>>>> scripted fields like
>>>>> + * {@code loc[0]} or {@code loc[1]['foo']}.
>>>>> + */
>>>>> + private static class LocalNode extends Node {
>>>>> + private LocalNode(Settings settings, Collection<Class<? extends
>>>> Plugin>> classpathPlugins) {
>>>>> + super(InternalSettingsPreparer.prepareEnvironment(settings,
>> null),
>>>>> + Version.CURRENT,
>>>>> + classpathPlugins);
>>>>> + }
>>>>> + }
>>>>> +
>>>>> + /**
>>>>> + * Creates an instance with existing settings
>>>>> + */
>>>>> + private static EmbeddedElasticNode create(Settings settings) {
>>>>> + // ensure GroovyPlugin is installed or otherwise scripted fields
>>>> would not work
>>>>> + LocalNode node = new LocalNode(settings,
>>>> Collections.singleton(GroovyPlugin.class));
>>>>> + return new EmbeddedElasticNode(node);
>>>>> + }
>>>>> +
>>>>> + /**
>>>>> + * Creates elastic node as single member of a cluster. Node will not
>>>> be started
>>>>> + * unless {@link #start()} is explicitly called.
>>>>> + */
>>>>> + public static EmbeddedElasticNode create() {
>>>>> + File data = Files.createTempDir();
>>>>> + data.deleteOnExit();
>>>>> + File home = Files.createTempDir();
>>>>> + home.deleteOnExit();
>>>>> +
>>>>> + Settings settings = Settings.builder()
>>>>> + .put("node.name", "fake-elastic")
>>>>> + .put("path.home", home.getAbsolutePath())
>>>>> + .put("path.data", data.getAbsolutePath())
>>>>> + .put("script.inline", true) // requires GroovyPlugin
>>>>> + .put("script.indexed", true) // requires GroovyPlugin
>>>>> + .put("cluster.routing.allocation.disk.threshold_enabled",
>>>> false)
>>>>> + .put("node.local", true)
>>>>> + .put("node.data", true)
>>>>> + .put("network.host", "localhost")
>>>>> + .build();
>>>>> +
>>>>> + return create(settings);
>>>>> + }
>>>>> +
>>>>> + /**
>>>>> + * Starts current node
>>>>> + */
>>>>> + public void start() {
>>>>> + Preconditions.checkState(!isStarted, "already started");
>>>>> + node.start();
>>>>> + this.isStarted = true;
>>>>> + }
>>>>> +
>>>>> + /**
>>>>> + * Returns current address to connect to with HTTP client.
>>>>> + */
>>>>> + public TransportAddress httpAddress() {
>>>>> + Preconditions.checkState(isStarted, "node is not started");
>>>>> +
>>>>> + NodesInfoResponse response =
>>>> client().admin().cluster().prepareNodesInfo()
>>>>> + .execute().actionGet();
>>>>> + if (response.getNodes().length != 1) {
>>>>> + throw new IllegalStateException("Expected single node but got "
>>>>> + + response.getNodes().length);
>>>>> + }
>>>>> + NodeInfo node = response.getNodes()[0];
>>>>> + return node.getHttp().address().boundAddresses()[0];
>>>>> + }
>>>>> +
>>>>> + /**
>>>>> + * Exposes elastic
>>>>> + * <a href="
>>>>
>> https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html
>> ">transport
>>>> client</a>
>>>>> + *
>>>>> + * (use of HTTP client is preferred).
>>>>> + */
>>>>> + public Client client() {
>>>>> + Preconditions.checkState(isStarted, "node is not started");
>>>>> + return node.client();
>>>>> + }
>>>>> +
>>>>> + @Override public void close() throws Exception {
>>>>> + node.close();
>>>>> + // cleanup data dirs
>>>>> + File data = new File(node.settings().get("path.data"));
>>>>> + File home = new File(node.settings().get("path.home"));
>>>>> + for (File file: Arrays.asList(data, home)) {
>>>>> + if (file.exists()) {
>>>>> + file.delete();
>>>>> + }
>>>>> + }
>>>>> + }
>>>>> +}
>>>>> +
>>>>> +// End EmbeddedElasticNode.java
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>>>> ----------------------------------------------------------------------
>>>>> diff --git
>>>>
>> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>>>> new file mode 100644
>>>>> index 0000000..a633078
>>>>> --- /dev/null
>>>>> +++
>>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>>>> @@ -0,0 +1,97 @@
>>>>> +/*
>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>>>> + * contributor license agreements. See the NOTICE file distributed
>> with
>>>>> + * this work for additional information regarding copyright ownership.
>>>>> + * The ASF licenses this file to you under the Apache License, Version
>>>> 2.0
>>>>> + * (the "License"); you may not use this file except in compliance
>> with
>>>>> + * the License. You may obtain a copy of the License at
>>>>> + *
>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>>>> + *
>>>>> + * Unless required by applicable law or agreed to in writing, software
>>>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>>>> implied.
>>>>> + * See the License for the specific language governing permissions and
>>>>> + * limitations under the License.
>>>>> + */
>>>>> +package org.apache.calcite.adapter.elasticsearch2;
>>>>> +
>>>>> +import com.google.common.base.Preconditions;
>>>>> +
>>>>> +import org.elasticsearch.client.Client;
>>>>> +import org.elasticsearch.common.transport.TransportAddress;
>>>>> +import org.junit.rules.ExternalResource;
>>>>> +
>>>>> +/**
>>>>> + * Used to initialize a single elastic node. For performance reasons
>>>> (node startup costs),
>>>>> + * same instance is usually shared across multiple tests.
>>>>> + *
>>>>> + * This rule should be used as follows:
>>>>> + * <pre>
>>>>> + * {@code
>>>>> + *
>>>>> + * public class MyTest {
>>>>> + * @literal @ClassRule
>>>>> + * public static final ElasticSearchRule RULE =
>>>> ElasticSearchRule.create();
>>>>> + *
>>>>> + * @literal @BeforeClass
>>>>> + * public void setup() {
>>>>> + * // ... populate instance
>>>>> + * }
>>>>> + *
>>>>> + * @literal @Test
>>>>> + * public void myTest() {
>>>>> + * TransportAddress address = RULE.httpAddress();
>>>>> + * // ....
>>>>> + * }
>>>>> + * }
>>>>> + * }
>>>>> + * </pre>
>>>>> + *
>>>>> + * @see ExternalResource
>>>>> + */
>>>>> +class EmbeddedElasticRule extends ExternalResource {
>>>>> +
>>>>> + private final EmbeddedElasticNode node;
>>>>> +
>>>>> + private EmbeddedElasticRule(EmbeddedElasticNode resource) {
>>>>> + this.node = Preconditions.checkNotNull(resource, "resource");
>>>>> + }
>>>>> +
>>>>> + @Override protected void before() throws Throwable {
>>>>> + node.start();
>>>>> + }
>>>>> +
>>>>> + @Override protected void after() {
>>>>> + try {
>>>>> + node.close();
>>>>> + } catch (Exception e) {
>>>>> + throw new RuntimeException(e);
>>>>> + }
>>>>> + }
>>>>> +
>>>>> + /**
>>>>> + * Factory method to create this rule.
>>>>> + */
>>>>> + public static EmbeddedElasticRule create() {
>>>>> + return new EmbeddedElasticRule(EmbeddedElasticNode.create());
>>>>> + }
>>>>> +
>>>>> + /**
>>>>> + * Exposes current ES transport client.
>>>>> + */
>>>>> + Client client() {
>>>>> + return node.client();
>>>>> + }
>>>>> +
>>>>> + /**
>>>>> + * HTTP address for rest clients (can be ES native or any other).
>>>>> + */
>>>>> + TransportAddress httpAddress() {
>>>>> + return node.httpAddress();
>>>>> + }
>>>>> +
>>>>> +
>>>>> +}
>>>>> +
>>>>> +// End EmbeddedElasticRule.java
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>>>> ----------------------------------------------------------------------
>>>>> diff --git
>>>>
>> a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>>>> new file mode 100644
>>>>> index 0000000..21fc491
>>>>> --- /dev/null
>>>>> +++
>>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>>>> @@ -0,0 +1,49 @@
>>>>> +/*
>>>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>>>> + * contributor license agreements. See the NOTICE file distributed
>> with
>>>>> + * this work for additional information regarding copyright ownership.
>>>>> + * The ASF licenses this file to you under the Apache License, Version
>>>> 2.0
>>>>> + * (the "License"); you may not use this file except in compliance
>> with
>>>>> + * the License. You may obtain a copy of the License at
>>>>> + *
>>>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>>>> + *
>>>>> + * Unless required by applicable law or agreed to in writing, software
>>>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>>>> implied.
>>>>> + * See the License for the specific language governing permissions and
>>>>> + * limitations under the License.
>>>>> + */
>>>>> +package org.apache.calcite.test;
>>>>> +
>>>>> +import com.google.common.base.Function;
>>>>> +
>>>>> +import java.util.List;
>>>>> +
>>>>> +import javax.annotation.Nullable;
>>>>> +
>>>>> +/**
>>>>> + * Internal util methods for ElasticSearch tests
>>>>> + */
>>>>> +public class ElasticChecker {
>>>>> +
>>>>> + private ElasticChecker() {}
>>>>> +
>>>>> +
>>>>> + /** Returns a function that checks that a particular Elasticsearch
>>>> pipeline is
>>>>> + * generated to implement a query. */
>>>>> + public static Function<List, Void> elasticsearchChecker(final
>>>> String... strings) {
>>>>> + return new Function<List, Void>() {
>>>>> + @Nullable
>>>>> + @Override public Void apply(@Nullable List actual) {
>>>>> + Object[] actualArray = actual == null || actual.isEmpty() ?
>> null
>>>>> + : ((List) actual.get(0)).toArray();
>>>>> + CalciteAssert.assertArrayEqual("expected Elasticsearch query
>>>> not found", strings,
>>>>> + actualArray);
>>>>> + return null;
>>>>> + }
>>>>> + };
>>>>> + }
>>>>> +}
>>>>> +
>>>>> +// End ElasticChecker.java
>>>>>
>>>>>
>>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>>>> ----------------------------------------------------------------------
>>>>> diff --git
>>>>
>> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>>>
>> b/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>>>> deleted file mode 100644
>>>>> index 4e0c2b6..0000000
>>>>> ---
>>>>
>> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>>>> +++ /dev/null
>>>>> @@ -1,270 +0,0 @@
>>>>> -/*
>>>>> - * Licensed to the Apache Software Foundation (ASF) under one or more
>>>>> - * contributor license agreements. See the NOTICE file distributed
>> with
>>>>> - * this work for additional information regarding copyright ownership.
>>>>> - * The ASF licenses this file to you under the Apache License, Version
>>>> 2.0
>>>>> - * (the "License"); you may not use this file except in compliance
>> with
>>>>> - * the License. You may obtain a copy of the License at
>>>>> - *
>>>>> - * http://www.apache.org/licenses/LICENSE-2.0
>>>>> - *
>>>>> - * Unless required by applicable law or agreed to in writing, software
>>>>> - * distributed under the License is distributed on an "AS IS" BASIS,
>>>>> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>>>> implied.
>>>>> - * See the License for the specific language governing permissions and
>>>>> - * limitations under the License.
>>>>> - */
>>>>> -package org.apache.calcite.test;
>>>>> -
>>>>> -import org.apache.calcite.util.Util;
>>>>> -
>>>>> -import com.google.common.base.Function;
>>>>> -import com.google.common.collect.ImmutableMap;
>>>>> -
>>>>> -import org.junit.Test;
>>>>> -
>>>>> -import java.util.List;
>>>>> -import javax.annotation.Nullable;
>>>>> -
>>>>> -/**
>>>>> - * Tests for the {@code org.apache.calcite.adapter.elasticsearch2}
>>>> package.
>>>>> - *
>>>>> - * <p>Before calling this test, you need to populate Elasticsearch, as
>>>> follows:
>>>>> - *
>>>>> - * <blockquote><code>
>>>>> - * git clone https://github.com/vlsi/calcite-test-dataset<br>
>>>>> - * cd calcite-test-dataset<br>
>>>>> - * mvn install
>>>>> - * </code></blockquote>
>>>>> - *
>>>>> - * <p>This will create a virtual machine with Elastic
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Andrei Sereda <an...@sereda.cc>.
Hmm. Both master and c12cb4b0de work for me. Perhaps the issue is JDK 8 vs
9 ?
# JDK8 (oracle) / macOS
$ mvn clean -DskipTests site
On Fri, Jun 22, 2018 at 5:56 PM Julian Hyde <jh...@apache.org> wrote:
> I just tried it again. The following fails for me (jdk9, ubuntu):
>
> $ git checkout c12cb4b0de
> $ mvn clean -DskipTests site
>
> Constructing Javadoc information...
> 1 error
> [INFO]
> ------------------------------------------------------------------------
> [INFO] Reactor Summary:
> [INFO]
> [INFO] Calcite 1.17.0-SNAPSHOT ............................ FAILURE [02:29
> min]
> [INFO] Calcite Linq4j ..................................... SKIPPED
> [INFO] Calcite Core ....................................... SKIPPED
> [INFO] Calcite Cassandra .................................. SKIPPED
> [INFO] Calcite Druid ...................................... SKIPPED
> [INFO] Calcite Elasticsearch .............................. SKIPPED
> [INFO] Calcite Elasticsearch5 ............................. SKIPPED
> [INFO] Calcite Examples ................................... SKIPPED
> [INFO] Calcite Example CSV ................................ SKIPPED
> [INFO] Calcite Example Function ........................... SKIPPED
> [INFO] Calcite File ....................................... SKIPPED
> [INFO] Calcite Geode ...................................... SKIPPED
> [INFO] Calcite MongoDB .................................... SKIPPED
> [INFO] Calcite Pig ........................................ SKIPPED
> [INFO] Calcite Piglet ..................................... SKIPPED
> [INFO] Calcite Plus ....................................... SKIPPED
> [INFO] Calcite Server ..................................... SKIPPED
> [INFO] Calcite Spark ...................................... SKIPPED
> [INFO] Calcite Splunk ..................................... SKIPPED
> [INFO] Calcite Ubenchmark 1.17.0-SNAPSHOT ................. SKIPPED
> [INFO]
> ------------------------------------------------------------------------
> [INFO] BUILD FAILURE
> [INFO]
> ------------------------------------------------------------------------
> [INFO] Total time: 02:29 min
> [INFO] Finished at: 2018-06-22T14:53:13-07:00
> [INFO]
> ------------------------------------------------------------------------
> [ERROR] Failed to execute goal
> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate
> report:
> [ERROR] Exit code: 1 -
> /home/jhyde/open1/calcite.4/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> error: package org.elasticsearch.node.internal does not exist
> [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
> [ERROR] ^
> [ERROR]
> [ERROR] Command line was: /usr/lib/jvm/jdk9/bin/javadoc @options @packages
> @argfile
> [ERROR]
> [ERROR] Refer to the generated Javadoc files in
> '/home/jhyde/open1/calcite.4/target/site/testapidocs' dir.
> [ERROR] -> [Help 1]
> [ERROR]
> [ERROR] To see the full stack trace of the errors, re-run Maven with the
> -e switch.
> [ERROR] Re-run Maven using the -X switch to enable full debug logging.
> [ERROR]
> [ERROR] For more information about the errors and possible solutions,
> please read the following articles:
> [ERROR] [Help 1]
> http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException
>
>
> > On Jun 22, 2018, at 2:41 PM, Michael Mior <mm...@apache.org> wrote:
> >
> > Odd, mvn clean site still works fine for me.
> > --
> > Michael Mior
> > mmior@apache.org
> >
> >
> >
> > Le ven. 22 juin 2018 à 17:12, Julian Hyde <jh...@apache.org> a écrit :
> >
> >> Looks like this change broke “mvn site” (perhaps also “mvn
> >> javadoc:test-javadoc”).
> >>
> >> [ERROR] Failed to execute goal
> >> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> >> project calcite: Error generating
> maven-javadoc-plugin:3.0.1:test-aggregate
> >> report:
> >> [ERROR] Exit code: 1 -
> >>
> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> >> error: package org.elasticsearch.node.internal does not exist
> >> [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
> >> [ERROR] ^
> >> [ERROR]
> >>
> >>> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
> >>>
> >>> [CALCITE-2347] running ElasticSearch in embedded mode for unit tests of
> >> ES adapter (Andrei Sereda)
> >>>
> >>> After discussion on dev-list Integration tests (for ES) have been
> >> removed. They're now
> >>> superseded by unit tests (which execute queries against a real elastic
> >> instance)
> >>>
> >>> Added local file (zips-mini.json) which contains a small subset of
> >> original zips.json
> >>> (allows to bootstrap tests faster)
> >>>
> >>> Created separate ES JUnit rule which can be re-used across different
> >> tests.
> >>>
> >>> Both v2 and v5 of ES adapters are supported.
> >>>
> >>> Close apache/calcite#716
> >>>
> >>>
> >>> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
> >>> Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
> >>> Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
> >>> Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
> >>>
> >>> Branch: refs/heads/master
> >>> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
> >>> Parents: 37944bb
> >>> Author: Andrei Sereda <an...@nospam.com>
> >>> Authored: Thu May 31 18:19:10 2018 -0400
> >>> Committer: Michael Mior <mm...@uwaterloo.ca>
> >>> Committed: Thu Jun 21 06:38:50 2018 -0400
> >>>
> >>> ----------------------------------------------------------------------
> >>> .../AbstractElasticsearchTable.java | 12 +
> >>> .../elasticsearch/ElasticsearchProject.java | 61 ++-
> >>> elasticsearch2/pom.xml | 6 +
> >>> .../Elasticsearch2Enumerator.java | 12 +-
> >>> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
> >>> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
> >>> .../ElasticSearch2AdapterTest.java | 395
> ++++++++++++++++++
> >>> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
> >>> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
> >>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> >>> .../calcite/test/Elasticsearch2AdapterIT.java | 270 -------------
> >>> .../resources/elasticsearch-zips-model.json | 50 ---
> >>> .../src/test/resources/zips-mini.json | 149 +++++++
> >>> elasticsearch5/pom.xml | 31 ++
> >>> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
> >>> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
> >>> .../ElasticSearch5AdapterTest.java | 399
> +++++++++++++++++++
> >>> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
> >>> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
> >>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> >>> .../calcite/test/Elasticsearch5AdapterIT.java | 270 -------------
> >>> .../resources/elasticsearch-zips-model.json | 50 ---
> >>> elasticsearch5/src/test/resources/log4j2.xml | 16 +
> >>> .../src/test/resources/zips-mini.json | 149 +++++++
> >>> pom.xml | 20 +-
> >>> 25 files changed, 1866 insertions(+), 670 deletions(-)
> >>> ----------------------------------------------------------------------
> >>>
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>> ----------------------------------------------------------------------
> >>> diff --git
> >>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>> index 0980469..8cc5933 100644
> >>> ---
> >>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>> +++
> >>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >>> @@ -75,6 +75,18 @@ public abstract class AbstractElasticsearchTable
> >> extends AbstractQueryableTable
> >>> relOptTable, this, null);
> >>> }
> >>>
> >>> + /**
> >>> + * In ES 5.x scripted fields start with {@code params._source.foo}
> >> while in ES2.x
> >>> + * {@code _source.foo}. Helper method to build correct query based
> on
> >> runtime version of elastic.
> >>> + *
> >>> + * @see <a href="
> >> https://github.com/elastic/elasticsearch/issues/20068">_source
> >> variable</a>
> >>> + * @see <a href="
> >>
> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html
> ">Scripted
> >> Fields</a>
> >>> + */
> >>> + protected String scriptedFieldPrefix() {
> >>> + // this is default pattern starting 5.x
> >>> + return "params._source";
> >>> + }
> >>> +
> >>> /** Executes a "find" operation on the underlying type.
> >>> *
> >>> * <p>For example,
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>> ----------------------------------------------------------------------
> >>> diff --git
> >>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>> index b42abd7..961c8b0 100644
> >>> ---
> >>
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>> +++
> >>
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >>> @@ -27,11 +27,15 @@ import
> >> org.apache.calcite.rel.metadata.RelMetadataQuery;
> >>> import org.apache.calcite.rel.type.RelDataType;
> >>> import org.apache.calcite.rex.RexNode;
> >>> import org.apache.calcite.util.Pair;
> >>> -import org.apache.calcite.util.Util;
> >>> +
> >>> +import com.google.common.base.Function;
> >>> +import com.google.common.collect.Lists;
> >>>
> >>> import java.util.ArrayList;
> >>> import java.util.List;
> >>>
> >>> +import javax.annotation.Nullable;
> >>> +
> >>> /**
> >>> * Implementation of {@link org.apache.calcite.rel.core.Project}
> >>> * relational expression in Elasticsearch.
> >>> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends Project
> >> implements ElasticsearchRel {
> >>> implementor.visitChild(0, getInput());
> >>>
> >>> final List<String> inFields =
> >>> -
> >> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> >>> +
> >> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> >>> final ElasticsearchRules.RexToElasticsearchTranslator translator =
> >>> - new ElasticsearchRules.RexToElasticsearchTranslator(
> >>> - (JavaTypeFactory) getCluster().getTypeFactory(),
> inFields);
> >>> + new ElasticsearchRules.RexToElasticsearchTranslator(
> >>> + (JavaTypeFactory) getCluster().getTypeFactory(),
> >> inFields);
> >>>
> >>> - final List<String> findItems = new ArrayList<>();
> >>> - final List<String> scriptFieldItems = new ArrayList<>();
> >>> + final List<String> fields = new ArrayList<>();
> >>> + final List<String> scriptFields = new ArrayList<>();
> >>> for (Pair<RexNode, String> pair: getNamedProjects()) {
> >>> final String name = pair.right;
> >>> final String expr = pair.left.accept(translator);
> >>>
> >>> if (expr.equals("\"" + name + "\"")) {
> >>> - findItems.add(ElasticsearchRules.quote(name));
> >>> + fields.add(name);
> >>> } else if (expr.matches("\"literal\":.+")) {
> >>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> >>> - + ":{\"script\": "
> >>> - + expr.split(":")[1] + "}");
> >>> + scriptFields.add(ElasticsearchRules.quote(name)
> >>> + + ":{\"script\": "
> >>> + + expr.split(":")[1] + "}");
> >>> } else {
> >>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> >>> - + ":{\"script\":\"params._source."
> >>> - + expr.replaceAll("\"", "") + "\"}");
> >>> + scriptFields.add(ElasticsearchRules.quote(name)
> >>> + + ":{\"script\":"
> >>> + // _source (ES2) vs params._source (ES5)
> >>> + + "\"" +
> >> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> >>> + + expr.replaceAll("\"", "") + "\"}");
> >>> + }
> >>> + }
> >>> +
> >>> + StringBuilder query = new StringBuilder();
> >>> + if (scriptFields.isEmpty()) {
> >>> + List<String> newList = Lists.transform(fields, new
> >> Function<String, String>() {
> >>> + @Nullable
> >>> + @Override public String apply(@Nullable String input) {
> >>> + return ElasticsearchRules.quote(input);
> >>> + }
> >>> + });
> >>> +
> >>> + final String findString = String.join(", ", newList);
> >>> + query.append("\"_source\" : [").append(findString).append("]");
> >>> + } else {
> >>> + // if scripted fields are present, ES ignores _source attribute
> >>> + for (String field: fields) {
> >>> + scriptFields.add(ElasticsearchRules.quote(field) +
> >> ":{\"script\": "
> >>> + // _source (ES2) vs params._source (ES5)
> >>> + + "\"" +
> >> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> >>> + + field + "\"}");
> >>> }
> >>> + query.append("\"script_fields\": {" + String.join(", ",
> >> scriptFields) + "}");
> >>> }
> >>> - final String findString = Util.toString(findItems, "", ", ", "");
> >>> - final String scriptFieldString = "\"script_fields\": {"
> >>> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
> >>> - final String fieldString = "\"_source\" : [" + findString + "]"
> >>> - + ", " + scriptFieldString;
> >>>
> >>> for (String opfield : implementor.list) {
> >>> if (opfield.startsWith("\"_source\"")) {
> >>> implementor.list.remove(opfield);
> >>> }
> >>> }
> >>> - implementor.add(fieldString);
> >>> + implementor.add(query.toString());
> >>> }
> >>> }
> >>>
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
> >>> ----------------------------------------------------------------------
> >>> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
> >>> index f24622c..6fbee03 100644
> >>> --- a/elasticsearch2/pom.xml
> >>> +++ b/elasticsearch2/pom.xml
> >>> @@ -73,6 +73,12 @@ limitations under the License.
> >>> <version>${elasticsearch-java-driver.version}</version>
> >>> </dependency>
> >>> <dependency>
> >>> + <!-- Lang groovy dependency is needed for testing with embedded
> >> ES (scripted fields like loc[0]) -->
> >>> + <groupId>org.elasticsearch.module</groupId>
> >>> + <artifactId>lang-groovy</artifactId>
> >>> + <scope>test</scope>
> >>> + </dependency>
> >>> + <dependency>
> >>> <groupId>com.carrotsearch</groupId>
> >>> <artifactId>hppc</artifactId>
> >>> <version>${hppc.version}</version>
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>> ----------------------------------------------------------------------
> >>> diff --git
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>> index 84370ab..c3d2ac0 100644
> >>> ---
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>> +++
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >>> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
> >>> import java.util.Date;
> >>> import java.util.Iterator;
> >>> import java.util.List;
> >>> +import java.util.Locale;
> >>> import java.util.Map;
> >>>
> >>> /**
> >>> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator implements
> >> Enumerator<Object> {
> >>> private static Function1<SearchHit, Object[]> listGetter(
> >>> final List<Map.Entry<String, Class>> fields) {
> >>> return new Function1<SearchHit, Object[]>() {
> >>> - public Object[] apply(SearchHit searchHitFields) {
> >>> + public Object[] apply(SearchHit hit) {
> >>> Object[] objects = new Object[fields.size()];
> >>> for (int i = 0; i < fields.size(); i++) {
> >>> final Map.Entry<String, Class> field = fields.get(i);
> >>> final String name = field.getKey();
> >>> - if (searchHitFields.fields().isEmpty()) {
> >>> - objects[i] =
> convert(searchHitFields.getSource().get(name),
> >> field.getValue());
> >>> + if (hit.fields().isEmpty()) {
> >>> + objects[i] = convert(hit.getSource().get(name),
> >> field.getValue());
> >>> + } else if (hit.fields().containsKey(name)) {
> >>> + objects[i] = convert(hit.field(name).getValue(),
> >> field.getValue());
> >>> } else {
> >>> - objects[i] =
> >> convert(searchHitFields.field(name).getValue(), field.getValue());
> >>> + throw new IllegalStateException(
> >>> + String.format(Locale.getDefault(), "No result for
> >> %s", field));
> >>> }
> >>> }
> >>> return objects;
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>> ----------------------------------------------------------------------
> >>> diff --git
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>> index 668402b..46e3fc5 100644
> >>> ---
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>> +++
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >>> @@ -22,6 +22,9 @@ import org.apache.calcite.schema.impl.AbstractSchema;
> >>>
> >>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
> >>>
> >>> +import com.google.common.annotations.VisibleForTesting;
> >>> +import com.google.common.base.Preconditions;
> >>> +
> >>> import com.google.common.collect.ImmutableList;
> >>> import com.google.common.collect.ImmutableMap;
> >>>
> >>> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
> >> AbstractSchema
> >>> }
> >>> }
> >>>
> >>> + /**
> >>> + * Allows schema to be instantiated from existing elastic search
> >> client.
> >>> + * This constructor is used in tests.
> >>> + */
> >>> + @VisibleForTesting
> >>> + Elasticsearch2Schema(Client client, String index) {
> >>> + this.client = Preconditions.checkNotNull(client, "client");
> >>> + this.index = Preconditions.checkNotNull(index, "index");
> >>> + }
> >>> +
> >>> @Override protected Map<String, Table> getTableMap() {
> >>> final ImmutableMap.Builder<String, Table> builder =
> >> ImmutableMap.builder();
> >>>
> >>> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
> >> AbstractSchema
> >>>
> >>> final List<DiscoveryNode> nodes =
> >> ImmutableList.copyOf(transportClient.connectedNodes());
> >>> if (nodes.isEmpty()) {
> >>> - throw new RuntimeException("Cannot connect to any elasticsearch
> >> nodes");
> >>> + throw new IllegalStateException("Cannot connect to any
> >> elasticsearch node: "
> >>> + + transportNodes);
> >>> }
> >>>
> >>> client = transportClient;
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>> ----------------------------------------------------------------------
> >>> diff --git
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>> index 636aa5f..2928835 100644
> >>> ---
> >>
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>> +++
> >>
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >>> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
> >> AbstractElasticsearchTable {
> >>> this.client = client;
> >>> }
> >>>
> >>> + /**
> >>> + * ES version 2.x. To access document attributes ES2 uses {@code
> >> _source.foo} syntax.
> >>> + */
> >>> + @Override protected String scriptedFieldPrefix() {
> >>> + return "_source";
> >>> + }
> >>> +
> >>> @Override protected Enumerable<Object> find(String index, List<String>
> >> ops,
> >>> - List<Map.Entry<String, Class>> fields) {
> >>> + List<Map.Entry<String,
> >> Class>> fields) {
> >>> final String dbName = index;
> >>>
> >>> final String queryString = "{" + Util.toString(ops, "", ", ", "") +
> >> "}";
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>> ----------------------------------------------------------------------
> >>> diff --git
> >>
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>> new file mode 100644
> >>> index 0000000..287e094
> >>> --- /dev/null
> >>> +++
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >>> @@ -0,0 +1,395 @@
> >>> +/*
> >>> + * Licensed to the Apache Software Foundation (ASF) under one or more
> >>> + * contributor license agreements. See the NOTICE file distributed
> with
> >>> + * this work for additional information regarding copyright ownership.
> >>> + * The ASF licenses this file to you under the Apache License, Version
> >> 2.0
> >>> + * (the "License"); you may not use this file except in compliance
> with
> >>> + * the License. You may obtain a copy of the License at
> >>> + *
> >>> + * http://www.apache.org/licenses/LICENSE-2.0
> >>> + *
> >>> + * Unless required by applicable law or agreed to in writing, software
> >>> + * distributed under the License is distributed on an "AS IS" BASIS,
> >>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> >> implied.
> >>> + * See the License for the specific language governing permissions and
> >>> + * limitations under the License.
> >>> + */
> >>> +package org.apache.calcite.adapter.elasticsearch2;
> >>> +
> >>> +import org.apache.calcite.jdbc.CalciteConnection;
> >>> +import org.apache.calcite.schema.SchemaPlus;
> >>> +import org.apache.calcite.schema.impl.ViewTable;
> >>> +import org.apache.calcite.schema.impl.ViewTableMacro;
> >>> +import org.apache.calcite.test.CalciteAssert;
> >>> +import org.apache.calcite.test.ElasticChecker;
> >>> +
> >>> +import com.google.common.io.LineProcessor;
> >>> +import com.google.common.io.Resources;
> >>> +
> >>> +import org.elasticsearch.action.bulk.BulkItemResponse;
> >>> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
> >>> +import org.elasticsearch.action.bulk.BulkResponse;
> >>> +import org.elasticsearch.common.xcontent.XContentBuilder;
> >>> +import org.elasticsearch.common.xcontent.XContentFactory;
> >>> +
> >>> +import org.junit.BeforeClass;
> >>> +import org.junit.ClassRule;
> >>> +import org.junit.Test;
> >>> +
> >>> +import java.io.IOException;
> >>> +import java.nio.charset.StandardCharsets;
> >>> +import java.sql.Connection;
> >>> +import java.sql.DriverManager;
> >>> +import java.sql.SQLException;
> >>> +import java.util.Arrays;
> >>> +import java.util.Collections;
> >>> +import java.util.Locale;
> >>> +
> >>> +/**
> >>> + * Set of tests for ES adapter. Uses real instance via {@link
> >> EmbeddedElasticRule}. Document
> >>> + * source is local {@code zips-mini.json} file (located in the
> >> classpath).
> >>> + */
> >>> +public class ElasticSearch2AdapterTest {
> >>> +
> >>> + @ClassRule //init once for all tests
> >>> + public static final EmbeddedElasticRule NODE =
> >> EmbeddedElasticRule.create();
> >>> +
> >>> + private static final String ZIPS = "zips";
> >>> +
> >>> + /**
> >>> + * Used to create {@code zips} index and insert some data
> >>> + */
> >>> + @BeforeClass
> >>> + public static void setupInstance() throws Exception {
> >>> + // define mapping so fields are searchable (term query)
> >>> + XContentBuilder mapping =
> >> XContentFactory.jsonBuilder().startObject()
> >>> + .startObject("properties")
> >>> + .startObject("city").field("type", "string")
> >>> + .field("index", "not_analyzed").endObject()
> >>> + .startObject("state").field("type", "string")
> >>> + .field("index", "not_analyzed").endObject()
> >>> + .startObject("pop").field("type", "long").endObject()
> >>> + .endObject()
> >>> + .endObject();
> >>> +
> >>> + // create index
> >>> + NODE.client().admin().indices()
> >>> + .prepareCreate(ZIPS)
> >>> + .addMapping(ZIPS, mapping)
> >>> + .get();
> >>> +
> >>> + BulkRequestBuilder bulk =
> >> NODE.client().prepareBulk().setRefresh(true);
> >>> +
> >>> + // load records from file
> >>> +
> >>
> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
> >>> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
> >>> + @Override public boolean processLine(String line) throws
> >> IOException {
> >>> + line = line.replaceAll("_id", "id"); // _id is a
> >> reserved attribute in ES
> >>> + bulk.add(NODE.client().prepareIndex(ZIPS,
> >> ZIPS).setSource(line));
> >>> + return true;
> >>> + }
> >>> +
> >>> + @Override public Void getResult() {
> >>> + return null;
> >>> + }
> >>> + });
> >>> +
> >>> + if (bulk.numberOfActions() == 0) {
> >>> + throw new IllegalStateException("No records to be indexed");
> >>> + }
> >>> +
> >>> + BulkResponse response = bulk.execute().get();
> >>> +
> >>> + if (response.hasFailures()) {
> >>> + throw new IllegalStateException(
> >>> + String.format(Locale.getDefault(), "Failed to populate
> >> %s:\n%s", NODE.httpAddress(),
> >>> +
> >> Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
> >>> +
> >>
> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
> >>> + }
> >>> +
> >>> + }
> >>> +
> >>> + private CalciteAssert.ConnectionFactory newConnectionFactory() {
> >>> + return new CalciteAssert.ConnectionFactory() {
> >>> + @Override public Connection createConnection() throws
> >> SQLException {
> >>> + final Connection connection =
> >> DriverManager.getConnection("jdbc:calcite:");
> >>> + final SchemaPlus root =
> >> connection.unwrap(CalciteConnection.class).getRootSchema();
> >>> +
> >>> + root.add("elastic", new Elasticsearch2Schema(NODE.client(),
> >> ZIPS));
> >>> +
> >>> + // add calcite view programmatically
> >>> + final String viewSql = "select cast(_MAP['city'] AS
> >> varchar(20)) AS \"city\", "
> >>> + + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n"
> >>> + + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n"
> >>> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
> >>> + + " cast(_MAP['state'] AS varchar(2)) AS \"state\", "
> >>> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
> >>> + + "from \"elastic\".\"zips\"";
> >>> +
> >>> + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
> >>> + Collections.singletonList("elastic"),
> >> Arrays.asList("elastic", "view"), false);
> >>> + root.add("ZIPS", macro);
> >>> +
> >>> + return connection;
> >>> + }
> >>> + };
> >>> + }
> >>> +
> >>> + private CalciteAssert.AssertThat calciteAssert() {
> >>> + return CalciteAssert.that()
> >>> + .with(newConnectionFactory());
> >>> + }
> >>> +
> >>> + /**
> >>> + * Tests using calcite view
> >>> + */
> >>> + @Test
> >>> + public void view() throws Exception {
> >>> + calciteAssert()
> >>> + .query("select * from zips where \"city\" = 'BROOKLYN'")
> >>> + .returns("city=BROOKLYN; longitude=-73.956985;
> >> latitude=40.646694; "
> >>> + + "pop=111396; state=NY; id=11226\n")
> >>> + .returnsCount(1);
> >>> + }
> >>> +
> >>> + @Test
> >>> + public void emptyResult() {
> >>> + CalciteAssert.that()
> >>> + .with(newConnectionFactory())
> >>> + .query("select * from zips limit 0")
> >>> + .returnsCount(0);
> >>> +
> >>> + CalciteAssert.that()
> >>> + .with(newConnectionFactory())
> >>> + .query("select * from \"elastic\".\"zips\" where
> >> _MAP['Foo'] = '_MISSING_'")
> >>> + .returnsCount(0);
> >>> + }
> >>> +
> >>> + @Test
> >>> + public void basic() throws Exception {
> >>> + CalciteAssert.that()
> >>> + .with(newConnectionFactory())
> >>> + .query("select * from \"elastic\".\"zips\" where
> >> _MAP['city'] = 'BROOKLYN'")
> >>> + .returnsCount(1);
> >>> +
> >>> + CalciteAssert.that()
> >>> + .with(newConnectionFactory())
> >>> + .query("select * from \"elastic\".\"zips\" where"
> >>> + + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')")
> >>> + .returnsCount(2);
> >>> +
> >>> + // lower-case
> >>> + CalciteAssert.that()
> >>> + .with(newConnectionFactory())
> >>> + .query("select * from \"elastic\".\"zips\" where "
> >>> + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
> >> 'BROOK') ")
> >>> + .returnsCount(0);
> >>> +
> >>> + // missing field
> >>> + CalciteAssert.that()
> >>> + .with(newConnectionFactory())
> >>> + .query("select * from \"elastic\".\"zips\" where
> >> _MAP['CITY'] = 'BROOKLYN'")
> >>> + .returnsCount(0);
> >>> +
> >>> + // limit works
> >>> + CalciteAssert.that()
> >>> + .with(newConnectionFactory())
> >>> + .query("select * from \"elastic\".\"zips\" limit 42")
> >>> + .returnsCount(42);
> >>> +
> >>> + }
> >>> +
> >>> + @Test public void testSort() {
> >>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> >>> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> >>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> >> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> >> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> >> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> >> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> >> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> >> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> >>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
> >>> +
> >>> + calciteAssert()
> >>> + .query("select * from zips order by \"state\"")
> >>> + .returnsCount(10)
> >>> + .explainContains(explain);
> >>> + }
> >>> +
> >>> + @Test public void testSortLimit() {
> >>> + final String sql = "select \"state\", \"id\" from zips\n"
> >>> + + "order by \"state\", \"id\" offset 2 rows fetch next 3
> >> rows only";
> >>> + calciteAssert()
> >>> + .query(sql)
> >>> + .returnsUnordered("state=AK; id=99801",
> >>> + "state=AL; id=35215",
> >>> + "state=AL; id=35401")
> >>> + .queryContains(
> >>> + ElasticChecker.elasticsearchChecker(
> >>> + "\"_source\" : [\"state\", \"id\"]",
> >>> + "\"sort\": [ {\"state\": \"asc\"},
> {\"id\":
> >> \"asc\"}]",
> >>> + "\"from\": 2",
> >>> + "\"size\": 3"));
> >>> + }
> >>> +
> >>> +
> >>> +
> >>> + @Test public void testOffsetLimit() {
> >>> + final String sql = "select \"state\", \"id\" from zips\n"
> >>> + + "offset 2 fetch next 3 rows only";
> >>> + calciteAssert()
> >>> + .query(sql)
> >>> + .runs()
> >>> + .queryContains(
> >>> + ElasticChecker.elasticsearchChecker(
> >>> + "\"from\": 2",
> >>> + "\"size\": 3",
> >>> + "\"_source\" : [\"state\", \"id\"]"));
> >>> + }
> >>> +
> >>> + @Test public void testLimit() {
> >>> + final String sql = "select \"state\", \"id\" from zips\n"
> >>> + + "fetch next 3 rows only";
> >>> +
> >>> + calciteAssert()
> >>> + .query(sql)
> >>> + .runs()
> >>> + .queryContains(
> >>> + ElasticChecker.elasticsearchChecker(
> >>> + "\"size\": 3",
> >>> + "\"_source\" : [\"state\", \"id\"]"));
> >>> + }
> >>> +
> >>> + @Test public void testFilterSort() {
> >>> + final String sql = "select * from zips\n"
> >>> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
> >>> + + "order by \"state\", \"id\"";
> >>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> >>> + + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC],
> >> dir1=[ASC])\n"
> >>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> >> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> >> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> >> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> >> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> >> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> >> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> >>> + + "
> ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
> >> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >> \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0, 'id')):VARCHAR(5)
> >> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\",
> >> '70000'))])\n"
> >>> + + " ElasticsearchTableScan(table=[[elastic,
> zips]])";
> >>> + calciteAssert()
> >>> + .query(sql)
> >>> + .returnsOrdered("city=LOS ANGELES; longitude=-118.258189;
> >> latitude=34.007856; "
> >>> + + "pop=96074; state=CA; id=90011",
> >>> + "city=BELL GARDENS; longitude=-118.17205;
> >> latitude=33.969177; "
> >>> + + "pop=99568; state=CA; id=90201",
> >>> + "city=NORWALK; longitude=-118.081767;
> >> latitude=33.90564; "
> >>> + + "pop=94188; state=CA; id=90650")
> >>> + .queryContains(
> >>> + ElasticChecker.elasticsearchChecker("\"query\" : "
> >>> + +
> >> "{\"constant_score\":{\"filter\":{\"bool\":"
> >>> + +
> >> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
> >>> + +
> >> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
> >>> + "\"script_fields\":
> >> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> >>> + +
> >> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> >>> + + "\"city\":{\"script\":
> >> \"_source.city\"}, "
> >>> + + "\"pop\":{\"script\":
> >> \"_source.pop\"}, "
> >>> + + "\"state\":{\"script\":
> >> \"_source.state\"}, "
> >>> + + "\"id\":{\"script\": \"_
> source.id
> >> \"}}",
> >>> + "\"sort\": [ {\"state\": \"asc\"},
> {\"id\":
> >> \"asc\"}]"))
> >>> + .explainContains(explain);
> >>> + }
> >>> +
> >>> + @Test public void testFilterSortDesc() {
> >>> + final String sql = "select * from zips\n"
> >>> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
> >>> + + "order by \"state\" desc, \"pop\"";
> >>> + calciteAssert()
> >>> + .query(sql)
> >>> + .limit(4)
> >>> + .returnsOrdered(
> >>> + "city=LOS ANGELES; longitude=-118.258189;
> >> latitude=34.007856; pop=96074; state=CA; id=90011",
> >>> + "city=BELL GARDENS; longitude=-118.17205;
> >> latitude=33.969177; pop=99568; state=CA; id=90201");
> >>> + }
> >>> +
> >>> + @Test public void testFilterRedundant() {
> >>> + final String sql = "select * from zips\n"
> >>> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and
> >> \"state\" = 'OK'";
> >>> + calciteAssert()
> >>> + .query(sql)
> >>> + .runs()
> >>> + .queryContains(
> >>> + ElasticChecker.elasticsearchChecker(""
> >>> + + "\"query\" :
> >> {\"constant_score\":{\"filter\":{\"bool\":"
> >>> + +
> >> "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
> >>> + "\"script_fields\":
> >> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> >>> + +
> >> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> >>> + + "\"city\":{\"script\":
> >> \"_source.city\"}, "
> >>> + + "\"pop\":{\"script\":
> >> \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
> >>> + + "\"id\":{\"script\":
> >> \"_source.id\"}}"
> >>> + ));
> >>> + }
> >>> +
> >>> + @Test public void testInPlan() {
> >>> + final String[] searches = {
> >>> + "\"query\" :
> >> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
> >>> + +
> >>
> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
> >>> + + "{\"pop\":99568}}]}}]}}}}",
> >>> + "\"script_fields\":
> >> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> >>> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> >>> + + "\"city\":{\"script\": \"_source.city\"}, "
> >>> + + "\"pop\":{\"script\": \"_source.pop\"}, "
> >>> + + "\"state\":{\"script\": \"_source.state\"}, "
> >>> + + "\"id\":{\"script\": \"_source.id\"}}"
> >>> + };
> >>> +
> >>> + calciteAssert()
> >>> + .query("select * from zips where \"pop\" in (96074,
> 99568)")
> >>> + .returnsUnordered(
> >>> + "city=BELL GARDENS; longitude=-118.17205;
> >> latitude=33.969177; pop=99568; state=CA; id=90201",
> >>> + "city=LOS ANGELES; longitude=-118.258189;
> >> latitude=34.007856; pop=96074; state=CA; id=90011"
> >>> + )
> >>> +
> >> .queryContains(ElasticChecker.elasticsearchChecker(searches));
> >>> + }
> >>> +
> >>> + @Test public void testZips() {
> >>> + calciteAssert()
> >>> + .query("select \"state\", \"city\" from zips")
> >>> + .returnsCount(10);
> >>> + }
> >>> +
> >>> + @Test public void testProject() {
> >>> + final String sql = "select \"state\", \"city\", 0 as \"zero\"\n"
> >>> + + "from zips\n"
> >>> + + "order by \"state\", \"city\"";
> >>> +
> >>> + calciteAssert()
> >>> + .query(sql)
> >>> + .limit(2)
> >>> + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0",
> >>> + "state=AK; city=FAIRBANKS; zero=0")
> >>> + .queryContains(
> >>> +
> >> ElasticChecker.elasticsearchChecker("\"script_fields\": "
> >>> + + "{\"zero\":{\"script\": \"0\"},
> "
> >>> + + "\"state\":{\"script\":
> >> \"_source.state\"}, "
> >>> + + "\"city\":{\"script\":
> >> \"_source.city\"}}",
> >>> + "\"sort\": [ {\"state\": \"asc\"},
> >> {\"city\": \"asc\"}]"));
> >>> + }
> >>> +
> >>> + @Test public void testFilter() {
> >>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> >>> + + " ElasticsearchProject(state=[CAST(ITEM($0,
> >> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >> \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20)
> >> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> >>> + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0,
> >> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> >> \"ISO-8859-1$en_US$primary\", 'CA')])\n"
> >>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
> >>> + calciteAssert()
> >>> + .query("select \"state\", \"city\" from zips where
> >> \"state\" = 'CA'")
> >>> + .limit(3)
> >>> + .returnsUnordered("state=CA; city=BELL GARDENS",
> >>> + "state=CA; city=LOS ANGELES",
> >>> + "state=CA; city=NORWALK")
> >>> + .explainContains(explain);
> >>> + }
> >>> +
> >>> + @Test public void testFilterReversed() {
> >>> + calciteAssert()
> >>> + .query("select \"state\", \"city\" from zips where 'WI' <
> >> \"state\" order by \"city\"")
> >>> + .limit(2)
> >>> + .returnsUnordered("state=WV; city=BECKLEY",
> >>> + "state=WY; city=CHEYENNE");
> >>> + calciteAssert()
> >>> + .query("select \"state\", \"city\" from zips where \"state\"
> >>> 'WI' order by \"city\"")
> >>> + .limit(2)
> >>> + .returnsUnordered("state=WV; city=BECKLEY",
> >>> + "state=WY; city=CHEYENNE");
> >>> + }
> >>> +
> >>> +}
> >>> +
> >>> +// End ElasticSearch2AdapterTest.java
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >>> ----------------------------------------------------------------------
> >>> diff --git
> >>
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >>> new file mode 100644
> >>> index 0000000..4474add
> >>> --- /dev/null
> >>> +++
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >>> @@ -0,0 +1,147 @@
> >>> +/*
> >>> + * Licensed to the Apache Software Foundation (ASF) under one or more
> >>> + * contributor license agreements. See the NOTICE file distributed
> with
> >>> + * this work for additional information regarding copyright ownership.
> >>> + * The ASF licenses this file to you under the Apache License, Version
> >> 2.0
> >>> + * (the "License"); you may not use this file except in compliance
> with
> >>> + * the License. You may obtain a copy of the License at
> >>> + *
> >>> + * http://www.apache.org/licenses/LICENSE-2.0
> >>> + *
> >>> + * Unless required by applicable law or agreed to in writing, software
> >>> + * distributed under the License is distributed on an "AS IS" BASIS,
> >>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> >> implied.
> >>> + * See the License for the specific language governing permissions and
> >>> + * limitations under the License.
> >>> + */
> >>> +package org.apache.calcite.adapter.elasticsearch2;
> >>> +
> >>> +import com.google.common.base.Preconditions;
> >>> +import com.google.common.io.Files;
> >>> +
> >>> +import org.elasticsearch.Version;
> >>> +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
> >>> +import org.elasticsearch.action.admin.cluster.node.info
> >> .NodesInfoResponse;
> >>> +import org.elasticsearch.client.Client;
> >>> +import org.elasticsearch.common.settings.Settings;
> >>> +import org.elasticsearch.common.transport.TransportAddress;
> >>> +import org.elasticsearch.node.Node;
> >>> +import org.elasticsearch.node.internal.InternalSettingsPreparer;
> >>> +import org.elasticsearch.plugins.Plugin;
> >>> +import org.elasticsearch.script.groovy.GroovyPlugin;
> >>> +
> >>> +import java.io.File;
> >>> +import java.util.Arrays;
> >>> +import java.util.Collection;
> >>> +import java.util.Collections;
> >>> +
> >>> +/**
> >>> + * Represents a single elastic search node which can run embedded in a
> >> java application.
> >>> + * Intended for unit and integration tests. Settings and plugins are
> >> crafted for Calcite.
> >>> + */
> >>> +class EmbeddedElasticNode implements AutoCloseable {
> >>> +
> >>> + private final LocalNode node;
> >>> + private volatile boolean isStarted;
> >>> +
> >>> + private EmbeddedElasticNode(LocalNode node) {
> >>> + this.node = Preconditions.checkNotNull(node, "node");
> >>> + }
> >>> +
> >>> + /**
> >>> + * Having separate class to expose (protected) constructor which
> >> allows to install
> >>> + * different plugins. In our case it is {@code GroovyPlugin} for
> >> scripted fields like
> >>> + * {@code loc[0]} or {@code loc[1]['foo']}.
> >>> + */
> >>> + private static class LocalNode extends Node {
> >>> + private LocalNode(Settings settings, Collection<Class<? extends
> >> Plugin>> classpathPlugins) {
> >>> + super(InternalSettingsPreparer.prepareEnvironment(settings,
> null),
> >>> + Version.CURRENT,
> >>> + classpathPlugins);
> >>> + }
> >>> + }
> >>> +
> >>> + /**
> >>> + * Creates an instance with existing settings
> >>> + */
> >>> + private static EmbeddedElasticNode create(Settings settings) {
> >>> + // ensure GroovyPlugin is installed or otherwise scripted fields
> >> would not work
> >>> + LocalNode node = new LocalNode(settings,
> >> Collections.singleton(GroovyPlugin.class));
> >>> + return new EmbeddedElasticNode(node);
> >>> + }
> >>> +
> >>> + /**
> >>> + * Creates elastic node as single member of a cluster. Node will not
> >> be started
> >>> + * unless {@link #start()} is explicitly called.
> >>> + */
> >>> + public static EmbeddedElasticNode create() {
> >>> + File data = Files.createTempDir();
> >>> + data.deleteOnExit();
> >>> + File home = Files.createTempDir();
> >>> + home.deleteOnExit();
> >>> +
> >>> + Settings settings = Settings.builder()
> >>> + .put("node.name", "fake-elastic")
> >>> + .put("path.home", home.getAbsolutePath())
> >>> + .put("path.data", data.getAbsolutePath())
> >>> + .put("script.inline", true) // requires GroovyPlugin
> >>> + .put("script.indexed", true) // requires GroovyPlugin
> >>> + .put("cluster.routing.allocation.disk.threshold_enabled",
> >> false)
> >>> + .put("node.local", true)
> >>> + .put("node.data", true)
> >>> + .put("network.host", "localhost")
> >>> + .build();
> >>> +
> >>> + return create(settings);
> >>> + }
> >>> +
> >>> + /**
> >>> + * Starts current node
> >>> + */
> >>> + public void start() {
> >>> + Preconditions.checkState(!isStarted, "already started");
> >>> + node.start();
> >>> + this.isStarted = true;
> >>> + }
> >>> +
> >>> + /**
> >>> + * Returns current address to connect to with HTTP client.
> >>> + */
> >>> + public TransportAddress httpAddress() {
> >>> + Preconditions.checkState(isStarted, "node is not started");
> >>> +
> >>> + NodesInfoResponse response =
> >> client().admin().cluster().prepareNodesInfo()
> >>> + .execute().actionGet();
> >>> + if (response.getNodes().length != 1) {
> >>> + throw new IllegalStateException("Expected single node but got "
> >>> + + response.getNodes().length);
> >>> + }
> >>> + NodeInfo node = response.getNodes()[0];
> >>> + return node.getHttp().address().boundAddresses()[0];
> >>> + }
> >>> +
> >>> + /**
> >>> + * Exposes elastic
> >>> + * <a href="
> >>
> https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html
> ">transport
> >> client</a>
> >>> + *
> >>> + * (use of HTTP client is preferred).
> >>> + */
> >>> + public Client client() {
> >>> + Preconditions.checkState(isStarted, "node is not started");
> >>> + return node.client();
> >>> + }
> >>> +
> >>> + @Override public void close() throws Exception {
> >>> + node.close();
> >>> + // cleanup data dirs
> >>> + File data = new File(node.settings().get("path.data"));
> >>> + File home = new File(node.settings().get("path.home"));
> >>> + for (File file: Arrays.asList(data, home)) {
> >>> + if (file.exists()) {
> >>> + file.delete();
> >>> + }
> >>> + }
> >>> + }
> >>> +}
> >>> +
> >>> +// End EmbeddedElasticNode.java
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> >>> ----------------------------------------------------------------------
> >>> diff --git
> >>
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> >>> new file mode 100644
> >>> index 0000000..a633078
> >>> --- /dev/null
> >>> +++
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> >>> @@ -0,0 +1,97 @@
> >>> +/*
> >>> + * Licensed to the Apache Software Foundation (ASF) under one or more
> >>> + * contributor license agreements. See the NOTICE file distributed
> with
> >>> + * this work for additional information regarding copyright ownership.
> >>> + * The ASF licenses this file to you under the Apache License, Version
> >> 2.0
> >>> + * (the "License"); you may not use this file except in compliance
> with
> >>> + * the License. You may obtain a copy of the License at
> >>> + *
> >>> + * http://www.apache.org/licenses/LICENSE-2.0
> >>> + *
> >>> + * Unless required by applicable law or agreed to in writing, software
> >>> + * distributed under the License is distributed on an "AS IS" BASIS,
> >>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> >> implied.
> >>> + * See the License for the specific language governing permissions and
> >>> + * limitations under the License.
> >>> + */
> >>> +package org.apache.calcite.adapter.elasticsearch2;
> >>> +
> >>> +import com.google.common.base.Preconditions;
> >>> +
> >>> +import org.elasticsearch.client.Client;
> >>> +import org.elasticsearch.common.transport.TransportAddress;
> >>> +import org.junit.rules.ExternalResource;
> >>> +
> >>> +/**
> >>> + * Used to initialize a single elastic node. For performance reasons
> >> (node startup costs),
> >>> + * same instance is usually shared across multiple tests.
> >>> + *
> >>> + * This rule should be used as follows:
> >>> + * <pre>
> >>> + * {@code
> >>> + *
> >>> + * public class MyTest {
> >>> + * @literal @ClassRule
> >>> + * public static final ElasticSearchRule RULE =
> >> ElasticSearchRule.create();
> >>> + *
> >>> + * @literal @BeforeClass
> >>> + * public void setup() {
> >>> + * // ... populate instance
> >>> + * }
> >>> + *
> >>> + * @literal @Test
> >>> + * public void myTest() {
> >>> + * TransportAddress address = RULE.httpAddress();
> >>> + * // ....
> >>> + * }
> >>> + * }
> >>> + * }
> >>> + * </pre>
> >>> + *
> >>> + * @see ExternalResource
> >>> + */
> >>> +class EmbeddedElasticRule extends ExternalResource {
> >>> +
> >>> + private final EmbeddedElasticNode node;
> >>> +
> >>> + private EmbeddedElasticRule(EmbeddedElasticNode resource) {
> >>> + this.node = Preconditions.checkNotNull(resource, "resource");
> >>> + }
> >>> +
> >>> + @Override protected void before() throws Throwable {
> >>> + node.start();
> >>> + }
> >>> +
> >>> + @Override protected void after() {
> >>> + try {
> >>> + node.close();
> >>> + } catch (Exception e) {
> >>> + throw new RuntimeException(e);
> >>> + }
> >>> + }
> >>> +
> >>> + /**
> >>> + * Factory method to create this rule.
> >>> + */
> >>> + public static EmbeddedElasticRule create() {
> >>> + return new EmbeddedElasticRule(EmbeddedElasticNode.create());
> >>> + }
> >>> +
> >>> + /**
> >>> + * Exposes current ES transport client.
> >>> + */
> >>> + Client client() {
> >>> + return node.client();
> >>> + }
> >>> +
> >>> + /**
> >>> + * HTTP address for rest clients (can be ES native or any other).
> >>> + */
> >>> + TransportAddress httpAddress() {
> >>> + return node.httpAddress();
> >>> + }
> >>> +
> >>> +
> >>> +}
> >>> +
> >>> +// End EmbeddedElasticRule.java
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> >>> ----------------------------------------------------------------------
> >>> diff --git
> >>
> a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> >>> new file mode 100644
> >>> index 0000000..21fc491
> >>> --- /dev/null
> >>> +++
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> >>> @@ -0,0 +1,49 @@
> >>> +/*
> >>> + * Licensed to the Apache Software Foundation (ASF) under one or more
> >>> + * contributor license agreements. See the NOTICE file distributed
> with
> >>> + * this work for additional information regarding copyright ownership.
> >>> + * The ASF licenses this file to you under the Apache License, Version
> >> 2.0
> >>> + * (the "License"); you may not use this file except in compliance
> with
> >>> + * the License. You may obtain a copy of the License at
> >>> + *
> >>> + * http://www.apache.org/licenses/LICENSE-2.0
> >>> + *
> >>> + * Unless required by applicable law or agreed to in writing, software
> >>> + * distributed under the License is distributed on an "AS IS" BASIS,
> >>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> >> implied.
> >>> + * See the License for the specific language governing permissions and
> >>> + * limitations under the License.
> >>> + */
> >>> +package org.apache.calcite.test;
> >>> +
> >>> +import com.google.common.base.Function;
> >>> +
> >>> +import java.util.List;
> >>> +
> >>> +import javax.annotation.Nullable;
> >>> +
> >>> +/**
> >>> + * Internal util methods for ElasticSearch tests
> >>> + */
> >>> +public class ElasticChecker {
> >>> +
> >>> + private ElasticChecker() {}
> >>> +
> >>> +
> >>> + /** Returns a function that checks that a particular Elasticsearch
> >> pipeline is
> >>> + * generated to implement a query. */
> >>> + public static Function<List, Void> elasticsearchChecker(final
> >> String... strings) {
> >>> + return new Function<List, Void>() {
> >>> + @Nullable
> >>> + @Override public Void apply(@Nullable List actual) {
> >>> + Object[] actualArray = actual == null || actual.isEmpty() ?
> null
> >>> + : ((List) actual.get(0)).toArray();
> >>> + CalciteAssert.assertArrayEqual("expected Elasticsearch query
> >> not found", strings,
> >>> + actualArray);
> >>> + return null;
> >>> + }
> >>> + };
> >>> + }
> >>> +}
> >>> +
> >>> +// End ElasticChecker.java
> >>>
> >>>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> >>> ----------------------------------------------------------------------
> >>> diff --git
> >>
> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> >>
> b/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> >>> deleted file mode 100644
> >>> index 4e0c2b6..0000000
> >>> ---
> >>
> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> >>> +++ /dev/null
> >>> @@ -1,270 +0,0 @@
> >>> -/*
> >>> - * Licensed to the Apache Software Foundation (ASF) under one or more
> >>> - * contributor license agreements. See the NOTICE file distributed
> with
> >>> - * this work for additional information regarding copyright ownership.
> >>> - * The ASF licenses this file to you under the Apache License, Version
> >> 2.0
> >>> - * (the "License"); you may not use this file except in compliance
> with
> >>> - * the License. You may obtain a copy of the License at
> >>> - *
> >>> - * http://www.apache.org/licenses/LICENSE-2.0
> >>> - *
> >>> - * Unless required by applicable law or agreed to in writing, software
> >>> - * distributed under the License is distributed on an "AS IS" BASIS,
> >>> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> >> implied.
> >>> - * See the License for the specific language governing permissions and
> >>> - * limitations under the License.
> >>> - */
> >>> -package org.apache.calcite.test;
> >>> -
> >>> -import org.apache.calcite.util.Util;
> >>> -
> >>> -import com.google.common.base.Function;
> >>> -import com.google.common.collect.ImmutableMap;
> >>> -
> >>> -import org.junit.Test;
> >>> -
> >>> -import java.util.List;
> >>> -import javax.annotation.Nullable;
> >>> -
> >>> -/**
> >>> - * Tests for the {@code org.apache.calcite.adapter.elasticsearch2}
> >> package.
> >>> - *
> >>> - * <p>Before calling this test, you need to populate Elasticsearch, as
> >> follows:
> >>> - *
> >>> - * <blockquote><code>
> >>> - * git clone https://github.com/vlsi/calcite-test-dataset<br>
> >>> - * cd calcite-test-dataset<br>
> >>> - * mvn install
> >>> - * </code></blockquote>
> >>> - *
> >>> - * <p>This will create a virtual machine with Elastic
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Julian Hyde <jh...@apache.org>.
I just tried it again. The following fails for me (jdk9, ubuntu):
$ git checkout c12cb4b0de
$ mvn clean -DskipTests site
Constructing Javadoc information...
1 error
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Summary:
[INFO]
[INFO] Calcite 1.17.0-SNAPSHOT ............................ FAILURE [02:29 min]
[INFO] Calcite Linq4j ..................................... SKIPPED
[INFO] Calcite Core ....................................... SKIPPED
[INFO] Calcite Cassandra .................................. SKIPPED
[INFO] Calcite Druid ...................................... SKIPPED
[INFO] Calcite Elasticsearch .............................. SKIPPED
[INFO] Calcite Elasticsearch5 ............................. SKIPPED
[INFO] Calcite Examples ................................... SKIPPED
[INFO] Calcite Example CSV ................................ SKIPPED
[INFO] Calcite Example Function ........................... SKIPPED
[INFO] Calcite File ....................................... SKIPPED
[INFO] Calcite Geode ...................................... SKIPPED
[INFO] Calcite MongoDB .................................... SKIPPED
[INFO] Calcite Pig ........................................ SKIPPED
[INFO] Calcite Piglet ..................................... SKIPPED
[INFO] Calcite Plus ....................................... SKIPPED
[INFO] Calcite Server ..................................... SKIPPED
[INFO] Calcite Spark ...................................... SKIPPED
[INFO] Calcite Splunk ..................................... SKIPPED
[INFO] Calcite Ubenchmark 1.17.0-SNAPSHOT ................. SKIPPED
[INFO] ------------------------------------------------------------------------
[INFO] BUILD FAILURE
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 02:29 min
[INFO] Finished at: 2018-06-22T14:53:13-07:00
[INFO] ------------------------------------------------------------------------
[ERROR] Failed to execute goal org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate report:
[ERROR] Exit code: 1 - /home/jhyde/open1/calcite.4/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29: error: package org.elasticsearch.node.internal does not exist
[ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
[ERROR] ^
[ERROR]
[ERROR] Command line was: /usr/lib/jvm/jdk9/bin/javadoc @options @packages @argfile
[ERROR]
[ERROR] Refer to the generated Javadoc files in '/home/jhyde/open1/calcite.4/target/site/testapidocs' dir.
[ERROR] -> [Help 1]
[ERROR]
[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch.
[ERROR] Re-run Maven using the -X switch to enable full debug logging.
[ERROR]
[ERROR] For more information about the errors and possible solutions, please read the following articles:
[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException
> On Jun 22, 2018, at 2:41 PM, Michael Mior <mm...@apache.org> wrote:
>
> Odd, mvn clean site still works fine for me.
> --
> Michael Mior
> mmior@apache.org
>
>
>
> Le ven. 22 juin 2018 à 17:12, Julian Hyde <jh...@apache.org> a écrit :
>
>> Looks like this change broke “mvn site” (perhaps also “mvn
>> javadoc:test-javadoc”).
>>
>> [ERROR] Failed to execute goal
>> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
>> project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate
>> report:
>> [ERROR] Exit code: 1 -
>> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
>> error: package org.elasticsearch.node.internal does not exist
>> [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
>> [ERROR] ^
>> [ERROR]
>>
>>> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
>>>
>>> [CALCITE-2347] running ElasticSearch in embedded mode for unit tests of
>> ES adapter (Andrei Sereda)
>>>
>>> After discussion on dev-list Integration tests (for ES) have been
>> removed. They're now
>>> superseded by unit tests (which execute queries against a real elastic
>> instance)
>>>
>>> Added local file (zips-mini.json) which contains a small subset of
>> original zips.json
>>> (allows to bootstrap tests faster)
>>>
>>> Created separate ES JUnit rule which can be re-used across different
>> tests.
>>>
>>> Both v2 and v5 of ES adapters are supported.
>>>
>>> Close apache/calcite#716
>>>
>>>
>>> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
>>> Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
>>> Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
>>> Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
>>>
>>> Branch: refs/heads/master
>>> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
>>> Parents: 37944bb
>>> Author: Andrei Sereda <an...@nospam.com>
>>> Authored: Thu May 31 18:19:10 2018 -0400
>>> Committer: Michael Mior <mm...@uwaterloo.ca>
>>> Committed: Thu Jun 21 06:38:50 2018 -0400
>>>
>>> ----------------------------------------------------------------------
>>> .../AbstractElasticsearchTable.java | 12 +
>>> .../elasticsearch/ElasticsearchProject.java | 61 ++-
>>> elasticsearch2/pom.xml | 6 +
>>> .../Elasticsearch2Enumerator.java | 12 +-
>>> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
>>> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
>>> .../ElasticSearch2AdapterTest.java | 395 ++++++++++++++++++
>>> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
>>> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
>>> .../calcite/test/Elasticsearch2AdapterIT.java | 270 -------------
>>> .../resources/elasticsearch-zips-model.json | 50 ---
>>> .../src/test/resources/zips-mini.json | 149 +++++++
>>> elasticsearch5/pom.xml | 31 ++
>>> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
>>> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
>>> .../ElasticSearch5AdapterTest.java | 399 +++++++++++++++++++
>>> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
>>> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
>>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
>>> .../calcite/test/Elasticsearch5AdapterIT.java | 270 -------------
>>> .../resources/elasticsearch-zips-model.json | 50 ---
>>> elasticsearch5/src/test/resources/log4j2.xml | 16 +
>>> .../src/test/resources/zips-mini.json | 149 +++++++
>>> pom.xml | 20 +-
>>> 25 files changed, 1866 insertions(+), 670 deletions(-)
>>> ----------------------------------------------------------------------
>>>
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>> ----------------------------------------------------------------------
>>> diff --git
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>> index 0980469..8cc5933 100644
>>> ---
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>> +++
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>>> @@ -75,6 +75,18 @@ public abstract class AbstractElasticsearchTable
>> extends AbstractQueryableTable
>>> relOptTable, this, null);
>>> }
>>>
>>> + /**
>>> + * In ES 5.x scripted fields start with {@code params._source.foo}
>> while in ES2.x
>>> + * {@code _source.foo}. Helper method to build correct query based on
>> runtime version of elastic.
>>> + *
>>> + * @see <a href="
>> https://github.com/elastic/elasticsearch/issues/20068">_source
>> variable</a>
>>> + * @see <a href="
>> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html">Scripted
>> Fields</a>
>>> + */
>>> + protected String scriptedFieldPrefix() {
>>> + // this is default pattern starting 5.x
>>> + return "params._source";
>>> + }
>>> +
>>> /** Executes a "find" operation on the underlying type.
>>> *
>>> * <p>For example,
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>> ----------------------------------------------------------------------
>>> diff --git
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>> index b42abd7..961c8b0 100644
>>> ---
>> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>> +++
>> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>>> @@ -27,11 +27,15 @@ import
>> org.apache.calcite.rel.metadata.RelMetadataQuery;
>>> import org.apache.calcite.rel.type.RelDataType;
>>> import org.apache.calcite.rex.RexNode;
>>> import org.apache.calcite.util.Pair;
>>> -import org.apache.calcite.util.Util;
>>> +
>>> +import com.google.common.base.Function;
>>> +import com.google.common.collect.Lists;
>>>
>>> import java.util.ArrayList;
>>> import java.util.List;
>>>
>>> +import javax.annotation.Nullable;
>>> +
>>> /**
>>> * Implementation of {@link org.apache.calcite.rel.core.Project}
>>> * relational expression in Elasticsearch.
>>> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends Project
>> implements ElasticsearchRel {
>>> implementor.visitChild(0, getInput());
>>>
>>> final List<String> inFields =
>>> -
>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
>>> +
>> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
>>> final ElasticsearchRules.RexToElasticsearchTranslator translator =
>>> - new ElasticsearchRules.RexToElasticsearchTranslator(
>>> - (JavaTypeFactory) getCluster().getTypeFactory(), inFields);
>>> + new ElasticsearchRules.RexToElasticsearchTranslator(
>>> + (JavaTypeFactory) getCluster().getTypeFactory(),
>> inFields);
>>>
>>> - final List<String> findItems = new ArrayList<>();
>>> - final List<String> scriptFieldItems = new ArrayList<>();
>>> + final List<String> fields = new ArrayList<>();
>>> + final List<String> scriptFields = new ArrayList<>();
>>> for (Pair<RexNode, String> pair: getNamedProjects()) {
>>> final String name = pair.right;
>>> final String expr = pair.left.accept(translator);
>>>
>>> if (expr.equals("\"" + name + "\"")) {
>>> - findItems.add(ElasticsearchRules.quote(name));
>>> + fields.add(name);
>>> } else if (expr.matches("\"literal\":.+")) {
>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
>>> - + ":{\"script\": "
>>> - + expr.split(":")[1] + "}");
>>> + scriptFields.add(ElasticsearchRules.quote(name)
>>> + + ":{\"script\": "
>>> + + expr.split(":")[1] + "}");
>>> } else {
>>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
>>> - + ":{\"script\":\"params._source."
>>> - + expr.replaceAll("\"", "") + "\"}");
>>> + scriptFields.add(ElasticsearchRules.quote(name)
>>> + + ":{\"script\":"
>>> + // _source (ES2) vs params._source (ES5)
>>> + + "\"" +
>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
>>> + + expr.replaceAll("\"", "") + "\"}");
>>> + }
>>> + }
>>> +
>>> + StringBuilder query = new StringBuilder();
>>> + if (scriptFields.isEmpty()) {
>>> + List<String> newList = Lists.transform(fields, new
>> Function<String, String>() {
>>> + @Nullable
>>> + @Override public String apply(@Nullable String input) {
>>> + return ElasticsearchRules.quote(input);
>>> + }
>>> + });
>>> +
>>> + final String findString = String.join(", ", newList);
>>> + query.append("\"_source\" : [").append(findString).append("]");
>>> + } else {
>>> + // if scripted fields are present, ES ignores _source attribute
>>> + for (String field: fields) {
>>> + scriptFields.add(ElasticsearchRules.quote(field) +
>> ":{\"script\": "
>>> + // _source (ES2) vs params._source (ES5)
>>> + + "\"" +
>> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
>>> + + field + "\"}");
>>> }
>>> + query.append("\"script_fields\": {" + String.join(", ",
>> scriptFields) + "}");
>>> }
>>> - final String findString = Util.toString(findItems, "", ", ", "");
>>> - final String scriptFieldString = "\"script_fields\": {"
>>> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
>>> - final String fieldString = "\"_source\" : [" + findString + "]"
>>> - + ", " + scriptFieldString;
>>>
>>> for (String opfield : implementor.list) {
>>> if (opfield.startsWith("\"_source\"")) {
>>> implementor.list.remove(opfield);
>>> }
>>> }
>>> - implementor.add(fieldString);
>>> + implementor.add(query.toString());
>>> }
>>> }
>>>
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
>>> ----------------------------------------------------------------------
>>> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
>>> index f24622c..6fbee03 100644
>>> --- a/elasticsearch2/pom.xml
>>> +++ b/elasticsearch2/pom.xml
>>> @@ -73,6 +73,12 @@ limitations under the License.
>>> <version>${elasticsearch-java-driver.version}</version>
>>> </dependency>
>>> <dependency>
>>> + <!-- Lang groovy dependency is needed for testing with embedded
>> ES (scripted fields like loc[0]) -->
>>> + <groupId>org.elasticsearch.module</groupId>
>>> + <artifactId>lang-groovy</artifactId>
>>> + <scope>test</scope>
>>> + </dependency>
>>> + <dependency>
>>> <groupId>com.carrotsearch</groupId>
>>> <artifactId>hppc</artifactId>
>>> <version>${hppc.version}</version>
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>> ----------------------------------------------------------------------
>>> diff --git
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>> index 84370ab..c3d2ac0 100644
>>> ---
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>> +++
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>>> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
>>> import java.util.Date;
>>> import java.util.Iterator;
>>> import java.util.List;
>>> +import java.util.Locale;
>>> import java.util.Map;
>>>
>>> /**
>>> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator implements
>> Enumerator<Object> {
>>> private static Function1<SearchHit, Object[]> listGetter(
>>> final List<Map.Entry<String, Class>> fields) {
>>> return new Function1<SearchHit, Object[]>() {
>>> - public Object[] apply(SearchHit searchHitFields) {
>>> + public Object[] apply(SearchHit hit) {
>>> Object[] objects = new Object[fields.size()];
>>> for (int i = 0; i < fields.size(); i++) {
>>> final Map.Entry<String, Class> field = fields.get(i);
>>> final String name = field.getKey();
>>> - if (searchHitFields.fields().isEmpty()) {
>>> - objects[i] = convert(searchHitFields.getSource().get(name),
>> field.getValue());
>>> + if (hit.fields().isEmpty()) {
>>> + objects[i] = convert(hit.getSource().get(name),
>> field.getValue());
>>> + } else if (hit.fields().containsKey(name)) {
>>> + objects[i] = convert(hit.field(name).getValue(),
>> field.getValue());
>>> } else {
>>> - objects[i] =
>> convert(searchHitFields.field(name).getValue(), field.getValue());
>>> + throw new IllegalStateException(
>>> + String.format(Locale.getDefault(), "No result for
>> %s", field));
>>> }
>>> }
>>> return objects;
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>> ----------------------------------------------------------------------
>>> diff --git
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>> index 668402b..46e3fc5 100644
>>> ---
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>> +++
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>>> @@ -22,6 +22,9 @@ import org.apache.calcite.schema.impl.AbstractSchema;
>>>
>>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
>>>
>>> +import com.google.common.annotations.VisibleForTesting;
>>> +import com.google.common.base.Preconditions;
>>> +
>>> import com.google.common.collect.ImmutableList;
>>> import com.google.common.collect.ImmutableMap;
>>>
>>> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
>> AbstractSchema
>>> }
>>> }
>>>
>>> + /**
>>> + * Allows schema to be instantiated from existing elastic search
>> client.
>>> + * This constructor is used in tests.
>>> + */
>>> + @VisibleForTesting
>>> + Elasticsearch2Schema(Client client, String index) {
>>> + this.client = Preconditions.checkNotNull(client, "client");
>>> + this.index = Preconditions.checkNotNull(index, "index");
>>> + }
>>> +
>>> @Override protected Map<String, Table> getTableMap() {
>>> final ImmutableMap.Builder<String, Table> builder =
>> ImmutableMap.builder();
>>>
>>> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
>> AbstractSchema
>>>
>>> final List<DiscoveryNode> nodes =
>> ImmutableList.copyOf(transportClient.connectedNodes());
>>> if (nodes.isEmpty()) {
>>> - throw new RuntimeException("Cannot connect to any elasticsearch
>> nodes");
>>> + throw new IllegalStateException("Cannot connect to any
>> elasticsearch node: "
>>> + + transportNodes);
>>> }
>>>
>>> client = transportClient;
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>> ----------------------------------------------------------------------
>>> diff --git
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>> index 636aa5f..2928835 100644
>>> ---
>> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>> +++
>> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>>> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
>> AbstractElasticsearchTable {
>>> this.client = client;
>>> }
>>>
>>> + /**
>>> + * ES version 2.x. To access document attributes ES2 uses {@code
>> _source.foo} syntax.
>>> + */
>>> + @Override protected String scriptedFieldPrefix() {
>>> + return "_source";
>>> + }
>>> +
>>> @Override protected Enumerable<Object> find(String index, List<String>
>> ops,
>>> - List<Map.Entry<String, Class>> fields) {
>>> + List<Map.Entry<String,
>> Class>> fields) {
>>> final String dbName = index;
>>>
>>> final String queryString = "{" + Util.toString(ops, "", ", ", "") +
>> "}";
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>> ----------------------------------------------------------------------
>>> diff --git
>> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>> new file mode 100644
>>> index 0000000..287e094
>>> --- /dev/null
>>> +++
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>>> @@ -0,0 +1,395 @@
>>> +/*
>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>> + * contributor license agreements. See the NOTICE file distributed with
>>> + * this work for additional information regarding copyright ownership.
>>> + * The ASF licenses this file to you under the Apache License, Version
>> 2.0
>>> + * (the "License"); you may not use this file except in compliance with
>>> + * the License. You may obtain a copy of the License at
>>> + *
>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>> + *
>>> + * Unless required by applicable law or agreed to in writing, software
>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>> implied.
>>> + * See the License for the specific language governing permissions and
>>> + * limitations under the License.
>>> + */
>>> +package org.apache.calcite.adapter.elasticsearch2;
>>> +
>>> +import org.apache.calcite.jdbc.CalciteConnection;
>>> +import org.apache.calcite.schema.SchemaPlus;
>>> +import org.apache.calcite.schema.impl.ViewTable;
>>> +import org.apache.calcite.schema.impl.ViewTableMacro;
>>> +import org.apache.calcite.test.CalciteAssert;
>>> +import org.apache.calcite.test.ElasticChecker;
>>> +
>>> +import com.google.common.io.LineProcessor;
>>> +import com.google.common.io.Resources;
>>> +
>>> +import org.elasticsearch.action.bulk.BulkItemResponse;
>>> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
>>> +import org.elasticsearch.action.bulk.BulkResponse;
>>> +import org.elasticsearch.common.xcontent.XContentBuilder;
>>> +import org.elasticsearch.common.xcontent.XContentFactory;
>>> +
>>> +import org.junit.BeforeClass;
>>> +import org.junit.ClassRule;
>>> +import org.junit.Test;
>>> +
>>> +import java.io.IOException;
>>> +import java.nio.charset.StandardCharsets;
>>> +import java.sql.Connection;
>>> +import java.sql.DriverManager;
>>> +import java.sql.SQLException;
>>> +import java.util.Arrays;
>>> +import java.util.Collections;
>>> +import java.util.Locale;
>>> +
>>> +/**
>>> + * Set of tests for ES adapter. Uses real instance via {@link
>> EmbeddedElasticRule}. Document
>>> + * source is local {@code zips-mini.json} file (located in the
>> classpath).
>>> + */
>>> +public class ElasticSearch2AdapterTest {
>>> +
>>> + @ClassRule //init once for all tests
>>> + public static final EmbeddedElasticRule NODE =
>> EmbeddedElasticRule.create();
>>> +
>>> + private static final String ZIPS = "zips";
>>> +
>>> + /**
>>> + * Used to create {@code zips} index and insert some data
>>> + */
>>> + @BeforeClass
>>> + public static void setupInstance() throws Exception {
>>> + // define mapping so fields are searchable (term query)
>>> + XContentBuilder mapping =
>> XContentFactory.jsonBuilder().startObject()
>>> + .startObject("properties")
>>> + .startObject("city").field("type", "string")
>>> + .field("index", "not_analyzed").endObject()
>>> + .startObject("state").field("type", "string")
>>> + .field("index", "not_analyzed").endObject()
>>> + .startObject("pop").field("type", "long").endObject()
>>> + .endObject()
>>> + .endObject();
>>> +
>>> + // create index
>>> + NODE.client().admin().indices()
>>> + .prepareCreate(ZIPS)
>>> + .addMapping(ZIPS, mapping)
>>> + .get();
>>> +
>>> + BulkRequestBuilder bulk =
>> NODE.client().prepareBulk().setRefresh(true);
>>> +
>>> + // load records from file
>>> +
>> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
>>> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
>>> + @Override public boolean processLine(String line) throws
>> IOException {
>>> + line = line.replaceAll("_id", "id"); // _id is a
>> reserved attribute in ES
>>> + bulk.add(NODE.client().prepareIndex(ZIPS,
>> ZIPS).setSource(line));
>>> + return true;
>>> + }
>>> +
>>> + @Override public Void getResult() {
>>> + return null;
>>> + }
>>> + });
>>> +
>>> + if (bulk.numberOfActions() == 0) {
>>> + throw new IllegalStateException("No records to be indexed");
>>> + }
>>> +
>>> + BulkResponse response = bulk.execute().get();
>>> +
>>> + if (response.hasFailures()) {
>>> + throw new IllegalStateException(
>>> + String.format(Locale.getDefault(), "Failed to populate
>> %s:\n%s", NODE.httpAddress(),
>>> +
>> Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
>>> +
>> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
>>> + }
>>> +
>>> + }
>>> +
>>> + private CalciteAssert.ConnectionFactory newConnectionFactory() {
>>> + return new CalciteAssert.ConnectionFactory() {
>>> + @Override public Connection createConnection() throws
>> SQLException {
>>> + final Connection connection =
>> DriverManager.getConnection("jdbc:calcite:");
>>> + final SchemaPlus root =
>> connection.unwrap(CalciteConnection.class).getRootSchema();
>>> +
>>> + root.add("elastic", new Elasticsearch2Schema(NODE.client(),
>> ZIPS));
>>> +
>>> + // add calcite view programmatically
>>> + final String viewSql = "select cast(_MAP['city'] AS
>> varchar(20)) AS \"city\", "
>>> + + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n"
>>> + + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n"
>>> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
>>> + + " cast(_MAP['state'] AS varchar(2)) AS \"state\", "
>>> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
>>> + + "from \"elastic\".\"zips\"";
>>> +
>>> + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
>>> + Collections.singletonList("elastic"),
>> Arrays.asList("elastic", "view"), false);
>>> + root.add("ZIPS", macro);
>>> +
>>> + return connection;
>>> + }
>>> + };
>>> + }
>>> +
>>> + private CalciteAssert.AssertThat calciteAssert() {
>>> + return CalciteAssert.that()
>>> + .with(newConnectionFactory());
>>> + }
>>> +
>>> + /**
>>> + * Tests using calcite view
>>> + */
>>> + @Test
>>> + public void view() throws Exception {
>>> + calciteAssert()
>>> + .query("select * from zips where \"city\" = 'BROOKLYN'")
>>> + .returns("city=BROOKLYN; longitude=-73.956985;
>> latitude=40.646694; "
>>> + + "pop=111396; state=NY; id=11226\n")
>>> + .returnsCount(1);
>>> + }
>>> +
>>> + @Test
>>> + public void emptyResult() {
>>> + CalciteAssert.that()
>>> + .with(newConnectionFactory())
>>> + .query("select * from zips limit 0")
>>> + .returnsCount(0);
>>> +
>>> + CalciteAssert.that()
>>> + .with(newConnectionFactory())
>>> + .query("select * from \"elastic\".\"zips\" where
>> _MAP['Foo'] = '_MISSING_'")
>>> + .returnsCount(0);
>>> + }
>>> +
>>> + @Test
>>> + public void basic() throws Exception {
>>> + CalciteAssert.that()
>>> + .with(newConnectionFactory())
>>> + .query("select * from \"elastic\".\"zips\" where
>> _MAP['city'] = 'BROOKLYN'")
>>> + .returnsCount(1);
>>> +
>>> + CalciteAssert.that()
>>> + .with(newConnectionFactory())
>>> + .query("select * from \"elastic\".\"zips\" where"
>>> + + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')")
>>> + .returnsCount(2);
>>> +
>>> + // lower-case
>>> + CalciteAssert.that()
>>> + .with(newConnectionFactory())
>>> + .query("select * from \"elastic\".\"zips\" where "
>>> + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
>> 'BROOK') ")
>>> + .returnsCount(0);
>>> +
>>> + // missing field
>>> + CalciteAssert.that()
>>> + .with(newConnectionFactory())
>>> + .query("select * from \"elastic\".\"zips\" where
>> _MAP['CITY'] = 'BROOKLYN'")
>>> + .returnsCount(0);
>>> +
>>> + // limit works
>>> + CalciteAssert.that()
>>> + .with(newConnectionFactory())
>>> + .query("select * from \"elastic\".\"zips\" limit 42")
>>> + .returnsCount(42);
>>> +
>>> + }
>>> +
>>> + @Test public void testSort() {
>>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
>>> +
>>> + calciteAssert()
>>> + .query("select * from zips order by \"state\"")
>>> + .returnsCount(10)
>>> + .explainContains(explain);
>>> + }
>>> +
>>> + @Test public void testSortLimit() {
>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>> + + "order by \"state\", \"id\" offset 2 rows fetch next 3
>> rows only";
>>> + calciteAssert()
>>> + .query(sql)
>>> + .returnsUnordered("state=AK; id=99801",
>>> + "state=AL; id=35215",
>>> + "state=AL; id=35401")
>>> + .queryContains(
>>> + ElasticChecker.elasticsearchChecker(
>>> + "\"_source\" : [\"state\", \"id\"]",
>>> + "\"sort\": [ {\"state\": \"asc\"}, {\"id\":
>> \"asc\"}]",
>>> + "\"from\": 2",
>>> + "\"size\": 3"));
>>> + }
>>> +
>>> +
>>> +
>>> + @Test public void testOffsetLimit() {
>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>> + + "offset 2 fetch next 3 rows only";
>>> + calciteAssert()
>>> + .query(sql)
>>> + .runs()
>>> + .queryContains(
>>> + ElasticChecker.elasticsearchChecker(
>>> + "\"from\": 2",
>>> + "\"size\": 3",
>>> + "\"_source\" : [\"state\", \"id\"]"));
>>> + }
>>> +
>>> + @Test public void testLimit() {
>>> + final String sql = "select \"state\", \"id\" from zips\n"
>>> + + "fetch next 3 rows only";
>>> +
>>> + calciteAssert()
>>> + .query(sql)
>>> + .runs()
>>> + .queryContains(
>>> + ElasticChecker.elasticsearchChecker(
>>> + "\"size\": 3",
>>> + "\"_source\" : [\"state\", \"id\"]"));
>>> + }
>>> +
>>> + @Test public void testFilterSort() {
>>> + final String sql = "select * from zips\n"
>>> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
>>> + + "order by \"state\", \"id\"";
>>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>> + + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC],
>> dir1=[ASC])\n"
>>> + + " ElasticsearchProject(city=[CAST(ITEM($0,
>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>> + + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0, 'id')):VARCHAR(5)
>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\",
>> '70000'))])\n"
>>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
>>> + calciteAssert()
>>> + .query(sql)
>>> + .returnsOrdered("city=LOS ANGELES; longitude=-118.258189;
>> latitude=34.007856; "
>>> + + "pop=96074; state=CA; id=90011",
>>> + "city=BELL GARDENS; longitude=-118.17205;
>> latitude=33.969177; "
>>> + + "pop=99568; state=CA; id=90201",
>>> + "city=NORWALK; longitude=-118.081767;
>> latitude=33.90564; "
>>> + + "pop=94188; state=CA; id=90650")
>>> + .queryContains(
>>> + ElasticChecker.elasticsearchChecker("\"query\" : "
>>> + +
>> "{\"constant_score\":{\"filter\":{\"bool\":"
>>> + +
>> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
>>> + +
>> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
>>> + "\"script_fields\":
>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>> + +
>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>> + + "\"city\":{\"script\":
>> \"_source.city\"}, "
>>> + + "\"pop\":{\"script\":
>> \"_source.pop\"}, "
>>> + + "\"state\":{\"script\":
>> \"_source.state\"}, "
>>> + + "\"id\":{\"script\": \"_source.id
>> \"}}",
>>> + "\"sort\": [ {\"state\": \"asc\"}, {\"id\":
>> \"asc\"}]"))
>>> + .explainContains(explain);
>>> + }
>>> +
>>> + @Test public void testFilterSortDesc() {
>>> + final String sql = "select * from zips\n"
>>> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
>>> + + "order by \"state\" desc, \"pop\"";
>>> + calciteAssert()
>>> + .query(sql)
>>> + .limit(4)
>>> + .returnsOrdered(
>>> + "city=LOS ANGELES; longitude=-118.258189;
>> latitude=34.007856; pop=96074; state=CA; id=90011",
>>> + "city=BELL GARDENS; longitude=-118.17205;
>> latitude=33.969177; pop=99568; state=CA; id=90201");
>>> + }
>>> +
>>> + @Test public void testFilterRedundant() {
>>> + final String sql = "select * from zips\n"
>>> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and
>> \"state\" = 'OK'";
>>> + calciteAssert()
>>> + .query(sql)
>>> + .runs()
>>> + .queryContains(
>>> + ElasticChecker.elasticsearchChecker(""
>>> + + "\"query\" :
>> {\"constant_score\":{\"filter\":{\"bool\":"
>>> + +
>> "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
>>> + "\"script_fields\":
>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>> + +
>> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>> + + "\"city\":{\"script\":
>> \"_source.city\"}, "
>>> + + "\"pop\":{\"script\":
>> \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
>>> + + "\"id\":{\"script\":
>> \"_source.id\"}}"
>>> + ));
>>> + }
>>> +
>>> + @Test public void testInPlan() {
>>> + final String[] searches = {
>>> + "\"query\" :
>> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
>>> + +
>> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
>>> + + "{\"pop\":99568}}]}}]}}}}",
>>> + "\"script_fields\":
>> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>>> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>>> + + "\"city\":{\"script\": \"_source.city\"}, "
>>> + + "\"pop\":{\"script\": \"_source.pop\"}, "
>>> + + "\"state\":{\"script\": \"_source.state\"}, "
>>> + + "\"id\":{\"script\": \"_source.id\"}}"
>>> + };
>>> +
>>> + calciteAssert()
>>> + .query("select * from zips where \"pop\" in (96074, 99568)")
>>> + .returnsUnordered(
>>> + "city=BELL GARDENS; longitude=-118.17205;
>> latitude=33.969177; pop=99568; state=CA; id=90201",
>>> + "city=LOS ANGELES; longitude=-118.258189;
>> latitude=34.007856; pop=96074; state=CA; id=90011"
>>> + )
>>> +
>> .queryContains(ElasticChecker.elasticsearchChecker(searches));
>>> + }
>>> +
>>> + @Test public void testZips() {
>>> + calciteAssert()
>>> + .query("select \"state\", \"city\" from zips")
>>> + .returnsCount(10);
>>> + }
>>> +
>>> + @Test public void testProject() {
>>> + final String sql = "select \"state\", \"city\", 0 as \"zero\"\n"
>>> + + "from zips\n"
>>> + + "order by \"state\", \"city\"";
>>> +
>>> + calciteAssert()
>>> + .query(sql)
>>> + .limit(2)
>>> + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0",
>>> + "state=AK; city=FAIRBANKS; zero=0")
>>> + .queryContains(
>>> +
>> ElasticChecker.elasticsearchChecker("\"script_fields\": "
>>> + + "{\"zero\":{\"script\": \"0\"}, "
>>> + + "\"state\":{\"script\":
>> \"_source.state\"}, "
>>> + + "\"city\":{\"script\":
>> \"_source.city\"}}",
>>> + "\"sort\": [ {\"state\": \"asc\"},
>> {\"city\": \"asc\"}]"));
>>> + }
>>> +
>>> + @Test public void testFilter() {
>>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>> + + " ElasticsearchProject(state=[CAST(ITEM($0,
>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20)
>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>> + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0,
>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\", 'CA')])\n"
>>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
>>> + calciteAssert()
>>> + .query("select \"state\", \"city\" from zips where
>> \"state\" = 'CA'")
>>> + .limit(3)
>>> + .returnsUnordered("state=CA; city=BELL GARDENS",
>>> + "state=CA; city=LOS ANGELES",
>>> + "state=CA; city=NORWALK")
>>> + .explainContains(explain);
>>> + }
>>> +
>>> + @Test public void testFilterReversed() {
>>> + calciteAssert()
>>> + .query("select \"state\", \"city\" from zips where 'WI' <
>> \"state\" order by \"city\"")
>>> + .limit(2)
>>> + .returnsUnordered("state=WV; city=BECKLEY",
>>> + "state=WY; city=CHEYENNE");
>>> + calciteAssert()
>>> + .query("select \"state\", \"city\" from zips where \"state\"
>>> 'WI' order by \"city\"")
>>> + .limit(2)
>>> + .returnsUnordered("state=WV; city=BECKLEY",
>>> + "state=WY; city=CHEYENNE");
>>> + }
>>> +
>>> +}
>>> +
>>> +// End ElasticSearch2AdapterTest.java
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>> ----------------------------------------------------------------------
>>> diff --git
>> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>> new file mode 100644
>>> index 0000000..4474add
>>> --- /dev/null
>>> +++
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>>> @@ -0,0 +1,147 @@
>>> +/*
>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>> + * contributor license agreements. See the NOTICE file distributed with
>>> + * this work for additional information regarding copyright ownership.
>>> + * The ASF licenses this file to you under the Apache License, Version
>> 2.0
>>> + * (the "License"); you may not use this file except in compliance with
>>> + * the License. You may obtain a copy of the License at
>>> + *
>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>> + *
>>> + * Unless required by applicable law or agreed to in writing, software
>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>> implied.
>>> + * See the License for the specific language governing permissions and
>>> + * limitations under the License.
>>> + */
>>> +package org.apache.calcite.adapter.elasticsearch2;
>>> +
>>> +import com.google.common.base.Preconditions;
>>> +import com.google.common.io.Files;
>>> +
>>> +import org.elasticsearch.Version;
>>> +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
>>> +import org.elasticsearch.action.admin.cluster.node.info
>> .NodesInfoResponse;
>>> +import org.elasticsearch.client.Client;
>>> +import org.elasticsearch.common.settings.Settings;
>>> +import org.elasticsearch.common.transport.TransportAddress;
>>> +import org.elasticsearch.node.Node;
>>> +import org.elasticsearch.node.internal.InternalSettingsPreparer;
>>> +import org.elasticsearch.plugins.Plugin;
>>> +import org.elasticsearch.script.groovy.GroovyPlugin;
>>> +
>>> +import java.io.File;
>>> +import java.util.Arrays;
>>> +import java.util.Collection;
>>> +import java.util.Collections;
>>> +
>>> +/**
>>> + * Represents a single elastic search node which can run embedded in a
>> java application.
>>> + * Intended for unit and integration tests. Settings and plugins are
>> crafted for Calcite.
>>> + */
>>> +class EmbeddedElasticNode implements AutoCloseable {
>>> +
>>> + private final LocalNode node;
>>> + private volatile boolean isStarted;
>>> +
>>> + private EmbeddedElasticNode(LocalNode node) {
>>> + this.node = Preconditions.checkNotNull(node, "node");
>>> + }
>>> +
>>> + /**
>>> + * Having separate class to expose (protected) constructor which
>> allows to install
>>> + * different plugins. In our case it is {@code GroovyPlugin} for
>> scripted fields like
>>> + * {@code loc[0]} or {@code loc[1]['foo']}.
>>> + */
>>> + private static class LocalNode extends Node {
>>> + private LocalNode(Settings settings, Collection<Class<? extends
>> Plugin>> classpathPlugins) {
>>> + super(InternalSettingsPreparer.prepareEnvironment(settings, null),
>>> + Version.CURRENT,
>>> + classpathPlugins);
>>> + }
>>> + }
>>> +
>>> + /**
>>> + * Creates an instance with existing settings
>>> + */
>>> + private static EmbeddedElasticNode create(Settings settings) {
>>> + // ensure GroovyPlugin is installed or otherwise scripted fields
>> would not work
>>> + LocalNode node = new LocalNode(settings,
>> Collections.singleton(GroovyPlugin.class));
>>> + return new EmbeddedElasticNode(node);
>>> + }
>>> +
>>> + /**
>>> + * Creates elastic node as single member of a cluster. Node will not
>> be started
>>> + * unless {@link #start()} is explicitly called.
>>> + */
>>> + public static EmbeddedElasticNode create() {
>>> + File data = Files.createTempDir();
>>> + data.deleteOnExit();
>>> + File home = Files.createTempDir();
>>> + home.deleteOnExit();
>>> +
>>> + Settings settings = Settings.builder()
>>> + .put("node.name", "fake-elastic")
>>> + .put("path.home", home.getAbsolutePath())
>>> + .put("path.data", data.getAbsolutePath())
>>> + .put("script.inline", true) // requires GroovyPlugin
>>> + .put("script.indexed", true) // requires GroovyPlugin
>>> + .put("cluster.routing.allocation.disk.threshold_enabled",
>> false)
>>> + .put("node.local", true)
>>> + .put("node.data", true)
>>> + .put("network.host", "localhost")
>>> + .build();
>>> +
>>> + return create(settings);
>>> + }
>>> +
>>> + /**
>>> + * Starts current node
>>> + */
>>> + public void start() {
>>> + Preconditions.checkState(!isStarted, "already started");
>>> + node.start();
>>> + this.isStarted = true;
>>> + }
>>> +
>>> + /**
>>> + * Returns current address to connect to with HTTP client.
>>> + */
>>> + public TransportAddress httpAddress() {
>>> + Preconditions.checkState(isStarted, "node is not started");
>>> +
>>> + NodesInfoResponse response =
>> client().admin().cluster().prepareNodesInfo()
>>> + .execute().actionGet();
>>> + if (response.getNodes().length != 1) {
>>> + throw new IllegalStateException("Expected single node but got "
>>> + + response.getNodes().length);
>>> + }
>>> + NodeInfo node = response.getNodes()[0];
>>> + return node.getHttp().address().boundAddresses()[0];
>>> + }
>>> +
>>> + /**
>>> + * Exposes elastic
>>> + * <a href="
>> https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html">transport
>> client</a>
>>> + *
>>> + * (use of HTTP client is preferred).
>>> + */
>>> + public Client client() {
>>> + Preconditions.checkState(isStarted, "node is not started");
>>> + return node.client();
>>> + }
>>> +
>>> + @Override public void close() throws Exception {
>>> + node.close();
>>> + // cleanup data dirs
>>> + File data = new File(node.settings().get("path.data"));
>>> + File home = new File(node.settings().get("path.home"));
>>> + for (File file: Arrays.asList(data, home)) {
>>> + if (file.exists()) {
>>> + file.delete();
>>> + }
>>> + }
>>> + }
>>> +}
>>> +
>>> +// End EmbeddedElasticNode.java
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>> ----------------------------------------------------------------------
>>> diff --git
>> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>> new file mode 100644
>>> index 0000000..a633078
>>> --- /dev/null
>>> +++
>> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>>> @@ -0,0 +1,97 @@
>>> +/*
>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>> + * contributor license agreements. See the NOTICE file distributed with
>>> + * this work for additional information regarding copyright ownership.
>>> + * The ASF licenses this file to you under the Apache License, Version
>> 2.0
>>> + * (the "License"); you may not use this file except in compliance with
>>> + * the License. You may obtain a copy of the License at
>>> + *
>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>> + *
>>> + * Unless required by applicable law or agreed to in writing, software
>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>> implied.
>>> + * See the License for the specific language governing permissions and
>>> + * limitations under the License.
>>> + */
>>> +package org.apache.calcite.adapter.elasticsearch2;
>>> +
>>> +import com.google.common.base.Preconditions;
>>> +
>>> +import org.elasticsearch.client.Client;
>>> +import org.elasticsearch.common.transport.TransportAddress;
>>> +import org.junit.rules.ExternalResource;
>>> +
>>> +/**
>>> + * Used to initialize a single elastic node. For performance reasons
>> (node startup costs),
>>> + * same instance is usually shared across multiple tests.
>>> + *
>>> + * This rule should be used as follows:
>>> + * <pre>
>>> + * {@code
>>> + *
>>> + * public class MyTest {
>>> + * @literal @ClassRule
>>> + * public static final ElasticSearchRule RULE =
>> ElasticSearchRule.create();
>>> + *
>>> + * @literal @BeforeClass
>>> + * public void setup() {
>>> + * // ... populate instance
>>> + * }
>>> + *
>>> + * @literal @Test
>>> + * public void myTest() {
>>> + * TransportAddress address = RULE.httpAddress();
>>> + * // ....
>>> + * }
>>> + * }
>>> + * }
>>> + * </pre>
>>> + *
>>> + * @see ExternalResource
>>> + */
>>> +class EmbeddedElasticRule extends ExternalResource {
>>> +
>>> + private final EmbeddedElasticNode node;
>>> +
>>> + private EmbeddedElasticRule(EmbeddedElasticNode resource) {
>>> + this.node = Preconditions.checkNotNull(resource, "resource");
>>> + }
>>> +
>>> + @Override protected void before() throws Throwable {
>>> + node.start();
>>> + }
>>> +
>>> + @Override protected void after() {
>>> + try {
>>> + node.close();
>>> + } catch (Exception e) {
>>> + throw new RuntimeException(e);
>>> + }
>>> + }
>>> +
>>> + /**
>>> + * Factory method to create this rule.
>>> + */
>>> + public static EmbeddedElasticRule create() {
>>> + return new EmbeddedElasticRule(EmbeddedElasticNode.create());
>>> + }
>>> +
>>> + /**
>>> + * Exposes current ES transport client.
>>> + */
>>> + Client client() {
>>> + return node.client();
>>> + }
>>> +
>>> + /**
>>> + * HTTP address for rest clients (can be ES native or any other).
>>> + */
>>> + TransportAddress httpAddress() {
>>> + return node.httpAddress();
>>> + }
>>> +
>>> +
>>> +}
>>> +
>>> +// End EmbeddedElasticRule.java
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>> ----------------------------------------------------------------------
>>> diff --git
>> a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>> new file mode 100644
>>> index 0000000..21fc491
>>> --- /dev/null
>>> +++
>> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>>> @@ -0,0 +1,49 @@
>>> +/*
>>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>>> + * contributor license agreements. See the NOTICE file distributed with
>>> + * this work for additional information regarding copyright ownership.
>>> + * The ASF licenses this file to you under the Apache License, Version
>> 2.0
>>> + * (the "License"); you may not use this file except in compliance with
>>> + * the License. You may obtain a copy of the License at
>>> + *
>>> + * http://www.apache.org/licenses/LICENSE-2.0
>>> + *
>>> + * Unless required by applicable law or agreed to in writing, software
>>> + * distributed under the License is distributed on an "AS IS" BASIS,
>>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>> implied.
>>> + * See the License for the specific language governing permissions and
>>> + * limitations under the License.
>>> + */
>>> +package org.apache.calcite.test;
>>> +
>>> +import com.google.common.base.Function;
>>> +
>>> +import java.util.List;
>>> +
>>> +import javax.annotation.Nullable;
>>> +
>>> +/**
>>> + * Internal util methods for ElasticSearch tests
>>> + */
>>> +public class ElasticChecker {
>>> +
>>> + private ElasticChecker() {}
>>> +
>>> +
>>> + /** Returns a function that checks that a particular Elasticsearch
>> pipeline is
>>> + * generated to implement a query. */
>>> + public static Function<List, Void> elasticsearchChecker(final
>> String... strings) {
>>> + return new Function<List, Void>() {
>>> + @Nullable
>>> + @Override public Void apply(@Nullable List actual) {
>>> + Object[] actualArray = actual == null || actual.isEmpty() ? null
>>> + : ((List) actual.get(0)).toArray();
>>> + CalciteAssert.assertArrayEqual("expected Elasticsearch query
>> not found", strings,
>>> + actualArray);
>>> + return null;
>>> + }
>>> + };
>>> + }
>>> +}
>>> +
>>> +// End ElasticChecker.java
>>>
>>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>> ----------------------------------------------------------------------
>>> diff --git
>> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>> b/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>> deleted file mode 100644
>>> index 4e0c2b6..0000000
>>> ---
>> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>>> +++ /dev/null
>>> @@ -1,270 +0,0 @@
>>> -/*
>>> - * Licensed to the Apache Software Foundation (ASF) under one or more
>>> - * contributor license agreements. See the NOTICE file distributed with
>>> - * this work for additional information regarding copyright ownership.
>>> - * The ASF licenses this file to you under the Apache License, Version
>> 2.0
>>> - * (the "License"); you may not use this file except in compliance with
>>> - * the License. You may obtain a copy of the License at
>>> - *
>>> - * http://www.apache.org/licenses/LICENSE-2.0
>>> - *
>>> - * Unless required by applicable law or agreed to in writing, software
>>> - * distributed under the License is distributed on an "AS IS" BASIS,
>>> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
>> implied.
>>> - * See the License for the specific language governing permissions and
>>> - * limitations under the License.
>>> - */
>>> -package org.apache.calcite.test;
>>> -
>>> -import org.apache.calcite.util.Util;
>>> -
>>> -import com.google.common.base.Function;
>>> -import com.google.common.collect.ImmutableMap;
>>> -
>>> -import org.junit.Test;
>>> -
>>> -import java.util.List;
>>> -import javax.annotation.Nullable;
>>> -
>>> -/**
>>> - * Tests for the {@code org.apache.calcite.adapter.elasticsearch2}
>> package.
>>> - *
>>> - * <p>Before calling this test, you need to populate Elasticsearch, as
>> follows:
>>> - *
>>> - * <blockquote><code>
>>> - * git clone https://github.com/vlsi/calcite-test-dataset<br>
>>> - * cd calcite-test-dataset<br>
>>> - * mvn install
>>> - * </code></blockquote>
>>> - *
>>> - * <p>This will create a virtual machine with Elasticsearch and the
>> "zips" test
>>> - * dataset.
>>> - */
>>> -public class Elasticsearch2AdapterIT {
>>> - /**
>>> - * Whether to run Elasticsearch tests. Enabled by default, however
>> test is only
>>> - * included if "it" profile is activated ({@code -Pit}). To disable,
>>> - * specify {@code -Dcalcite.test.elasticsearch=false} on the Java
>> command line.
>>> - */
>>> - private static final boolean ENABLED =
>> Util.getBooleanProperty("calcite.test.elasticsearch",
>>> - true);
>>> -
>>> - /** Connection factory based on the "zips-es" model. */
>>> - private static final ImmutableMap<String, String> ZIPS =
>> ImmutableMap.of("model",
>>> -
>> Elasticsearch2AdapterIT.class.getResource("/elasticsearch-zips-model.json").getPath());
>>> -
>>> - /** Whether to run this test. */
>>> - private boolean enabled() {
>>> - return ENABLED;
>>> - }
>>> -
>>> - /** Returns a function that checks that a particular Elasticsearch
>> pipeline is
>>> - * generated to implement a query. */
>>> - private static Function<List, Void> elasticsearchChecker(final
>> String... strings) {
>>> - return new Function<List, Void>() {
>>> - @Nullable
>>> - @Override public Void apply(@Nullable List actual) {
>>> - Object[] actualArray = actual == null || actual.isEmpty() ? null
>>> - : ((List) actual.get(0)).toArray();
>>> - CalciteAssert.assertArrayEqual("expected Elasticsearch query
>> not found", strings,
>>> - actualArray);
>>> - return null;
>>> - }
>>> - };
>>> - }
>>> -
>>> - @Test public void testSort() {
>>> - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>> - + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
>>> - + " ElasticsearchProject(city=[CAST(ITEM($0,
>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>> - + " ElasticsearchTableScan(table=[[elasticsearch_raw,
>> zips]])";
>>> - CalciteAssert.that()
>>> - .enable(enabled())
>>> - .with(ZIPS)
>>> - .query("select * from zips order by \"state\"")
>>> - .returnsCount(10)
>>> - .explainContains(explain);
>>> - }
>>> -
>>> - @Test public void testSortLimit() {
>>> - final String sql = "select \"state\", \"id\" from zips\n"
>>> - + "order by \"state\", \"id\" offset 2 rows fetch next 3 rows
>> only";
>>> - CalciteAssert.that()
>>> - .with(ZIPS)
>>> - .query(sql)
>>> - .returnsUnordered("state=AK; id=99503",
>>> - "state=AK; id=99504",
>>> - "state=AK; id=99505")
>>> - .queryContains(
>>> - elasticsearchChecker(
>>> - "\"fields\" : [\"state\", \"id\"], \"script_fields\":
>> {}",
>>> - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]",
>>> - "\"from\": 2",
>>> - "\"size\": 3"));
>>> - }
>>> -
>>> - @Test public void testOffsetLimit() {
>>> - final String sql = "select \"state\", \"id\" from zips\n"
>>> - + "offset 2 fetch next 3 rows only";
>>> - CalciteAssert.that()
>>> - .enable(enabled())
>>> - .with(ZIPS)
>>> - .query(sql)
>>> - .runs()
>>> - .queryContains(
>>> - elasticsearchChecker(
>>> - "\"from\": 2",
>>> - "\"size\": 3",
>>> - "\"fields\" : [\"state\", \"id\"], \"script_fields\":
>> {}"));
>>> - }
>>> -
>>> - @Test public void testLimit() {
>>> - final String sql = "select \"state\", \"id\" from zips\n"
>>> - + "fetch next 3 rows only";
>>> - CalciteAssert.that()
>>> - .enable(enabled())
>>> - .with(ZIPS)
>>> - .query(sql)
>>> - .runs()
>>> - .queryContains(
>>> - elasticsearchChecker(
>>> - "\"size\": 3",
>>> - "\"fields\" : [\"state\", \"id\"], \"script_fields\":
>> {}"));
>>> - }
>>> -
>>> - @Test public void testFilterSort() {
>>> - final String sql = "select * from zips\n"
>>> - + "where \"city\" = 'SPRINGFIELD' and \"id\" >= '70000'\n"
>>> - + "order by \"state\", \"id\"";
>>> - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>>> - + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC],
>> dir1=[ASC])\n"
>>> - + " ElasticsearchProject(city=[CAST(ITEM($0,
>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
>> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
>> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
>> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
>> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>>> - + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
>> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\", 'SPRINGFIELD'), >=(CAST(ITEM($0,
>> 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE
>> \"ISO-8859-1$en_US$primary\", '70000'))])\n"
>>> - + " ElasticsearchTableScan(table=[[elasticsearch_raw,
>> zips]])";
>>> - CalciteAssert.that()
>>> - .enable(enabled())
>>> - .with(ZIPS)
>>> - .query(sql)
>>> - .returnsOrdered(
>>> - "city=SPRINGFIELD; longitude=-92.54567; latitude=35.274879;
>> pop=752; state=AR; id=72157",
>>> - "city=SPRINGFIELD; longitude=-102.617322;
>> latitude=37.406727; pop=1992; state=CO; id=81073",
>>> - "city=SPRINGFIELD; longitude=-90.577479;
>> latitude=30.415738; pop=5597; state=LA; id=70462",
>>> - "city=SPRINGFIELD; longitude=-123.015259;
>> latitude=44.06106; pop=32384; state=OR; id=97477",
>>> - "city=SPRINGFIELD; longitude=-122.917108;
>> latitude=44.056056; pop=27521; state=OR; id=97478")
>>> - .queryContains(
>>> - elasticsearchChecker("\"query\" :
>> {\"constant_score\":{\"filter\":{\"bool\":"
>>> - +
>> "{\"must\":[{\"term\":{\"city\":\"springfield\"}},{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
>>> - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"],
>> \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"},
>> \"latitude\":{\"script\":\"_source.loc[1]\"}}",
>>> - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]"))
>>> - .explainContains(explain);
>>> - }
>>> -
>>> - @Test public void testFilterSortDesc() {
>>> - final String sql = "select * from zips\n"
>>> - + "where \"pop\" BETWEEN 20000 AND 20100\n"
>>> - + "order by \"state\" desc, \"pop\"";
>>> - CalciteAssert.that()
>>> - .enable(enabled())
>>> - .with(ZIPS)
>>> - .query(sql)
>>> - .limit(4)
>>> - .returnsOrdered(
>>> - "city=SHERIDAN; longitude=-106.964795; latitude=44.78486;
>> pop=20025; state=WY; id=82801",
>>> - "city=MOUNTLAKE TERRAC; longitude=-122.304036;
>> latitude=47.793061; pop=20059; state=WA; id=98043",
>>> - "city=FALMOUTH; longitude=-77.404537; latitude=38.314557;
>> pop=20039; state=VA; id=22405",
>>> - "city=FORT WORTH; longitude=-97.318409; latitude=32.725551;
>> pop=20012; state=TX; id=76104");
>>> - }
>>> -
>>> - @Test public void testFilterRedundant() {
>>> - final String sql = "select * from zips\n"
>>> - + "where \"state\" > 'CA' and \"state\" < 'AZ' and \"state\" =
>> 'OK'";
>>> - CalciteAssert.that()
>>> - .enable(enabled())
>>> - .with(ZIPS)
>>> - .query(sql)
>>> - .runs()
>>> - .queryContains(
>>> - elasticsearchChecker(""
>>> - + "\"query\" :
>> {\"constant_score\":{\"filter\":{\"bool\":"
>>> - + "{\"must\":[{\"term\":{\"state\":\"ok\"}}]}}}}",
>>> - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"],
>> \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"},
>> \"latitude\":{\"script\":\"_source.loc[1]\"}}"));
>>> - }
>>> -
>>> - @Test public void testInPlan() {
>>> - final String[] searches = {
>>> - "\"query\" :
>> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
>>> - +
>> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":20012}}]}},{\"bool\":{\"must\":[{\"term\":"
>>> - + "{\"pop\":15590}}]}}]}}}}",
>>> - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"],
>> \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"},
>> \"latitude\":{\"script\":\"_source.loc[1]\"}}"
>>> - };
>>> - CalciteAssert.that()
>>> - .enable(enabled())
>>> - .with(ZIPS)
>>> - .query("select * from zips where \"pop\" in (20012, 15590)")
>>> - .returnsUnordered(
>>> - "city=COVINA; longitude=-117.884285; latitude=34.08596;
>> pop=15590; state=CA; id=91723",
>>> - "city=ARL
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Michael Mior <mm...@apache.org>.
Odd, mvn clean site still works fine for me.
--
Michael Mior
mmior@apache.org
Le ven. 22 juin 2018 à 17:12, Julian Hyde <jh...@apache.org> a écrit :
> Looks like this change broke “mvn site” (perhaps also “mvn
> javadoc:test-javadoc”).
>
> [ERROR] Failed to execute goal
> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate
> report:
> [ERROR] Exit code: 1 -
> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> error: package org.elasticsearch.node.internal does not exist
> [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
> [ERROR] ^
> [ERROR]
>
> > On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
> >
> > [CALCITE-2347] running ElasticSearch in embedded mode for unit tests of
> ES adapter (Andrei Sereda)
> >
> > After discussion on dev-list Integration tests (for ES) have been
> removed. They're now
> > superseded by unit tests (which execute queries against a real elastic
> instance)
> >
> > Added local file (zips-mini.json) which contains a small subset of
> original zips.json
> > (allows to bootstrap tests faster)
> >
> > Created separate ES JUnit rule which can be re-used across different
> tests.
> >
> > Both v2 and v5 of ES adapters are supported.
> >
> > Close apache/calcite#716
> >
> >
> > Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
> > Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
> > Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
> > Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
> >
> > Branch: refs/heads/master
> > Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
> > Parents: 37944bb
> > Author: Andrei Sereda <an...@nospam.com>
> > Authored: Thu May 31 18:19:10 2018 -0400
> > Committer: Michael Mior <mm...@uwaterloo.ca>
> > Committed: Thu Jun 21 06:38:50 2018 -0400
> >
> > ----------------------------------------------------------------------
> > .../AbstractElasticsearchTable.java | 12 +
> > .../elasticsearch/ElasticsearchProject.java | 61 ++-
> > elasticsearch2/pom.xml | 6 +
> > .../Elasticsearch2Enumerator.java | 12 +-
> > .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
> > .../elasticsearch2/Elasticsearch2Table.java | 9 +-
> > .../ElasticSearch2AdapterTest.java | 395 ++++++++++++++++++
> > .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
> > .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
> > .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> > .../calcite/test/Elasticsearch2AdapterIT.java | 270 -------------
> > .../resources/elasticsearch-zips-model.json | 50 ---
> > .../src/test/resources/zips-mini.json | 149 +++++++
> > elasticsearch5/pom.xml | 31 ++
> > .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
> > .../elasticsearch5/Elasticsearch5Table.java | 11 +-
> > .../ElasticSearch5AdapterTest.java | 399 +++++++++++++++++++
> > .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
> > .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
> > .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> > .../calcite/test/Elasticsearch5AdapterIT.java | 270 -------------
> > .../resources/elasticsearch-zips-model.json | 50 ---
> > elasticsearch5/src/test/resources/log4j2.xml | 16 +
> > .../src/test/resources/zips-mini.json | 149 +++++++
> > pom.xml | 20 +-
> > 25 files changed, 1866 insertions(+), 670 deletions(-)
> > ----------------------------------------------------------------------
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > ----------------------------------------------------------------------
> > diff --git
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > index 0980469..8cc5933 100644
> > ---
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > +++
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> > @@ -75,6 +75,18 @@ public abstract class AbstractElasticsearchTable
> extends AbstractQueryableTable
> > relOptTable, this, null);
> > }
> >
> > + /**
> > + * In ES 5.x scripted fields start with {@code params._source.foo}
> while in ES2.x
> > + * {@code _source.foo}. Helper method to build correct query based on
> runtime version of elastic.
> > + *
> > + * @see <a href="
> https://github.com/elastic/elasticsearch/issues/20068">_source
> variable</a>
> > + * @see <a href="
> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html">Scripted
> Fields</a>
> > + */
> > + protected String scriptedFieldPrefix() {
> > + // this is default pattern starting 5.x
> > + return "params._source";
> > + }
> > +
> > /** Executes a "find" operation on the underlying type.
> > *
> > * <p>For example,
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > ----------------------------------------------------------------------
> > diff --git
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > index b42abd7..961c8b0 100644
> > ---
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > +++
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> > @@ -27,11 +27,15 @@ import
> org.apache.calcite.rel.metadata.RelMetadataQuery;
> > import org.apache.calcite.rel.type.RelDataType;
> > import org.apache.calcite.rex.RexNode;
> > import org.apache.calcite.util.Pair;
> > -import org.apache.calcite.util.Util;
> > +
> > +import com.google.common.base.Function;
> > +import com.google.common.collect.Lists;
> >
> > import java.util.ArrayList;
> > import java.util.List;
> >
> > +import javax.annotation.Nullable;
> > +
> > /**
> > * Implementation of {@link org.apache.calcite.rel.core.Project}
> > * relational expression in Elasticsearch.
> > @@ -57,41 +61,60 @@ public class ElasticsearchProject extends Project
> implements ElasticsearchRel {
> > implementor.visitChild(0, getInput());
> >
> > final List<String> inFields =
> > -
> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> > +
> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> > final ElasticsearchRules.RexToElasticsearchTranslator translator =
> > - new ElasticsearchRules.RexToElasticsearchTranslator(
> > - (JavaTypeFactory) getCluster().getTypeFactory(), inFields);
> > + new ElasticsearchRules.RexToElasticsearchTranslator(
> > + (JavaTypeFactory) getCluster().getTypeFactory(),
> inFields);
> >
> > - final List<String> findItems = new ArrayList<>();
> > - final List<String> scriptFieldItems = new ArrayList<>();
> > + final List<String> fields = new ArrayList<>();
> > + final List<String> scriptFields = new ArrayList<>();
> > for (Pair<RexNode, String> pair: getNamedProjects()) {
> > final String name = pair.right;
> > final String expr = pair.left.accept(translator);
> >
> > if (expr.equals("\"" + name + "\"")) {
> > - findItems.add(ElasticsearchRules.quote(name));
> > + fields.add(name);
> > } else if (expr.matches("\"literal\":.+")) {
> > - scriptFieldItems.add(ElasticsearchRules.quote(name)
> > - + ":{\"script\": "
> > - + expr.split(":")[1] + "}");
> > + scriptFields.add(ElasticsearchRules.quote(name)
> > + + ":{\"script\": "
> > + + expr.split(":")[1] + "}");
> > } else {
> > - scriptFieldItems.add(ElasticsearchRules.quote(name)
> > - + ":{\"script\":\"params._source."
> > - + expr.replaceAll("\"", "") + "\"}");
> > + scriptFields.add(ElasticsearchRules.quote(name)
> > + + ":{\"script\":"
> > + // _source (ES2) vs params._source (ES5)
> > + + "\"" +
> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> > + + expr.replaceAll("\"", "") + "\"}");
> > + }
> > + }
> > +
> > + StringBuilder query = new StringBuilder();
> > + if (scriptFields.isEmpty()) {
> > + List<String> newList = Lists.transform(fields, new
> Function<String, String>() {
> > + @Nullable
> > + @Override public String apply(@Nullable String input) {
> > + return ElasticsearchRules.quote(input);
> > + }
> > + });
> > +
> > + final String findString = String.join(", ", newList);
> > + query.append("\"_source\" : [").append(findString).append("]");
> > + } else {
> > + // if scripted fields are present, ES ignores _source attribute
> > + for (String field: fields) {
> > + scriptFields.add(ElasticsearchRules.quote(field) +
> ":{\"script\": "
> > + // _source (ES2) vs params._source (ES5)
> > + + "\"" +
> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> > + + field + "\"}");
> > }
> > + query.append("\"script_fields\": {" + String.join(", ",
> scriptFields) + "}");
> > }
> > - final String findString = Util.toString(findItems, "", ", ", "");
> > - final String scriptFieldString = "\"script_fields\": {"
> > - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
> > - final String fieldString = "\"_source\" : [" + findString + "]"
> > - + ", " + scriptFieldString;
> >
> > for (String opfield : implementor.list) {
> > if (opfield.startsWith("\"_source\"")) {
> > implementor.list.remove(opfield);
> > }
> > }
> > - implementor.add(fieldString);
> > + implementor.add(query.toString());
> > }
> > }
> >
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
> > ----------------------------------------------------------------------
> > diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
> > index f24622c..6fbee03 100644
> > --- a/elasticsearch2/pom.xml
> > +++ b/elasticsearch2/pom.xml
> > @@ -73,6 +73,12 @@ limitations under the License.
> > <version>${elasticsearch-java-driver.version}</version>
> > </dependency>
> > <dependency>
> > + <!-- Lang groovy dependency is needed for testing with embedded
> ES (scripted fields like loc[0]) -->
> > + <groupId>org.elasticsearch.module</groupId>
> > + <artifactId>lang-groovy</artifactId>
> > + <scope>test</scope>
> > + </dependency>
> > + <dependency>
> > <groupId>com.carrotsearch</groupId>
> > <artifactId>hppc</artifactId>
> > <version>${hppc.version}</version>
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > ----------------------------------------------------------------------
> > diff --git
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > index 84370ab..c3d2ac0 100644
> > ---
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > +++
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> > @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
> > import java.util.Date;
> > import java.util.Iterator;
> > import java.util.List;
> > +import java.util.Locale;
> > import java.util.Map;
> >
> > /**
> > @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator implements
> Enumerator<Object> {
> > private static Function1<SearchHit, Object[]> listGetter(
> > final List<Map.Entry<String, Class>> fields) {
> > return new Function1<SearchHit, Object[]>() {
> > - public Object[] apply(SearchHit searchHitFields) {
> > + public Object[] apply(SearchHit hit) {
> > Object[] objects = new Object[fields.size()];
> > for (int i = 0; i < fields.size(); i++) {
> > final Map.Entry<String, Class> field = fields.get(i);
> > final String name = field.getKey();
> > - if (searchHitFields.fields().isEmpty()) {
> > - objects[i] = convert(searchHitFields.getSource().get(name),
> field.getValue());
> > + if (hit.fields().isEmpty()) {
> > + objects[i] = convert(hit.getSource().get(name),
> field.getValue());
> > + } else if (hit.fields().containsKey(name)) {
> > + objects[i] = convert(hit.field(name).getValue(),
> field.getValue());
> > } else {
> > - objects[i] =
> convert(searchHitFields.field(name).getValue(), field.getValue());
> > + throw new IllegalStateException(
> > + String.format(Locale.getDefault(), "No result for
> %s", field));
> > }
> > }
> > return objects;
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > ----------------------------------------------------------------------
> > diff --git
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > index 668402b..46e3fc5 100644
> > ---
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > +++
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> > @@ -22,6 +22,9 @@ import org.apache.calcite.schema.impl.AbstractSchema;
> >
> > import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
> >
> > +import com.google.common.annotations.VisibleForTesting;
> > +import com.google.common.base.Preconditions;
> > +
> > import com.google.common.collect.ImmutableList;
> > import com.google.common.collect.ImmutableMap;
> >
> > @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
> AbstractSchema
> > }
> > }
> >
> > + /**
> > + * Allows schema to be instantiated from existing elastic search
> client.
> > + * This constructor is used in tests.
> > + */
> > + @VisibleForTesting
> > + Elasticsearch2Schema(Client client, String index) {
> > + this.client = Preconditions.checkNotNull(client, "client");
> > + this.index = Preconditions.checkNotNull(index, "index");
> > + }
> > +
> > @Override protected Map<String, Table> getTableMap() {
> > final ImmutableMap.Builder<String, Table> builder =
> ImmutableMap.builder();
> >
> > @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
> AbstractSchema
> >
> > final List<DiscoveryNode> nodes =
> ImmutableList.copyOf(transportClient.connectedNodes());
> > if (nodes.isEmpty()) {
> > - throw new RuntimeException("Cannot connect to any elasticsearch
> nodes");
> > + throw new IllegalStateException("Cannot connect to any
> elasticsearch node: "
> > + + transportNodes);
> > }
> >
> > client = transportClient;
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > ----------------------------------------------------------------------
> > diff --git
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > index 636aa5f..2928835 100644
> > ---
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > +++
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> > @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
> AbstractElasticsearchTable {
> > this.client = client;
> > }
> >
> > + /**
> > + * ES version 2.x. To access document attributes ES2 uses {@code
> _source.foo} syntax.
> > + */
> > + @Override protected String scriptedFieldPrefix() {
> > + return "_source";
> > + }
> > +
> > @Override protected Enumerable<Object> find(String index, List<String>
> ops,
> > - List<Map.Entry<String, Class>> fields) {
> > + List<Map.Entry<String,
> Class>> fields) {
> > final String dbName = index;
> >
> > final String queryString = "{" + Util.toString(ops, "", ", ", "") +
> "}";
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > ----------------------------------------------------------------------
> > diff --git
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > new file mode 100644
> > index 0000000..287e094
> > --- /dev/null
> > +++
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> > @@ -0,0 +1,395 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements. See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to you under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License. You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.calcite.adapter.elasticsearch2;
> > +
> > +import org.apache.calcite.jdbc.CalciteConnection;
> > +import org.apache.calcite.schema.SchemaPlus;
> > +import org.apache.calcite.schema.impl.ViewTable;
> > +import org.apache.calcite.schema.impl.ViewTableMacro;
> > +import org.apache.calcite.test.CalciteAssert;
> > +import org.apache.calcite.test.ElasticChecker;
> > +
> > +import com.google.common.io.LineProcessor;
> > +import com.google.common.io.Resources;
> > +
> > +import org.elasticsearch.action.bulk.BulkItemResponse;
> > +import org.elasticsearch.action.bulk.BulkRequestBuilder;
> > +import org.elasticsearch.action.bulk.BulkResponse;
> > +import org.elasticsearch.common.xcontent.XContentBuilder;
> > +import org.elasticsearch.common.xcontent.XContentFactory;
> > +
> > +import org.junit.BeforeClass;
> > +import org.junit.ClassRule;
> > +import org.junit.Test;
> > +
> > +import java.io.IOException;
> > +import java.nio.charset.StandardCharsets;
> > +import java.sql.Connection;
> > +import java.sql.DriverManager;
> > +import java.sql.SQLException;
> > +import java.util.Arrays;
> > +import java.util.Collections;
> > +import java.util.Locale;
> > +
> > +/**
> > + * Set of tests for ES adapter. Uses real instance via {@link
> EmbeddedElasticRule}. Document
> > + * source is local {@code zips-mini.json} file (located in the
> classpath).
> > + */
> > +public class ElasticSearch2AdapterTest {
> > +
> > + @ClassRule //init once for all tests
> > + public static final EmbeddedElasticRule NODE =
> EmbeddedElasticRule.create();
> > +
> > + private static final String ZIPS = "zips";
> > +
> > + /**
> > + * Used to create {@code zips} index and insert some data
> > + */
> > + @BeforeClass
> > + public static void setupInstance() throws Exception {
> > + // define mapping so fields are searchable (term query)
> > + XContentBuilder mapping =
> XContentFactory.jsonBuilder().startObject()
> > + .startObject("properties")
> > + .startObject("city").field("type", "string")
> > + .field("index", "not_analyzed").endObject()
> > + .startObject("state").field("type", "string")
> > + .field("index", "not_analyzed").endObject()
> > + .startObject("pop").field("type", "long").endObject()
> > + .endObject()
> > + .endObject();
> > +
> > + // create index
> > + NODE.client().admin().indices()
> > + .prepareCreate(ZIPS)
> > + .addMapping(ZIPS, mapping)
> > + .get();
> > +
> > + BulkRequestBuilder bulk =
> NODE.client().prepareBulk().setRefresh(true);
> > +
> > + // load records from file
> > +
> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
> > + StandardCharsets.UTF_8, new LineProcessor<Void>() {
> > + @Override public boolean processLine(String line) throws
> IOException {
> > + line = line.replaceAll("_id", "id"); // _id is a
> reserved attribute in ES
> > + bulk.add(NODE.client().prepareIndex(ZIPS,
> ZIPS).setSource(line));
> > + return true;
> > + }
> > +
> > + @Override public Void getResult() {
> > + return null;
> > + }
> > + });
> > +
> > + if (bulk.numberOfActions() == 0) {
> > + throw new IllegalStateException("No records to be indexed");
> > + }
> > +
> > + BulkResponse response = bulk.execute().get();
> > +
> > + if (response.hasFailures()) {
> > + throw new IllegalStateException(
> > + String.format(Locale.getDefault(), "Failed to populate
> %s:\n%s", NODE.httpAddress(),
> > +
> Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
> > +
> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
> > + }
> > +
> > + }
> > +
> > + private CalciteAssert.ConnectionFactory newConnectionFactory() {
> > + return new CalciteAssert.ConnectionFactory() {
> > + @Override public Connection createConnection() throws
> SQLException {
> > + final Connection connection =
> DriverManager.getConnection("jdbc:calcite:");
> > + final SchemaPlus root =
> connection.unwrap(CalciteConnection.class).getRootSchema();
> > +
> > + root.add("elastic", new Elasticsearch2Schema(NODE.client(),
> ZIPS));
> > +
> > + // add calcite view programmatically
> > + final String viewSql = "select cast(_MAP['city'] AS
> varchar(20)) AS \"city\", "
> > + + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n"
> > + + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n"
> > + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
> > + + " cast(_MAP['state'] AS varchar(2)) AS \"state\", "
> > + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
> > + + "from \"elastic\".\"zips\"";
> > +
> > + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
> > + Collections.singletonList("elastic"),
> Arrays.asList("elastic", "view"), false);
> > + root.add("ZIPS", macro);
> > +
> > + return connection;
> > + }
> > + };
> > + }
> > +
> > + private CalciteAssert.AssertThat calciteAssert() {
> > + return CalciteAssert.that()
> > + .with(newConnectionFactory());
> > + }
> > +
> > + /**
> > + * Tests using calcite view
> > + */
> > + @Test
> > + public void view() throws Exception {
> > + calciteAssert()
> > + .query("select * from zips where \"city\" = 'BROOKLYN'")
> > + .returns("city=BROOKLYN; longitude=-73.956985;
> latitude=40.646694; "
> > + + "pop=111396; state=NY; id=11226\n")
> > + .returnsCount(1);
> > + }
> > +
> > + @Test
> > + public void emptyResult() {
> > + CalciteAssert.that()
> > + .with(newConnectionFactory())
> > + .query("select * from zips limit 0")
> > + .returnsCount(0);
> > +
> > + CalciteAssert.that()
> > + .with(newConnectionFactory())
> > + .query("select * from \"elastic\".\"zips\" where
> _MAP['Foo'] = '_MISSING_'")
> > + .returnsCount(0);
> > + }
> > +
> > + @Test
> > + public void basic() throws Exception {
> > + CalciteAssert.that()
> > + .with(newConnectionFactory())
> > + .query("select * from \"elastic\".\"zips\" where
> _MAP['city'] = 'BROOKLYN'")
> > + .returnsCount(1);
> > +
> > + CalciteAssert.that()
> > + .with(newConnectionFactory())
> > + .query("select * from \"elastic\".\"zips\" where"
> > + + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')")
> > + .returnsCount(2);
> > +
> > + // lower-case
> > + CalciteAssert.that()
> > + .with(newConnectionFactory())
> > + .query("select * from \"elastic\".\"zips\" where "
> > + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
> 'BROOK') ")
> > + .returnsCount(0);
> > +
> > + // missing field
> > + CalciteAssert.that()
> > + .with(newConnectionFactory())
> > + .query("select * from \"elastic\".\"zips\" where
> _MAP['CITY'] = 'BROOKLYN'")
> > + .returnsCount(0);
> > +
> > + // limit works
> > + CalciteAssert.that()
> > + .with(newConnectionFactory())
> > + .query("select * from \"elastic\".\"zips\" limit 42")
> > + .returnsCount(42);
> > +
> > + }
> > +
> > + @Test public void testSort() {
> > + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> > + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> > + + " ElasticsearchProject(city=[CAST(ITEM($0,
> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> > + + " ElasticsearchTableScan(table=[[elastic, zips]])";
> > +
> > + calciteAssert()
> > + .query("select * from zips order by \"state\"")
> > + .returnsCount(10)
> > + .explainContains(explain);
> > + }
> > +
> > + @Test public void testSortLimit() {
> > + final String sql = "select \"state\", \"id\" from zips\n"
> > + + "order by \"state\", \"id\" offset 2 rows fetch next 3
> rows only";
> > + calciteAssert()
> > + .query(sql)
> > + .returnsUnordered("state=AK; id=99801",
> > + "state=AL; id=35215",
> > + "state=AL; id=35401")
> > + .queryContains(
> > + ElasticChecker.elasticsearchChecker(
> > + "\"_source\" : [\"state\", \"id\"]",
> > + "\"sort\": [ {\"state\": \"asc\"}, {\"id\":
> \"asc\"}]",
> > + "\"from\": 2",
> > + "\"size\": 3"));
> > + }
> > +
> > +
> > +
> > + @Test public void testOffsetLimit() {
> > + final String sql = "select \"state\", \"id\" from zips\n"
> > + + "offset 2 fetch next 3 rows only";
> > + calciteAssert()
> > + .query(sql)
> > + .runs()
> > + .queryContains(
> > + ElasticChecker.elasticsearchChecker(
> > + "\"from\": 2",
> > + "\"size\": 3",
> > + "\"_source\" : [\"state\", \"id\"]"));
> > + }
> > +
> > + @Test public void testLimit() {
> > + final String sql = "select \"state\", \"id\" from zips\n"
> > + + "fetch next 3 rows only";
> > +
> > + calciteAssert()
> > + .query(sql)
> > + .runs()
> > + .queryContains(
> > + ElasticChecker.elasticsearchChecker(
> > + "\"size\": 3",
> > + "\"_source\" : [\"state\", \"id\"]"));
> > + }
> > +
> > + @Test public void testFilterSort() {
> > + final String sql = "select * from zips\n"
> > + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
> > + + "order by \"state\", \"id\"";
> > + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> > + + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC],
> dir1=[ASC])\n"
> > + + " ElasticsearchProject(city=[CAST(ITEM($0,
> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> > + + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0, 'id')):VARCHAR(5)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\",
> '70000'))])\n"
> > + + " ElasticsearchTableScan(table=[[elastic, zips]])";
> > + calciteAssert()
> > + .query(sql)
> > + .returnsOrdered("city=LOS ANGELES; longitude=-118.258189;
> latitude=34.007856; "
> > + + "pop=96074; state=CA; id=90011",
> > + "city=BELL GARDENS; longitude=-118.17205;
> latitude=33.969177; "
> > + + "pop=99568; state=CA; id=90201",
> > + "city=NORWALK; longitude=-118.081767;
> latitude=33.90564; "
> > + + "pop=94188; state=CA; id=90650")
> > + .queryContains(
> > + ElasticChecker.elasticsearchChecker("\"query\" : "
> > + +
> "{\"constant_score\":{\"filter\":{\"bool\":"
> > + +
> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
> > + +
> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
> > + "\"script_fields\":
> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> > + +
> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> > + + "\"city\":{\"script\":
> \"_source.city\"}, "
> > + + "\"pop\":{\"script\":
> \"_source.pop\"}, "
> > + + "\"state\":{\"script\":
> \"_source.state\"}, "
> > + + "\"id\":{\"script\": \"_source.id
> \"}}",
> > + "\"sort\": [ {\"state\": \"asc\"}, {\"id\":
> \"asc\"}]"))
> > + .explainContains(explain);
> > + }
> > +
> > + @Test public void testFilterSortDesc() {
> > + final String sql = "select * from zips\n"
> > + + "where \"pop\" BETWEEN 95000 AND 100000\n"
> > + + "order by \"state\" desc, \"pop\"";
> > + calciteAssert()
> > + .query(sql)
> > + .limit(4)
> > + .returnsOrdered(
> > + "city=LOS ANGELES; longitude=-118.258189;
> latitude=34.007856; pop=96074; state=CA; id=90011",
> > + "city=BELL GARDENS; longitude=-118.17205;
> latitude=33.969177; pop=99568; state=CA; id=90201");
> > + }
> > +
> > + @Test public void testFilterRedundant() {
> > + final String sql = "select * from zips\n"
> > + + "where \"state\" > 'CA' and \"state\" < 'AZ' and
> \"state\" = 'OK'";
> > + calciteAssert()
> > + .query(sql)
> > + .runs()
> > + .queryContains(
> > + ElasticChecker.elasticsearchChecker(""
> > + + "\"query\" :
> {\"constant_score\":{\"filter\":{\"bool\":"
> > + +
> "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
> > + "\"script_fields\":
> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> > + +
> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> > + + "\"city\":{\"script\":
> \"_source.city\"}, "
> > + + "\"pop\":{\"script\":
> \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
> > + + "\"id\":{\"script\":
> \"_source.id\"}}"
> > + ));
> > + }
> > +
> > + @Test public void testInPlan() {
> > + final String[] searches = {
> > + "\"query\" :
> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
> > + +
> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
> > + + "{\"pop\":99568}}]}}]}}}}",
> > + "\"script_fields\":
> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> > + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> > + + "\"city\":{\"script\": \"_source.city\"}, "
> > + + "\"pop\":{\"script\": \"_source.pop\"}, "
> > + + "\"state\":{\"script\": \"_source.state\"}, "
> > + + "\"id\":{\"script\": \"_source.id\"}}"
> > + };
> > +
> > + calciteAssert()
> > + .query("select * from zips where \"pop\" in (96074, 99568)")
> > + .returnsUnordered(
> > + "city=BELL GARDENS; longitude=-118.17205;
> latitude=33.969177; pop=99568; state=CA; id=90201",
> > + "city=LOS ANGELES; longitude=-118.258189;
> latitude=34.007856; pop=96074; state=CA; id=90011"
> > + )
> > +
> .queryContains(ElasticChecker.elasticsearchChecker(searches));
> > + }
> > +
> > + @Test public void testZips() {
> > + calciteAssert()
> > + .query("select \"state\", \"city\" from zips")
> > + .returnsCount(10);
> > + }
> > +
> > + @Test public void testProject() {
> > + final String sql = "select \"state\", \"city\", 0 as \"zero\"\n"
> > + + "from zips\n"
> > + + "order by \"state\", \"city\"";
> > +
> > + calciteAssert()
> > + .query(sql)
> > + .limit(2)
> > + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0",
> > + "state=AK; city=FAIRBANKS; zero=0")
> > + .queryContains(
> > +
> ElasticChecker.elasticsearchChecker("\"script_fields\": "
> > + + "{\"zero\":{\"script\": \"0\"}, "
> > + + "\"state\":{\"script\":
> \"_source.state\"}, "
> > + + "\"city\":{\"script\":
> \"_source.city\"}}",
> > + "\"sort\": [ {\"state\": \"asc\"},
> {\"city\": \"asc\"}]"));
> > + }
> > +
> > + @Test public void testFilter() {
> > + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> > + + " ElasticsearchProject(state=[CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> > + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\", 'CA')])\n"
> > + + " ElasticsearchTableScan(table=[[elastic, zips]])";
> > + calciteAssert()
> > + .query("select \"state\", \"city\" from zips where
> \"state\" = 'CA'")
> > + .limit(3)
> > + .returnsUnordered("state=CA; city=BELL GARDENS",
> > + "state=CA; city=LOS ANGELES",
> > + "state=CA; city=NORWALK")
> > + .explainContains(explain);
> > + }
> > +
> > + @Test public void testFilterReversed() {
> > + calciteAssert()
> > + .query("select \"state\", \"city\" from zips where 'WI' <
> \"state\" order by \"city\"")
> > + .limit(2)
> > + .returnsUnordered("state=WV; city=BECKLEY",
> > + "state=WY; city=CHEYENNE");
> > + calciteAssert()
> > + .query("select \"state\", \"city\" from zips where \"state\"
> > 'WI' order by \"city\"")
> > + .limit(2)
> > + .returnsUnordered("state=WV; city=BECKLEY",
> > + "state=WY; city=CHEYENNE");
> > + }
> > +
> > +}
> > +
> > +// End ElasticSearch2AdapterTest.java
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> > ----------------------------------------------------------------------
> > diff --git
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> > new file mode 100644
> > index 0000000..4474add
> > --- /dev/null
> > +++
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> > @@ -0,0 +1,147 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements. See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to you under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License. You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.calcite.adapter.elasticsearch2;
> > +
> > +import com.google.common.base.Preconditions;
> > +import com.google.common.io.Files;
> > +
> > +import org.elasticsearch.Version;
> > +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
> > +import org.elasticsearch.action.admin.cluster.node.info
> .NodesInfoResponse;
> > +import org.elasticsearch.client.Client;
> > +import org.elasticsearch.common.settings.Settings;
> > +import org.elasticsearch.common.transport.TransportAddress;
> > +import org.elasticsearch.node.Node;
> > +import org.elasticsearch.node.internal.InternalSettingsPreparer;
> > +import org.elasticsearch.plugins.Plugin;
> > +import org.elasticsearch.script.groovy.GroovyPlugin;
> > +
> > +import java.io.File;
> > +import java.util.Arrays;
> > +import java.util.Collection;
> > +import java.util.Collections;
> > +
> > +/**
> > + * Represents a single elastic search node which can run embedded in a
> java application.
> > + * Intended for unit and integration tests. Settings and plugins are
> crafted for Calcite.
> > + */
> > +class EmbeddedElasticNode implements AutoCloseable {
> > +
> > + private final LocalNode node;
> > + private volatile boolean isStarted;
> > +
> > + private EmbeddedElasticNode(LocalNode node) {
> > + this.node = Preconditions.checkNotNull(node, "node");
> > + }
> > +
> > + /**
> > + * Having separate class to expose (protected) constructor which
> allows to install
> > + * different plugins. In our case it is {@code GroovyPlugin} for
> scripted fields like
> > + * {@code loc[0]} or {@code loc[1]['foo']}.
> > + */
> > + private static class LocalNode extends Node {
> > + private LocalNode(Settings settings, Collection<Class<? extends
> Plugin>> classpathPlugins) {
> > + super(InternalSettingsPreparer.prepareEnvironment(settings, null),
> > + Version.CURRENT,
> > + classpathPlugins);
> > + }
> > + }
> > +
> > + /**
> > + * Creates an instance with existing settings
> > + */
> > + private static EmbeddedElasticNode create(Settings settings) {
> > + // ensure GroovyPlugin is installed or otherwise scripted fields
> would not work
> > + LocalNode node = new LocalNode(settings,
> Collections.singleton(GroovyPlugin.class));
> > + return new EmbeddedElasticNode(node);
> > + }
> > +
> > + /**
> > + * Creates elastic node as single member of a cluster. Node will not
> be started
> > + * unless {@link #start()} is explicitly called.
> > + */
> > + public static EmbeddedElasticNode create() {
> > + File data = Files.createTempDir();
> > + data.deleteOnExit();
> > + File home = Files.createTempDir();
> > + home.deleteOnExit();
> > +
> > + Settings settings = Settings.builder()
> > + .put("node.name", "fake-elastic")
> > + .put("path.home", home.getAbsolutePath())
> > + .put("path.data", data.getAbsolutePath())
> > + .put("script.inline", true) // requires GroovyPlugin
> > + .put("script.indexed", true) // requires GroovyPlugin
> > + .put("cluster.routing.allocation.disk.threshold_enabled",
> false)
> > + .put("node.local", true)
> > + .put("node.data", true)
> > + .put("network.host", "localhost")
> > + .build();
> > +
> > + return create(settings);
> > + }
> > +
> > + /**
> > + * Starts current node
> > + */
> > + public void start() {
> > + Preconditions.checkState(!isStarted, "already started");
> > + node.start();
> > + this.isStarted = true;
> > + }
> > +
> > + /**
> > + * Returns current address to connect to with HTTP client.
> > + */
> > + public TransportAddress httpAddress() {
> > + Preconditions.checkState(isStarted, "node is not started");
> > +
> > + NodesInfoResponse response =
> client().admin().cluster().prepareNodesInfo()
> > + .execute().actionGet();
> > + if (response.getNodes().length != 1) {
> > + throw new IllegalStateException("Expected single node but got "
> > + + response.getNodes().length);
> > + }
> > + NodeInfo node = response.getNodes()[0];
> > + return node.getHttp().address().boundAddresses()[0];
> > + }
> > +
> > + /**
> > + * Exposes elastic
> > + * <a href="
> https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html">transport
> client</a>
> > + *
> > + * (use of HTTP client is preferred).
> > + */
> > + public Client client() {
> > + Preconditions.checkState(isStarted, "node is not started");
> > + return node.client();
> > + }
> > +
> > + @Override public void close() throws Exception {
> > + node.close();
> > + // cleanup data dirs
> > + File data = new File(node.settings().get("path.data"));
> > + File home = new File(node.settings().get("path.home"));
> > + for (File file: Arrays.asList(data, home)) {
> > + if (file.exists()) {
> > + file.delete();
> > + }
> > + }
> > + }
> > +}
> > +
> > +// End EmbeddedElasticNode.java
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> > ----------------------------------------------------------------------
> > diff --git
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> > new file mode 100644
> > index 0000000..a633078
> > --- /dev/null
> > +++
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> > @@ -0,0 +1,97 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements. See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to you under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License. You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.calcite.adapter.elasticsearch2;
> > +
> > +import com.google.common.base.Preconditions;
> > +
> > +import org.elasticsearch.client.Client;
> > +import org.elasticsearch.common.transport.TransportAddress;
> > +import org.junit.rules.ExternalResource;
> > +
> > +/**
> > + * Used to initialize a single elastic node. For performance reasons
> (node startup costs),
> > + * same instance is usually shared across multiple tests.
> > + *
> > + * This rule should be used as follows:
> > + * <pre>
> > + * {@code
> > + *
> > + * public class MyTest {
> > + * @literal @ClassRule
> > + * public static final ElasticSearchRule RULE =
> ElasticSearchRule.create();
> > + *
> > + * @literal @BeforeClass
> > + * public void setup() {
> > + * // ... populate instance
> > + * }
> > + *
> > + * @literal @Test
> > + * public void myTest() {
> > + * TransportAddress address = RULE.httpAddress();
> > + * // ....
> > + * }
> > + * }
> > + * }
> > + * </pre>
> > + *
> > + * @see ExternalResource
> > + */
> > +class EmbeddedElasticRule extends ExternalResource {
> > +
> > + private final EmbeddedElasticNode node;
> > +
> > + private EmbeddedElasticRule(EmbeddedElasticNode resource) {
> > + this.node = Preconditions.checkNotNull(resource, "resource");
> > + }
> > +
> > + @Override protected void before() throws Throwable {
> > + node.start();
> > + }
> > +
> > + @Override protected void after() {
> > + try {
> > + node.close();
> > + } catch (Exception e) {
> > + throw new RuntimeException(e);
> > + }
> > + }
> > +
> > + /**
> > + * Factory method to create this rule.
> > + */
> > + public static EmbeddedElasticRule create() {
> > + return new EmbeddedElasticRule(EmbeddedElasticNode.create());
> > + }
> > +
> > + /**
> > + * Exposes current ES transport client.
> > + */
> > + Client client() {
> > + return node.client();
> > + }
> > +
> > + /**
> > + * HTTP address for rest clients (can be ES native or any other).
> > + */
> > + TransportAddress httpAddress() {
> > + return node.httpAddress();
> > + }
> > +
> > +
> > +}
> > +
> > +// End EmbeddedElasticRule.java
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> > ----------------------------------------------------------------------
> > diff --git
> a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> > new file mode 100644
> > index 0000000..21fc491
> > --- /dev/null
> > +++
> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> > @@ -0,0 +1,49 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements. See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to you under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License. You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.calcite.test;
> > +
> > +import com.google.common.base.Function;
> > +
> > +import java.util.List;
> > +
> > +import javax.annotation.Nullable;
> > +
> > +/**
> > + * Internal util methods for ElasticSearch tests
> > + */
> > +public class ElasticChecker {
> > +
> > + private ElasticChecker() {}
> > +
> > +
> > + /** Returns a function that checks that a particular Elasticsearch
> pipeline is
> > + * generated to implement a query. */
> > + public static Function<List, Void> elasticsearchChecker(final
> String... strings) {
> > + return new Function<List, Void>() {
> > + @Nullable
> > + @Override public Void apply(@Nullable List actual) {
> > + Object[] actualArray = actual == null || actual.isEmpty() ? null
> > + : ((List) actual.get(0)).toArray();
> > + CalciteAssert.assertArrayEqual("expected Elasticsearch query
> not found", strings,
> > + actualArray);
> > + return null;
> > + }
> > + };
> > + }
> > +}
> > +
> > +// End ElasticChecker.java
> >
> >
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> > ----------------------------------------------------------------------
> > diff --git
> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> b/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> > deleted file mode 100644
> > index 4e0c2b6..0000000
> > ---
> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> > +++ /dev/null
> > @@ -1,270 +0,0 @@
> > -/*
> > - * Licensed to the Apache Software Foundation (ASF) under one or more
> > - * contributor license agreements. See the NOTICE file distributed with
> > - * this work for additional information regarding copyright ownership.
> > - * The ASF licenses this file to you under the Apache License, Version
> 2.0
> > - * (the "License"); you may not use this file except in compliance with
> > - * the License. You may obtain a copy of the License at
> > - *
> > - * http://www.apache.org/licenses/LICENSE-2.0
> > - *
> > - * Unless required by applicable law or agreed to in writing, software
> > - * distributed under the License is distributed on an "AS IS" BASIS,
> > - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > - * See the License for the specific language governing permissions and
> > - * limitations under the License.
> > - */
> > -package org.apache.calcite.test;
> > -
> > -import org.apache.calcite.util.Util;
> > -
> > -import com.google.common.base.Function;
> > -import com.google.common.collect.ImmutableMap;
> > -
> > -import org.junit.Test;
> > -
> > -import java.util.List;
> > -import javax.annotation.Nullable;
> > -
> > -/**
> > - * Tests for the {@code org.apache.calcite.adapter.elasticsearch2}
> package.
> > - *
> > - * <p>Before calling this test, you need to populate Elasticsearch, as
> follows:
> > - *
> > - * <blockquote><code>
> > - * git clone https://github.com/vlsi/calcite-test-dataset<br>
> > - * cd calcite-test-dataset<br>
> > - * mvn install
> > - * </code></blockquote>
> > - *
> > - * <p>This will create a virtual machine with Elasticsearch and the
> "zips" test
> > - * dataset.
> > - */
> > -public class Elasticsearch2AdapterIT {
> > - /**
> > - * Whether to run Elasticsearch tests. Enabled by default, however
> test is only
> > - * included if "it" profile is activated ({@code -Pit}). To disable,
> > - * specify {@code -Dcalcite.test.elasticsearch=false} on the Java
> command line.
> > - */
> > - private static final boolean ENABLED =
> Util.getBooleanProperty("calcite.test.elasticsearch",
> > - true);
> > -
> > - /** Connection factory based on the "zips-es" model. */
> > - private static final ImmutableMap<String, String> ZIPS =
> ImmutableMap.of("model",
> > -
> Elasticsearch2AdapterIT.class.getResource("/elasticsearch-zips-model.json").getPath());
> > -
> > - /** Whether to run this test. */
> > - private boolean enabled() {
> > - return ENABLED;
> > - }
> > -
> > - /** Returns a function that checks that a particular Elasticsearch
> pipeline is
> > - * generated to implement a query. */
> > - private static Function<List, Void> elasticsearchChecker(final
> String... strings) {
> > - return new Function<List, Void>() {
> > - @Nullable
> > - @Override public Void apply(@Nullable List actual) {
> > - Object[] actualArray = actual == null || actual.isEmpty() ? null
> > - : ((List) actual.get(0)).toArray();
> > - CalciteAssert.assertArrayEqual("expected Elasticsearch query
> not found", strings,
> > - actualArray);
> > - return null;
> > - }
> > - };
> > - }
> > -
> > - @Test public void testSort() {
> > - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> > - + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> > - + " ElasticsearchProject(city=[CAST(ITEM($0,
> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> > - + " ElasticsearchTableScan(table=[[elasticsearch_raw,
> zips]])";
> > - CalciteAssert.that()
> > - .enable(enabled())
> > - .with(ZIPS)
> > - .query("select * from zips order by \"state\"")
> > - .returnsCount(10)
> > - .explainContains(explain);
> > - }
> > -
> > - @Test public void testSortLimit() {
> > - final String sql = "select \"state\", \"id\" from zips\n"
> > - + "order by \"state\", \"id\" offset 2 rows fetch next 3 rows
> only";
> > - CalciteAssert.that()
> > - .with(ZIPS)
> > - .query(sql)
> > - .returnsUnordered("state=AK; id=99503",
> > - "state=AK; id=99504",
> > - "state=AK; id=99505")
> > - .queryContains(
> > - elasticsearchChecker(
> > - "\"fields\" : [\"state\", \"id\"], \"script_fields\":
> {}",
> > - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]",
> > - "\"from\": 2",
> > - "\"size\": 3"));
> > - }
> > -
> > - @Test public void testOffsetLimit() {
> > - final String sql = "select \"state\", \"id\" from zips\n"
> > - + "offset 2 fetch next 3 rows only";
> > - CalciteAssert.that()
> > - .enable(enabled())
> > - .with(ZIPS)
> > - .query(sql)
> > - .runs()
> > - .queryContains(
> > - elasticsearchChecker(
> > - "\"from\": 2",
> > - "\"size\": 3",
> > - "\"fields\" : [\"state\", \"id\"], \"script_fields\":
> {}"));
> > - }
> > -
> > - @Test public void testLimit() {
> > - final String sql = "select \"state\", \"id\" from zips\n"
> > - + "fetch next 3 rows only";
> > - CalciteAssert.that()
> > - .enable(enabled())
> > - .with(ZIPS)
> > - .query(sql)
> > - .runs()
> > - .queryContains(
> > - elasticsearchChecker(
> > - "\"size\": 3",
> > - "\"fields\" : [\"state\", \"id\"], \"script_fields\":
> {}"));
> > - }
> > -
> > - @Test public void testFilterSort() {
> > - final String sql = "select * from zips\n"
> > - + "where \"city\" = 'SPRINGFIELD' and \"id\" >= '70000'\n"
> > - + "order by \"state\", \"id\"";
> > - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> > - + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC],
> dir1=[ASC])\n"
> > - + " ElasticsearchProject(city=[CAST(ITEM($0,
> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> > - + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\", 'SPRINGFIELD'), >=(CAST(ITEM($0,
> 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\", '70000'))])\n"
> > - + " ElasticsearchTableScan(table=[[elasticsearch_raw,
> zips]])";
> > - CalciteAssert.that()
> > - .enable(enabled())
> > - .with(ZIPS)
> > - .query(sql)
> > - .returnsOrdered(
> > - "city=SPRINGFIELD; longitude=-92.54567; latitude=35.274879;
> pop=752; state=AR; id=72157",
> > - "city=SPRINGFIELD; longitude=-102.617322;
> latitude=37.406727; pop=1992; state=CO; id=81073",
> > - "city=SPRINGFIELD; longitude=-90.577479;
> latitude=30.415738; pop=5597; state=LA; id=70462",
> > - "city=SPRINGFIELD; longitude=-123.015259;
> latitude=44.06106; pop=32384; state=OR; id=97477",
> > - "city=SPRINGFIELD; longitude=-122.917108;
> latitude=44.056056; pop=27521; state=OR; id=97478")
> > - .queryContains(
> > - elasticsearchChecker("\"query\" :
> {\"constant_score\":{\"filter\":{\"bool\":"
> > - +
> "{\"must\":[{\"term\":{\"city\":\"springfield\"}},{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
> > - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"],
> \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"},
> \"latitude\":{\"script\":\"_source.loc[1]\"}}",
> > - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]"))
> > - .explainContains(explain);
> > - }
> > -
> > - @Test public void testFilterSortDesc() {
> > - final String sql = "select * from zips\n"
> > - + "where \"pop\" BETWEEN 20000 AND 20100\n"
> > - + "order by \"state\" desc, \"pop\"";
> > - CalciteAssert.that()
> > - .enable(enabled())
> > - .with(ZIPS)
> > - .query(sql)
> > - .limit(4)
> > - .returnsOrdered(
> > - "city=SHERIDAN; longitude=-106.964795; latitude=44.78486;
> pop=20025; state=WY; id=82801",
> > - "city=MOUNTLAKE TERRAC; longitude=-122.304036;
> latitude=47.793061; pop=20059; state=WA; id=98043",
> > - "city=FALMOUTH; longitude=-77.404537; latitude=38.314557;
> pop=20039; state=VA; id=22405",
> > - "city=FORT WORTH; longitude=-97.318409; latitude=32.725551;
> pop=20012; state=TX; id=76104");
> > - }
> > -
> > - @Test public void testFilterRedundant() {
> > - final String sql = "select * from zips\n"
> > - + "where \"state\" > 'CA' and \"state\" < 'AZ' and \"state\" =
> 'OK'";
> > - CalciteAssert.that()
> > - .enable(enabled())
> > - .with(ZIPS)
> > - .query(sql)
> > - .runs()
> > - .queryContains(
> > - elasticsearchChecker(""
> > - + "\"query\" :
> {\"constant_score\":{\"filter\":{\"bool\":"
> > - + "{\"must\":[{\"term\":{\"state\":\"ok\"}}]}}}}",
> > - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"],
> \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"},
> \"latitude\":{\"script\":\"_source.loc[1]\"}}"));
> > - }
> > -
> > - @Test public void testInPlan() {
> > - final String[] searches = {
> > - "\"query\" :
> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
> > - +
> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":20012}}]}},{\"bool\":{\"must\":[{\"term\":"
> > - + "{\"pop\":15590}}]}}]}}}}",
> > - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"],
> \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"},
> \"latitude\":{\"script\":\"_source.loc[1]\"}}"
> > - };
> > - CalciteAssert.that()
> > - .enable(enabled())
> > - .with(ZIPS)
> > - .query("select * from zips where \"pop\" in (20012, 15590)")
> > - .returnsUnordered(
> > - "city=COVINA; longitude=-117.884285; latitude=34.08596;
> pop=15590; state=CA; id=91723",
> > - "city=ARL
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Andrei Sereda <an...@sereda.cc>.
Travis doesn't invoke javadoc
<https://github.com/apache/calcite/blob/master/.travis.yml#L43>:
# Print the Maven version, *skip tests and javadoc*
$DOCKERRUN $IMAGE mvn install -DskipTests=true -Dmaven.javadoc.skip=true
-Djavax.net.ssl.trustStorePassword=changeit -B -V
On Fri, Jun 22, 2018 at 5:14 PM Christian Beikov <ch...@gmail.com>
wrote:
> Huh? How is that possible? I thought the checks that are run by the
> Travis CI build catch these kinds of errors.
>
>
> Mit freundlichen Grüßen,
> ------------------------------------------------------------------------
> *Christian Beikov*
> Am 22.06.2018 um 23:11 schrieb Julian Hyde:
> > Looks like this change broke “mvn site” (perhaps also “mvn
> javadoc:test-javadoc”).
> >
> > [ERROR] Failed to execute goal
> org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on
> project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate
> report:
> > [ERROR] Exit code: 1 -
> /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29:
> error: package org.elasticsearch.node.internal does not exist
> > [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
> > [ERROR] ^
> > [ERROR]
> >
> >> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
> >>
> >> [CALCITE-2347] running ElasticSearch in embedded mode for unit tests of
> ES adapter (Andrei Sereda)
> >>
> >> After discussion on dev-list Integration tests (for ES) have been
> removed. They're now
> >> superseded by unit tests (which execute queries against a real elastic
> instance)
> >>
> >> Added local file (zips-mini.json) which contains a small subset of
> original zips.json
> >> (allows to bootstrap tests faster)
> >>
> >> Created separate ES JUnit rule which can be re-used across different
> tests.
> >>
> >> Both v2 and v5 of ES adapters are supported.
> >>
> >> Close apache/calcite#716
> >>
> >>
> >> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
> >> Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
> >> Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
> >> Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
> >>
> >> Branch: refs/heads/master
> >> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
> >> Parents: 37944bb
> >> Author: Andrei Sereda <an...@nospam.com>
> >> Authored: Thu May 31 18:19:10 2018 -0400
> >> Committer: Michael Mior <mm...@uwaterloo.ca>
> >> Committed: Thu Jun 21 06:38:50 2018 -0400
> >>
> >> ----------------------------------------------------------------------
> >> .../AbstractElasticsearchTable.java | 12 +
> >> .../elasticsearch/ElasticsearchProject.java | 61 ++-
> >> elasticsearch2/pom.xml | 6 +
> >> .../Elasticsearch2Enumerator.java | 12 +-
> >> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
> >> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
> >> .../ElasticSearch2AdapterTest.java | 395 ++++++++++++++++++
> >> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
> >> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
> >> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> >> .../calcite/test/Elasticsearch2AdapterIT.java | 270 -------------
> >> .../resources/elasticsearch-zips-model.json | 50 ---
> >> .../src/test/resources/zips-mini.json | 149 +++++++
> >> elasticsearch5/pom.xml | 31 ++
> >> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
> >> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
> >> .../ElasticSearch5AdapterTest.java | 399
> +++++++++++++++++++
> >> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
> >> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
> >> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
> >> .../calcite/test/Elasticsearch5AdapterIT.java | 270 -------------
> >> .../resources/elasticsearch-zips-model.json | 50 ---
> >> elasticsearch5/src/test/resources/log4j2.xml | 16 +
> >> .../src/test/resources/zips-mini.json | 149 +++++++
> >> pom.xml | 20 +-
> >> 25 files changed, 1866 insertions(+), 670 deletions(-)
> >> ----------------------------------------------------------------------
> >>
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >> ----------------------------------------------------------------------
> >> diff --git
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >> index 0980469..8cc5933 100644
> >> ---
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >> +++
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
> >> @@ -75,6 +75,18 @@ public abstract class AbstractElasticsearchTable
> extends AbstractQueryableTable
> >> relOptTable, this, null);
> >> }
> >>
> >> + /**
> >> + * In ES 5.x scripted fields start with {@code params._source.foo}
> while in ES2.x
> >> + * {@code _source.foo}. Helper method to build correct query based
> on runtime version of elastic.
> >> + *
> >> + * @see <a href="
> https://github.com/elastic/elasticsearch/issues/20068">_source
> variable</a>
> >> + * @see <a href="
> https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html">Scripted
> Fields</a>
> >> + */
> >> + protected String scriptedFieldPrefix() {
> >> + // this is default pattern starting 5.x
> >> + return "params._source";
> >> + }
> >> +
> >> /** Executes a "find" operation on the underlying type.
> >> *
> >> * <p>For example,
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >> ----------------------------------------------------------------------
> >> diff --git
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >> index b42abd7..961c8b0 100644
> >> ---
> a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >> +++
> b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
> >> @@ -27,11 +27,15 @@ import
> org.apache.calcite.rel.metadata.RelMetadataQuery;
> >> import org.apache.calcite.rel.type.RelDataType;
> >> import org.apache.calcite.rex.RexNode;
> >> import org.apache.calcite.util.Pair;
> >> -import org.apache.calcite.util.Util;
> >> +
> >> +import com.google.common.base.Function;
> >> +import com.google.common.collect.Lists;
> >>
> >> import java.util.ArrayList;
> >> import java.util.List;
> >>
> >> +import javax.annotation.Nullable;
> >> +
> >> /**
> >> * Implementation of {@link org.apache.calcite.rel.core.Project}
> >> * relational expression in Elasticsearch.
> >> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends Project
> implements ElasticsearchRel {
> >> implementor.visitChild(0, getInput());
> >>
> >> final List<String> inFields =
> >> -
> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> >> +
> ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
> >> final ElasticsearchRules.RexToElasticsearchTranslator translator =
> >> - new ElasticsearchRules.RexToElasticsearchTranslator(
> >> - (JavaTypeFactory) getCluster().getTypeFactory(), inFields);
> >> + new ElasticsearchRules.RexToElasticsearchTranslator(
> >> + (JavaTypeFactory) getCluster().getTypeFactory(),
> inFields);
> >>
> >> - final List<String> findItems = new ArrayList<>();
> >> - final List<String> scriptFieldItems = new ArrayList<>();
> >> + final List<String> fields = new ArrayList<>();
> >> + final List<String> scriptFields = new ArrayList<>();
> >> for (Pair<RexNode, String> pair: getNamedProjects()) {
> >> final String name = pair.right;
> >> final String expr = pair.left.accept(translator);
> >>
> >> if (expr.equals("\"" + name + "\"")) {
> >> - findItems.add(ElasticsearchRules.quote(name));
> >> + fields.add(name);
> >> } else if (expr.matches("\"literal\":.+")) {
> >> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> >> - + ":{\"script\": "
> >> - + expr.split(":")[1] + "}");
> >> + scriptFields.add(ElasticsearchRules.quote(name)
> >> + + ":{\"script\": "
> >> + + expr.split(":")[1] + "}");
> >> } else {
> >> - scriptFieldItems.add(ElasticsearchRules.quote(name)
> >> - + ":{\"script\":\"params._source."
> >> - + expr.replaceAll("\"", "") + "\"}");
> >> + scriptFields.add(ElasticsearchRules.quote(name)
> >> + + ":{\"script\":"
> >> + // _source (ES2) vs params._source (ES5)
> >> + + "\"" +
> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> >> + + expr.replaceAll("\"", "") + "\"}");
> >> + }
> >> + }
> >> +
> >> + StringBuilder query = new StringBuilder();
> >> + if (scriptFields.isEmpty()) {
> >> + List<String> newList = Lists.transform(fields, new
> Function<String, String>() {
> >> + @Nullable
> >> + @Override public String apply(@Nullable String input) {
> >> + return ElasticsearchRules.quote(input);
> >> + }
> >> + });
> >> +
> >> + final String findString = String.join(", ", newList);
> >> + query.append("\"_source\" : [").append(findString).append("]");
> >> + } else {
> >> + // if scripted fields are present, ES ignores _source attribute
> >> + for (String field: fields) {
> >> + scriptFields.add(ElasticsearchRules.quote(field) +
> ":{\"script\": "
> >> + // _source (ES2) vs params._source (ES5)
> >> + + "\"" +
> implementor.elasticsearchTable.scriptedFieldPrefix() + "."
> >> + + field + "\"}");
> >> }
> >> + query.append("\"script_fields\": {" + String.join(", ",
> scriptFields) + "}");
> >> }
> >> - final String findString = Util.toString(findItems, "", ", ", "");
> >> - final String scriptFieldString = "\"script_fields\": {"
> >> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
> >> - final String fieldString = "\"_source\" : [" + findString + "]"
> >> - + ", " + scriptFieldString;
> >>
> >> for (String opfield : implementor.list) {
> >> if (opfield.startsWith("\"_source\"")) {
> >> implementor.list.remove(opfield);
> >> }
> >> }
> >> - implementor.add(fieldString);
> >> + implementor.add(query.toString());
> >> }
> >> }
> >>
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
> >> ----------------------------------------------------------------------
> >> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
> >> index f24622c..6fbee03 100644
> >> --- a/elasticsearch2/pom.xml
> >> +++ b/elasticsearch2/pom.xml
> >> @@ -73,6 +73,12 @@ limitations under the License.
> >> <version>${elasticsearch-java-driver.version}</version>
> >> </dependency>
> >> <dependency>
> >> + <!-- Lang groovy dependency is needed for testing with embedded
> ES (scripted fields like loc[0]) -->
> >> + <groupId>org.elasticsearch.module</groupId>
> >> + <artifactId>lang-groovy</artifactId>
> >> + <scope>test</scope>
> >> + </dependency>
> >> + <dependency>
> >> <groupId>com.carrotsearch</groupId>
> >> <artifactId>hppc</artifactId>
> >> <version>${hppc.version}</version>
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >> ----------------------------------------------------------------------
> >> diff --git
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >> index 84370ab..c3d2ac0 100644
> >> ---
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >> +++
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
> >> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
> >> import java.util.Date;
> >> import java.util.Iterator;
> >> import java.util.List;
> >> +import java.util.Locale;
> >> import java.util.Map;
> >>
> >> /**
> >> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator implements
> Enumerator<Object> {
> >> private static Function1<SearchHit, Object[]> listGetter(
> >> final List<Map.Entry<String, Class>> fields) {
> >> return new Function1<SearchHit, Object[]>() {
> >> - public Object[] apply(SearchHit searchHitFields) {
> >> + public Object[] apply(SearchHit hit) {
> >> Object[] objects = new Object[fields.size()];
> >> for (int i = 0; i < fields.size(); i++) {
> >> final Map.Entry<String, Class> field = fields.get(i);
> >> final String name = field.getKey();
> >> - if (searchHitFields.fields().isEmpty()) {
> >> - objects[i] =
> convert(searchHitFields.getSource().get(name), field.getValue());
> >> + if (hit.fields().isEmpty()) {
> >> + objects[i] = convert(hit.getSource().get(name),
> field.getValue());
> >> + } else if (hit.fields().containsKey(name)) {
> >> + objects[i] = convert(hit.field(name).getValue(),
> field.getValue());
> >> } else {
> >> - objects[i] =
> convert(searchHitFields.field(name).getValue(), field.getValue());
> >> + throw new IllegalStateException(
> >> + String.format(Locale.getDefault(), "No result for
> %s", field));
> >> }
> >> }
> >> return objects;
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >> ----------------------------------------------------------------------
> >> diff --git
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >> index 668402b..46e3fc5 100644
> >> ---
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >> +++
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
> >> @@ -22,6 +22,9 @@ import org.apache.calcite.schema.impl.AbstractSchema;
> >>
> >> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
> >>
> >> +import com.google.common.annotations.VisibleForTesting;
> >> +import com.google.common.base.Preconditions;
> >> +
> >> import com.google.common.collect.ImmutableList;
> >> import com.google.common.collect.ImmutableMap;
> >>
> >> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends
> AbstractSchema
> >> }
> >> }
> >>
> >> + /**
> >> + * Allows schema to be instantiated from existing elastic search
> client.
> >> + * This constructor is used in tests.
> >> + */
> >> + @VisibleForTesting
> >> + Elasticsearch2Schema(Client client, String index) {
> >> + this.client = Preconditions.checkNotNull(client, "client");
> >> + this.index = Preconditions.checkNotNull(index, "index");
> >> + }
> >> +
> >> @Override protected Map<String, Table> getTableMap() {
> >> final ImmutableMap.Builder<String, Table> builder =
> ImmutableMap.builder();
> >>
> >> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends
> AbstractSchema
> >>
> >> final List<DiscoveryNode> nodes =
> ImmutableList.copyOf(transportClient.connectedNodes());
> >> if (nodes.isEmpty()) {
> >> - throw new RuntimeException("Cannot connect to any elasticsearch
> nodes");
> >> + throw new IllegalStateException("Cannot connect to any
> elasticsearch node: "
> >> + + transportNodes);
> >> }
> >>
> >> client = transportClient;
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >> ----------------------------------------------------------------------
> >> diff --git
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >> index 636aa5f..2928835 100644
> >> ---
> a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >> +++
> b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
> >> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends
> AbstractElasticsearchTable {
> >> this.client = client;
> >> }
> >>
> >> + /**
> >> + * ES version 2.x. To access document attributes ES2 uses {@code
> _source.foo} syntax.
> >> + */
> >> + @Override protected String scriptedFieldPrefix() {
> >> + return "_source";
> >> + }
> >> +
> >> @Override protected Enumerable<Object> find(String index,
> List<String> ops,
> >> - List<Map.Entry<String, Class>> fields) {
> >> + List<Map.Entry<String,
> Class>> fields) {
> >> final String dbName = index;
> >>
> >> final String queryString = "{" + Util.toString(ops, "", ", ", "")
> + "}";
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >> ----------------------------------------------------------------------
> >> diff --git
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >> new file mode 100644
> >> index 0000000..287e094
> >> --- /dev/null
> >> +++
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
> >> @@ -0,0 +1,395 @@
> >> +/*
> >> + * Licensed to the Apache Software Foundation (ASF) under one or more
> >> + * contributor license agreements. See the NOTICE file distributed
> with
> >> + * this work for additional information regarding copyright ownership.
> >> + * The ASF licenses this file to you under the Apache License, Version
> 2.0
> >> + * (the "License"); you may not use this file except in compliance with
> >> + * the License. You may obtain a copy of the License at
> >> + *
> >> + * http://www.apache.org/licenses/LICENSE-2.0
> >> + *
> >> + * Unless required by applicable law or agreed to in writing, software
> >> + * distributed under the License is distributed on an "AS IS" BASIS,
> >> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> >> + * See the License for the specific language governing permissions and
> >> + * limitations under the License.
> >> + */
> >> +package org.apache.calcite.adapter.elasticsearch2;
> >> +
> >> +import org.apache.calcite.jdbc.CalciteConnection;
> >> +import org.apache.calcite.schema.SchemaPlus;
> >> +import org.apache.calcite.schema.impl.ViewTable;
> >> +import org.apache.calcite.schema.impl.ViewTableMacro;
> >> +import org.apache.calcite.test.CalciteAssert;
> >> +import org.apache.calcite.test.ElasticChecker;
> >> +
> >> +import com.google.common.io.LineProcessor;
> >> +import com.google.common.io.Resources;
> >> +
> >> +import org.elasticsearch.action.bulk.BulkItemResponse;
> >> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
> >> +import org.elasticsearch.action.bulk.BulkResponse;
> >> +import org.elasticsearch.common.xcontent.XContentBuilder;
> >> +import org.elasticsearch.common.xcontent.XContentFactory;
> >> +
> >> +import org.junit.BeforeClass;
> >> +import org.junit.ClassRule;
> >> +import org.junit.Test;
> >> +
> >> +import java.io.IOException;
> >> +import java.nio.charset.StandardCharsets;
> >> +import java.sql.Connection;
> >> +import java.sql.DriverManager;
> >> +import java.sql.SQLException;
> >> +import java.util.Arrays;
> >> +import java.util.Collections;
> >> +import java.util.Locale;
> >> +
> >> +/**
> >> + * Set of tests for ES adapter. Uses real instance via {@link
> EmbeddedElasticRule}. Document
> >> + * source is local {@code zips-mini.json} file (located in the
> classpath).
> >> + */
> >> +public class ElasticSearch2AdapterTest {
> >> +
> >> + @ClassRule //init once for all tests
> >> + public static final EmbeddedElasticRule NODE =
> EmbeddedElasticRule.create();
> >> +
> >> + private static final String ZIPS = "zips";
> >> +
> >> + /**
> >> + * Used to create {@code zips} index and insert some data
> >> + */
> >> + @BeforeClass
> >> + public static void setupInstance() throws Exception {
> >> + // define mapping so fields are searchable (term query)
> >> + XContentBuilder mapping =
> XContentFactory.jsonBuilder().startObject()
> >> + .startObject("properties")
> >> + .startObject("city").field("type", "string")
> >> + .field("index", "not_analyzed").endObject()
> >> + .startObject("state").field("type", "string")
> >> + .field("index", "not_analyzed").endObject()
> >> + .startObject("pop").field("type", "long").endObject()
> >> + .endObject()
> >> + .endObject();
> >> +
> >> + // create index
> >> + NODE.client().admin().indices()
> >> + .prepareCreate(ZIPS)
> >> + .addMapping(ZIPS, mapping)
> >> + .get();
> >> +
> >> + BulkRequestBuilder bulk =
> NODE.client().prepareBulk().setRefresh(true);
> >> +
> >> + // load records from file
> >> +
> Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
> >> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
> >> + @Override public boolean processLine(String line) throws
> IOException {
> >> + line = line.replaceAll("_id", "id"); // _id is a
> reserved attribute in ES
> >> + bulk.add(NODE.client().prepareIndex(ZIPS,
> ZIPS).setSource(line));
> >> + return true;
> >> + }
> >> +
> >> + @Override public Void getResult() {
> >> + return null;
> >> + }
> >> + });
> >> +
> >> + if (bulk.numberOfActions() == 0) {
> >> + throw new IllegalStateException("No records to be indexed");
> >> + }
> >> +
> >> + BulkResponse response = bulk.execute().get();
> >> +
> >> + if (response.hasFailures()) {
> >> + throw new IllegalStateException(
> >> + String.format(Locale.getDefault(), "Failed to populate
> %s:\n%s", NODE.httpAddress(),
> >> +
> Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
> >> +
> .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
> >> + }
> >> +
> >> + }
> >> +
> >> + private CalciteAssert.ConnectionFactory newConnectionFactory() {
> >> + return new CalciteAssert.ConnectionFactory() {
> >> + @Override public Connection createConnection() throws
> SQLException {
> >> + final Connection connection =
> DriverManager.getConnection("jdbc:calcite:");
> >> + final SchemaPlus root =
> connection.unwrap(CalciteConnection.class).getRootSchema();
> >> +
> >> + root.add("elastic", new Elasticsearch2Schema(NODE.client(),
> ZIPS));
> >> +
> >> + // add calcite view programmatically
> >> + final String viewSql = "select cast(_MAP['city'] AS
> varchar(20)) AS \"city\", "
> >> + + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n"
> >> + + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n"
> >> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
> >> + + " cast(_MAP['state'] AS varchar(2)) AS \"state\", "
> >> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
> >> + + "from \"elastic\".\"zips\"";
> >> +
> >> + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
> >> + Collections.singletonList("elastic"),
> Arrays.asList("elastic", "view"), false);
> >> + root.add("ZIPS", macro);
> >> +
> >> + return connection;
> >> + }
> >> + };
> >> + }
> >> +
> >> + private CalciteAssert.AssertThat calciteAssert() {
> >> + return CalciteAssert.that()
> >> + .with(newConnectionFactory());
> >> + }
> >> +
> >> + /**
> >> + * Tests using calcite view
> >> + */
> >> + @Test
> >> + public void view() throws Exception {
> >> + calciteAssert()
> >> + .query("select * from zips where \"city\" = 'BROOKLYN'")
> >> + .returns("city=BROOKLYN; longitude=-73.956985;
> latitude=40.646694; "
> >> + + "pop=111396; state=NY; id=11226\n")
> >> + .returnsCount(1);
> >> + }
> >> +
> >> + @Test
> >> + public void emptyResult() {
> >> + CalciteAssert.that()
> >> + .with(newConnectionFactory())
> >> + .query("select * from zips limit 0")
> >> + .returnsCount(0);
> >> +
> >> + CalciteAssert.that()
> >> + .with(newConnectionFactory())
> >> + .query("select * from \"elastic\".\"zips\" where
> _MAP['Foo'] = '_MISSING_'")
> >> + .returnsCount(0);
> >> + }
> >> +
> >> + @Test
> >> + public void basic() throws Exception {
> >> + CalciteAssert.that()
> >> + .with(newConnectionFactory())
> >> + .query("select * from \"elastic\".\"zips\" where
> _MAP['city'] = 'BROOKLYN'")
> >> + .returnsCount(1);
> >> +
> >> + CalciteAssert.that()
> >> + .with(newConnectionFactory())
> >> + .query("select * from \"elastic\".\"zips\" where"
> >> + + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')")
> >> + .returnsCount(2);
> >> +
> >> + // lower-case
> >> + CalciteAssert.that()
> >> + .with(newConnectionFactory())
> >> + .query("select * from \"elastic\".\"zips\" where "
> >> + + "_MAP['city'] in ('brooklyn', 'Brooklyn',
> 'BROOK') ")
> >> + .returnsCount(0);
> >> +
> >> + // missing field
> >> + CalciteAssert.that()
> >> + .with(newConnectionFactory())
> >> + .query("select * from \"elastic\".\"zips\" where
> _MAP['CITY'] = 'BROOKLYN'")
> >> + .returnsCount(0);
> >> +
> >> + // limit works
> >> + CalciteAssert.that()
> >> + .with(newConnectionFactory())
> >> + .query("select * from \"elastic\".\"zips\" limit 42")
> >> + .returnsCount(42);
> >> +
> >> + }
> >> +
> >> + @Test public void testSort() {
> >> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> >> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> >> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> >> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
> >> +
> >> + calciteAssert()
> >> + .query("select * from zips order by \"state\"")
> >> + .returnsCount(10)
> >> + .explainContains(explain);
> >> + }
> >> +
> >> + @Test public void testSortLimit() {
> >> + final String sql = "select \"state\", \"id\" from zips\n"
> >> + + "order by \"state\", \"id\" offset 2 rows fetch next 3
> rows only";
> >> + calciteAssert()
> >> + .query(sql)
> >> + .returnsUnordered("state=AK; id=99801",
> >> + "state=AL; id=35215",
> >> + "state=AL; id=35401")
> >> + .queryContains(
> >> + ElasticChecker.elasticsearchChecker(
> >> + "\"_source\" : [\"state\", \"id\"]",
> >> + "\"sort\": [ {\"state\": \"asc\"},
> {\"id\": \"asc\"}]",
> >> + "\"from\": 2",
> >> + "\"size\": 3"));
> >> + }
> >> +
> >> +
> >> +
> >> + @Test public void testOffsetLimit() {
> >> + final String sql = "select \"state\", \"id\" from zips\n"
> >> + + "offset 2 fetch next 3 rows only";
> >> + calciteAssert()
> >> + .query(sql)
> >> + .runs()
> >> + .queryContains(
> >> + ElasticChecker.elasticsearchChecker(
> >> + "\"from\": 2",
> >> + "\"size\": 3",
> >> + "\"_source\" : [\"state\", \"id\"]"));
> >> + }
> >> +
> >> + @Test public void testLimit() {
> >> + final String sql = "select \"state\", \"id\" from zips\n"
> >> + + "fetch next 3 rows only";
> >> +
> >> + calciteAssert()
> >> + .query(sql)
> >> + .runs()
> >> + .queryContains(
> >> + ElasticChecker.elasticsearchChecker(
> >> + "\"size\": 3",
> >> + "\"_source\" : [\"state\", \"id\"]"));
> >> + }
> >> +
> >> + @Test public void testFilterSort() {
> >> + final String sql = "select * from zips\n"
> >> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
> >> + + "order by \"state\", \"id\"";
> >> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> >> + + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC],
> dir1=[ASC])\n"
> >> + + " ElasticsearchProject(city=[CAST(ITEM($0,
> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> >> + + "
> ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0, 'state')):VARCHAR(2)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA'),
> >=(CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\", '70000'))])\n"
> >> + + " ElasticsearchTableScan(table=[[elastic,
> zips]])";
> >> + calciteAssert()
> >> + .query(sql)
> >> + .returnsOrdered("city=LOS ANGELES; longitude=-118.258189;
> latitude=34.007856; "
> >> + + "pop=96074; state=CA; id=90011",
> >> + "city=BELL GARDENS; longitude=-118.17205;
> latitude=33.969177; "
> >> + + "pop=99568; state=CA; id=90201",
> >> + "city=NORWALK; longitude=-118.081767;
> latitude=33.90564; "
> >> + + "pop=94188; state=CA; id=90650")
> >> + .queryContains(
> >> + ElasticChecker.elasticsearchChecker("\"query\" : "
> >> + +
> "{\"constant_score\":{\"filter\":{\"bool\":"
> >> + +
> "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
> >> + +
> "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
> >> + "\"script_fields\":
> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> >> + +
> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> >> + + "\"city\":{\"script\":
> \"_source.city\"}, "
> >> + + "\"pop\":{\"script\":
> \"_source.pop\"}, "
> >> + + "\"state\":{\"script\":
> \"_source.state\"}, "
> >> + + "\"id\":{\"script\": \"_
> source.id\"}}",
> >> + "\"sort\": [ {\"state\": \"asc\"},
> {\"id\": \"asc\"}]"))
> >> + .explainContains(explain);
> >> + }
> >> +
> >> + @Test public void testFilterSortDesc() {
> >> + final String sql = "select * from zips\n"
> >> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
> >> + + "order by \"state\" desc, \"pop\"";
> >> + calciteAssert()
> >> + .query(sql)
> >> + .limit(4)
> >> + .returnsOrdered(
> >> + "city=LOS ANGELES; longitude=-118.258189;
> latitude=34.007856; pop=96074; state=CA; id=90011",
> >> + "city=BELL GARDENS; longitude=-118.17205;
> latitude=33.969177; pop=99568; state=CA; id=90201");
> >> + }
> >> +
> >> + @Test public void testFilterRedundant() {
> >> + final String sql = "select * from zips\n"
> >> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and
> \"state\" = 'OK'";
> >> + calciteAssert()
> >> + .query(sql)
> >> + .runs()
> >> + .queryContains(
> >> + ElasticChecker.elasticsearchChecker(""
> >> + + "\"query\" :
> {\"constant_score\":{\"filter\":{\"bool\":"
> >> + +
> "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
> >> + "\"script_fields\":
> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> >> + +
> "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> >> + + "\"city\":{\"script\":
> \"_source.city\"}, "
> >> + + "\"pop\":{\"script\":
> \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
> >> + + "\"id\":{\"script\":
> \"_source.id\"}}"
> >> + ));
> >> + }
> >> +
> >> + @Test public void testInPlan() {
> >> + final String[] searches = {
> >> + "\"query\" :
> {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
> >> + +
> "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
> >> + + "{\"pop\":99568}}]}}]}}}}",
> >> + "\"script_fields\":
> {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
> >> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
> >> + + "\"city\":{\"script\": \"_source.city\"}, "
> >> + + "\"pop\":{\"script\": \"_source.pop\"}, "
> >> + + "\"state\":{\"script\": \"_source.state\"}, "
> >> + + "\"id\":{\"script\": \"_source.id\"}}"
> >> + };
> >> +
> >> + calciteAssert()
> >> + .query("select * from zips where \"pop\" in (96074,
> 99568)")
> >> + .returnsUnordered(
> >> + "city=BELL GARDENS; longitude=-118.17205;
> latitude=33.969177; pop=99568; state=CA; id=90201",
> >> + "city=LOS ANGELES; longitude=-118.258189;
> latitude=34.007856; pop=96074; state=CA; id=90011"
> >> + )
> >> +
> .queryContains(ElasticChecker.elasticsearchChecker(searches));
> >> + }
> >> +
> >> + @Test public void testZips() {
> >> + calciteAssert()
> >> + .query("select \"state\", \"city\" from zips")
> >> + .returnsCount(10);
> >> + }
> >> +
> >> + @Test public void testProject() {
> >> + final String sql = "select \"state\", \"city\", 0 as \"zero\"\n"
> >> + + "from zips\n"
> >> + + "order by \"state\", \"city\"";
> >> +
> >> + calciteAssert()
> >> + .query(sql)
> >> + .limit(2)
> >> + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0",
> >> + "state=AK; city=FAIRBANKS; zero=0")
> >> + .queryContains(
> >> +
> ElasticChecker.elasticsearchChecker("\"script_fields\": "
> >> + + "{\"zero\":{\"script\": \"0\"}, "
> >> + + "\"state\":{\"script\":
> \"_source.state\"}, "
> >> + + "\"city\":{\"script\":
> \"_source.city\"}}",
> >> + "\"sort\": [ {\"state\": \"asc\"},
> {\"city\": \"asc\"}]"));
> >> + }
> >> +
> >> + @Test public void testFilter() {
> >> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> >> + + " ElasticsearchProject(state=[CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> >> + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\", 'CA')])\n"
> >> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
> >> + calciteAssert()
> >> + .query("select \"state\", \"city\" from zips where
> \"state\" = 'CA'")
> >> + .limit(3)
> >> + .returnsUnordered("state=CA; city=BELL GARDENS",
> >> + "state=CA; city=LOS ANGELES",
> >> + "state=CA; city=NORWALK")
> >> + .explainContains(explain);
> >> + }
> >> +
> >> + @Test public void testFilterReversed() {
> >> + calciteAssert()
> >> + .query("select \"state\", \"city\" from zips where 'WI' <
> \"state\" order by \"city\"")
> >> + .limit(2)
> >> + .returnsUnordered("state=WV; city=BECKLEY",
> >> + "state=WY; city=CHEYENNE");
> >> + calciteAssert()
> >> + .query("select \"state\", \"city\" from zips where \"state\"
> > 'WI' order by \"city\"")
> >> + .limit(2)
> >> + .returnsUnordered("state=WV; city=BECKLEY",
> >> + "state=WY; city=CHEYENNE");
> >> + }
> >> +
> >> +}
> >> +
> >> +// End ElasticSearch2AdapterTest.java
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >> ----------------------------------------------------------------------
> >> diff --git
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >> new file mode 100644
> >> index 0000000..4474add
> >> --- /dev/null
> >> +++
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
> >> @@ -0,0 +1,147 @@
> >> +/*
> >> + * Licensed to the Apache Software Foundation (ASF) under one or more
> >> + * contributor license agreements. See the NOTICE file distributed
> with
> >> + * this work for additional information regarding copyright ownership.
> >> + * The ASF licenses this file to you under the Apache License, Version
> 2.0
> >> + * (the "License"); you may not use this file except in compliance with
> >> + * the License. You may obtain a copy of the License at
> >> + *
> >> + * http://www.apache.org/licenses/LICENSE-2.0
> >> + *
> >> + * Unless required by applicable law or agreed to in writing, software
> >> + * distributed under the License is distributed on an "AS IS" BASIS,
> >> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> >> + * See the License for the specific language governing permissions and
> >> + * limitations under the License.
> >> + */
> >> +package org.apache.calcite.adapter.elasticsearch2;
> >> +
> >> +import com.google.common.base.Preconditions;
> >> +import com.google.common.io.Files;
> >> +
> >> +import org.elasticsearch.Version;
> >> +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
> >> +import org.elasticsearch.action.admin.cluster.node.info
> .NodesInfoResponse;
> >> +import org.elasticsearch.client.Client;
> >> +import org.elasticsearch.common.settings.Settings;
> >> +import org.elasticsearch.common.transport.TransportAddress;
> >> +import org.elasticsearch.node.Node;
> >> +import org.elasticsearch.node.internal.InternalSettingsPreparer;
> >> +import org.elasticsearch.plugins.Plugin;
> >> +import org.elasticsearch.script.groovy.GroovyPlugin;
> >> +
> >> +import java.io.File;
> >> +import java.util.Arrays;
> >> +import java.util.Collection;
> >> +import java.util.Collections;
> >> +
> >> +/**
> >> + * Represents a single elastic search node which can run embedded in a
> java application.
> >> + * Intended for unit and integration tests. Settings and plugins are
> crafted for Calcite.
> >> + */
> >> +class EmbeddedElasticNode implements AutoCloseable {
> >> +
> >> + private final LocalNode node;
> >> + private volatile boolean isStarted;
> >> +
> >> + private EmbeddedElasticNode(LocalNode node) {
> >> + this.node = Preconditions.checkNotNull(node, "node");
> >> + }
> >> +
> >> + /**
> >> + * Having separate class to expose (protected) constructor which
> allows to install
> >> + * different plugins. In our case it is {@code GroovyPlugin} for
> scripted fields like
> >> + * {@code loc[0]} or {@code loc[1]['foo']}.
> >> + */
> >> + private static class LocalNode extends Node {
> >> + private LocalNode(Settings settings, Collection<Class<? extends
> Plugin>> classpathPlugins) {
> >> + super(InternalSettingsPreparer.prepareEnvironment(settings,
> null),
> >> + Version.CURRENT,
> >> + classpathPlugins);
> >> + }
> >> + }
> >> +
> >> + /**
> >> + * Creates an instance with existing settings
> >> + */
> >> + private static EmbeddedElasticNode create(Settings settings) {
> >> + // ensure GroovyPlugin is installed or otherwise scripted fields
> would not work
> >> + LocalNode node = new LocalNode(settings,
> Collections.singleton(GroovyPlugin.class));
> >> + return new EmbeddedElasticNode(node);
> >> + }
> >> +
> >> + /**
> >> + * Creates elastic node as single member of a cluster. Node will not
> be started
> >> + * unless {@link #start()} is explicitly called.
> >> + */
> >> + public static EmbeddedElasticNode create() {
> >> + File data = Files.createTempDir();
> >> + data.deleteOnExit();
> >> + File home = Files.createTempDir();
> >> + home.deleteOnExit();
> >> +
> >> + Settings settings = Settings.builder()
> >> + .put("node.name", "fake-elastic")
> >> + .put("path.home", home.getAbsolutePath())
> >> + .put("path.data", data.getAbsolutePath())
> >> + .put("script.inline", true) // requires GroovyPlugin
> >> + .put("script.indexed", true) // requires GroovyPlugin
> >> + .put("cluster.routing.allocation.disk.threshold_enabled",
> false)
> >> + .put("node.local", true)
> >> + .put("node.data", true)
> >> + .put("network.host", "localhost")
> >> + .build();
> >> +
> >> + return create(settings);
> >> + }
> >> +
> >> + /**
> >> + * Starts current node
> >> + */
> >> + public void start() {
> >> + Preconditions.checkState(!isStarted, "already started");
> >> + node.start();
> >> + this.isStarted = true;
> >> + }
> >> +
> >> + /**
> >> + * Returns current address to connect to with HTTP client.
> >> + */
> >> + public TransportAddress httpAddress() {
> >> + Preconditions.checkState(isStarted, "node is not started");
> >> +
> >> + NodesInfoResponse response =
> client().admin().cluster().prepareNodesInfo()
> >> + .execute().actionGet();
> >> + if (response.getNodes().length != 1) {
> >> + throw new IllegalStateException("Expected single node but got "
> >> + + response.getNodes().length);
> >> + }
> >> + NodeInfo node = response.getNodes()[0];
> >> + return node.getHttp().address().boundAddresses()[0];
> >> + }
> >> +
> >> + /**
> >> + * Exposes elastic
> >> + * <a href="
> https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html">transport
> client</a>
> >> + *
> >> + * (use of HTTP client is preferred).
> >> + */
> >> + public Client client() {
> >> + Preconditions.checkState(isStarted, "node is not started");
> >> + return node.client();
> >> + }
> >> +
> >> + @Override public void close() throws Exception {
> >> + node.close();
> >> + // cleanup data dirs
> >> + File data = new File(node.settings().get("path.data"));
> >> + File home = new File(node.settings().get("path.home"));
> >> + for (File file: Arrays.asList(data, home)) {
> >> + if (file.exists()) {
> >> + file.delete();
> >> + }
> >> + }
> >> + }
> >> +}
> >> +
> >> +// End EmbeddedElasticNode.java
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> >> ----------------------------------------------------------------------
> >> diff --git
> a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> >> new file mode 100644
> >> index 0000000..a633078
> >> --- /dev/null
> >> +++
> b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
> >> @@ -0,0 +1,97 @@
> >> +/*
> >> + * Licensed to the Apache Software Foundation (ASF) under one or more
> >> + * contributor license agreements. See the NOTICE file distributed
> with
> >> + * this work for additional information regarding copyright ownership.
> >> + * The ASF licenses this file to you under the Apache License, Version
> 2.0
> >> + * (the "License"); you may not use this file except in compliance with
> >> + * the License. You may obtain a copy of the License at
> >> + *
> >> + * http://www.apache.org/licenses/LICENSE-2.0
> >> + *
> >> + * Unless required by applicable law or agreed to in writing, software
> >> + * distributed under the License is distributed on an "AS IS" BASIS,
> >> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> >> + * See the License for the specific language governing permissions and
> >> + * limitations under the License.
> >> + */
> >> +package org.apache.calcite.adapter.elasticsearch2;
> >> +
> >> +import com.google.common.base.Preconditions;
> >> +
> >> +import org.elasticsearch.client.Client;
> >> +import org.elasticsearch.common.transport.TransportAddress;
> >> +import org.junit.rules.ExternalResource;
> >> +
> >> +/**
> >> + * Used to initialize a single elastic node. For performance reasons
> (node startup costs),
> >> + * same instance is usually shared across multiple tests.
> >> + *
> >> + * This rule should be used as follows:
> >> + * <pre>
> >> + * {@code
> >> + *
> >> + * public class MyTest {
> >> + * @literal @ClassRule
> >> + * public static final ElasticSearchRule RULE =
> ElasticSearchRule.create();
> >> + *
> >> + * @literal @BeforeClass
> >> + * public void setup() {
> >> + * // ... populate instance
> >> + * }
> >> + *
> >> + * @literal @Test
> >> + * public void myTest() {
> >> + * TransportAddress address = RULE.httpAddress();
> >> + * // ....
> >> + * }
> >> + * }
> >> + * }
> >> + * </pre>
> >> + *
> >> + * @see ExternalResource
> >> + */
> >> +class EmbeddedElasticRule extends ExternalResource {
> >> +
> >> + private final EmbeddedElasticNode node;
> >> +
> >> + private EmbeddedElasticRule(EmbeddedElasticNode resource) {
> >> + this.node = Preconditions.checkNotNull(resource, "resource");
> >> + }
> >> +
> >> + @Override protected void before() throws Throwable {
> >> + node.start();
> >> + }
> >> +
> >> + @Override protected void after() {
> >> + try {
> >> + node.close();
> >> + } catch (Exception e) {
> >> + throw new RuntimeException(e);
> >> + }
> >> + }
> >> +
> >> + /**
> >> + * Factory method to create this rule.
> >> + */
> >> + public static EmbeddedElasticRule create() {
> >> + return new EmbeddedElasticRule(EmbeddedElasticNode.create());
> >> + }
> >> +
> >> + /**
> >> + * Exposes current ES transport client.
> >> + */
> >> + Client client() {
> >> + return node.client();
> >> + }
> >> +
> >> + /**
> >> + * HTTP address for rest clients (can be ES native or any other).
> >> + */
> >> + TransportAddress httpAddress() {
> >> + return node.httpAddress();
> >> + }
> >> +
> >> +
> >> +}
> >> +
> >> +// End EmbeddedElasticRule.java
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> >> ----------------------------------------------------------------------
> >> diff --git
> a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> >> new file mode 100644
> >> index 0000000..21fc491
> >> --- /dev/null
> >> +++
> b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
> >> @@ -0,0 +1,49 @@
> >> +/*
> >> + * Licensed to the Apache Software Foundation (ASF) under one or more
> >> + * contributor license agreements. See the NOTICE file distributed
> with
> >> + * this work for additional information regarding copyright ownership.
> >> + * The ASF licenses this file to you under the Apache License, Version
> 2.0
> >> + * (the "License"); you may not use this file except in compliance with
> >> + * the License. You may obtain a copy of the License at
> >> + *
> >> + * http://www.apache.org/licenses/LICENSE-2.0
> >> + *
> >> + * Unless required by applicable law or agreed to in writing, software
> >> + * distributed under the License is distributed on an "AS IS" BASIS,
> >> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> >> + * See the License for the specific language governing permissions and
> >> + * limitations under the License.
> >> + */
> >> +package org.apache.calcite.test;
> >> +
> >> +import com.google.common.base.Function;
> >> +
> >> +import java.util.List;
> >> +
> >> +import javax.annotation.Nullable;
> >> +
> >> +/**
> >> + * Internal util methods for ElasticSearch tests
> >> + */
> >> +public class ElasticChecker {
> >> +
> >> + private ElasticChecker() {}
> >> +
> >> +
> >> + /** Returns a function that checks that a particular Elasticsearch
> pipeline is
> >> + * generated to implement a query. */
> >> + public static Function<List, Void> elasticsearchChecker(final
> String... strings) {
> >> + return new Function<List, Void>() {
> >> + @Nullable
> >> + @Override public Void apply(@Nullable List actual) {
> >> + Object[] actualArray = actual == null || actual.isEmpty() ?
> null
> >> + : ((List) actual.get(0)).toArray();
> >> + CalciteAssert.assertArrayEqual("expected Elasticsearch query
> not found", strings,
> >> + actualArray);
> >> + return null;
> >> + }
> >> + };
> >> + }
> >> +}
> >> +
> >> +// End ElasticChecker.java
> >>
> >>
> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> >> ----------------------------------------------------------------------
> >> diff --git
> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> b/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> >> deleted file mode 100644
> >> index 4e0c2b6..0000000
> >> ---
> a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
> >> +++ /dev/null
> >> @@ -1,270 +0,0 @@
> >> -/*
> >> - * Licensed to the Apache Software Foundation (ASF) under one or more
> >> - * contributor license agreements. See the NOTICE file distributed
> with
> >> - * this work for additional information regarding copyright ownership.
> >> - * The ASF licenses this file to you under the Apache License, Version
> 2.0
> >> - * (the "License"); you may not use this file except in compliance with
> >> - * the License. You may obtain a copy of the License at
> >> - *
> >> - * http://www.apache.org/licenses/LICENSE-2.0
> >> - *
> >> - * Unless required by applicable law or agreed to in writing, software
> >> - * distributed under the License is distributed on an "AS IS" BASIS,
> >> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> >> - * See the License for the specific language governing permissions and
> >> - * limitations under the License.
> >> - */
> >> -package org.apache.calcite.test;
> >> -
> >> -import org.apache.calcite.util.Util;
> >> -
> >> -import com.google.common.base.Function;
> >> -import com.google.common.collect.ImmutableMap;
> >> -
> >> -import org.junit.Test;
> >> -
> >> -import java.util.List;
> >> -import javax.annotation.Nullable;
> >> -
> >> -/**
> >> - * Tests for the {@code org.apache.calcite.adapter.elasticsearch2}
> package.
> >> - *
> >> - * <p>Before calling this test, you need to populate Elasticsearch, as
> follows:
> >> - *
> >> - * <blockquote><code>
> >> - * git clone https://github.com/vlsi/calcite-test-dataset<br>
> >> - * cd calcite-test-dataset<br>
> >> - * mvn install
> >> - * </code></blockquote>
> >> - *
> >> - * <p>This will create a virtual machine with Elasticsearch and the
> "zips" test
> >> - * dataset.
> >> - */
> >> -public class Elasticsearch2AdapterIT {
> >> - /**
> >> - * Whether to run Elasticsearch tests. Enabled by default, however
> test is only
> >> - * included if "it" profile is activated ({@code -Pit}). To disable,
> >> - * specify {@code -Dcalcite.test.elasticsearch=false} on the Java
> command line.
> >> - */
> >> - private static final boolean ENABLED =
> Util.getBooleanProperty("calcite.test.elasticsearch",
> >> - true);
> >> -
> >> - /** Connection factory based on the "zips-es" model. */
> >> - private static final ImmutableMap<String, String> ZIPS =
> ImmutableMap.of("model",
> >> -
> Elasticsearch2AdapterIT.class.getResource("/elasticsearch-zips-model.json").getPath());
> >> -
> >> - /** Whether to run this test. */
> >> - private boolean enabled() {
> >> - return ENABLED;
> >> - }
> >> -
> >> - /** Returns a function that checks that a particular Elasticsearch
> pipeline is
> >> - * generated to implement a query. */
> >> - private static Function<List, Void> elasticsearchChecker(final
> String... strings) {
> >> - return new Function<List, Void>() {
> >> - @Nullable
> >> - @Override public Void apply(@Nullable List actual) {
> >> - Object[] actualArray = actual == null || actual.isEmpty() ?
> null
> >> - : ((List) actual.get(0)).toArray();
> >> - CalciteAssert.assertArrayEqual("expected Elasticsearch query
> not found", strings,
> >> - actualArray);
> >> - return null;
> >> - }
> >> - };
> >> - }
> >> -
> >> - @Test public void testSort() {
> >> - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> >> - + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
> >> - + " ElasticsearchProject(city=[CAST(ITEM($0,
> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> >> - + " ElasticsearchTableScan(table=[[elasticsearch_raw,
> zips]])";
> >> - CalciteAssert.that()
> >> - .enable(enabled())
> >> - .with(ZIPS)
> >> - .query("select * from zips order by \"state\"")
> >> - .returnsCount(10)
> >> - .explainContains(explain);
> >> - }
> >> -
> >> - @Test public void testSortLimit() {
> >> - final String sql = "select \"state\", \"id\" from zips\n"
> >> - + "order by \"state\", \"id\" offset 2 rows fetch next 3 rows
> only";
> >> - CalciteAssert.that()
> >> - .with(ZIPS)
> >> - .query(sql)
> >> - .returnsUnordered("state=AK; id=99503",
> >> - "state=AK; id=99504",
> >> - "state=AK; id=99505")
> >> - .queryContains(
> >> - elasticsearchChecker(
> >> - "\"fields\" : [\"state\", \"id\"], \"script_fields\":
> {}",
> >> - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]",
> >> - "\"from\": 2",
> >> - "\"size\": 3"));
> >> - }
> >> -
> >> - @Test public void testOffsetLimit() {
> >> - final String sql = "select \"state\", \"id\" from zips\n"
> >> - + "offset 2 fetch next 3 rows only";
> >> - CalciteAssert.that()
> >> - .enable(enabled())
> >> - .with(ZIPS)
> >> - .query(sql)
> >> - .runs()
> >> - .queryContains(
> >> - elasticsearchChecker(
> >> - "\"from\": 2",
> >> - "\"size\": 3",
> >> - "\"fields\" : [\"state\", \"id\"], \"script_fields\":
> {}"));
> >> - }
> >> -
> >> - @Test public void testLimit() {
> >> - final String sql = "select \"state\", \"id\" from zips\n"
> >> - + "fetch next 3 rows only";
> >> - CalciteAssert.that()
> >> - .enable(enabled())
> >> - .with(ZIPS)
> >> - .query(sql)
> >> - .runs()
> >> - .queryContains(
> >> - elasticsearchChecker(
> >> - "\"size\": 3",
> >> - "\"fields\" : [\"state\", \"id\"], \"script_fields\":
> {}"));
> >> - }
> >> -
> >> - @Test public void testFilterSort() {
> >> - final String sql = "select * from zips\n"
> >> - + "where \"city\" = 'SPRINGFIELD' and \"id\" >= '70000'\n"
> >> - + "order by \"state\", \"id\"";
> >> - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
> >> - + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC],
> dir1=[ASC])\n"
> >> - + " ElasticsearchProject(city=[CAST(ITEM($0,
> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'),
> 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT],
> pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0,
> 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5)
> CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
> >> - + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0,
> 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\", 'SPRINGFIELD'), >=(CAST(ITEM($0,
> 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE
> \"ISO-8859-1$en_US$primary\", '70000'))])\n"
> >> - + " ElasticsearchTableScan(table=[[elasticsearch_raw,
> zips]])";
> >> - CalciteAssert.that()
> >> - .enable(enabled())
> >> - .with(ZIPS)
> >> - .query(sql)
> >> - .returnsOrdered(
> >> - "city=SPRINGFIELD; longitude=-92.54567;
> latitude=35.274879; pop=752; state=AR; id=72157",
> >> - "city=SPRINGFIELD; longitude=-102.617322;
> latitude=37.406727; pop=1992; state=CO; id=81073",
> >> - "city=SPRINGFIELD; longitude=-90.577479;
> latitude=30.415738; pop=5597; state=LA; id=70462",
> >> - "city=SPRINGFIELD; longitude=-123.015259;
> latitude=44.06106; pop=32384; state=OR; id=97477",
> >> - "city=SPRINGFIELD; longitude=-122.917108;
> latitude=44.056056; pop=27521; state=OR; id=97478")
> >> - .queryContains(
> >> - elasticsearchChecker("\"query\" :
> {\"constant_score\":{\"filter\":{\"bool\":"
> >> - + "{\"must\
Re: [2/2] calcite git commit: [CALCITE-2347] running ElasticSearch in
embedded mode for unit tests of ES adapter (Andrei Sereda)
Posted by Christian Beikov <ch...@gmail.com>.
Huh? How is that possible? I thought the checks that are run by the
Travis CI build catch these kinds of errors.
Mit freundlichen Grüßen,
------------------------------------------------------------------------
*Christian Beikov*
Am 22.06.2018 um 23:11 schrieb Julian Hyde:
> Looks like this change broke “mvn site” (perhaps also “mvn javadoc:test-javadoc”).
>
> [ERROR] Failed to execute goal org.apache.maven.plugins:maven-site-plugin:3.7:site (default-site) on project calcite: Error generating maven-javadoc-plugin:3.0.1:test-aggregate report:
> [ERROR] Exit code: 1 - /home/jhyde/regress/calcite/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java:29: error: package org.elasticsearch.node.internal does not exist
> [ERROR] import org.elasticsearch.node.internal.InternalSettingsPreparer;
> [ERROR] ^
> [ERROR]
>
>> On Jun 21, 2018, at 3:39 AM, mmior@apache.org wrote:
>>
>> [CALCITE-2347] running ElasticSearch in embedded mode for unit tests of ES adapter (Andrei Sereda)
>>
>> After discussion on dev-list Integration tests (for ES) have been removed. They're now
>> superseded by unit tests (which execute queries against a real elastic instance)
>>
>> Added local file (zips-mini.json) which contains a small subset of original zips.json
>> (allows to bootstrap tests faster)
>>
>> Created separate ES JUnit rule which can be re-used across different tests.
>>
>> Both v2 and v5 of ES adapters are supported.
>>
>> Close apache/calcite#716
>>
>>
>> Project: http://git-wip-us.apache.org/repos/asf/calcite/repo
>> Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/c12cb4b0
>> Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/c12cb4b0
>> Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/c12cb4b0
>>
>> Branch: refs/heads/master
>> Commit: c12cb4b0de1baa3f7cbb9952ee350fdd1701662d
>> Parents: 37944bb
>> Author: Andrei Sereda <an...@nospam.com>
>> Authored: Thu May 31 18:19:10 2018 -0400
>> Committer: Michael Mior <mm...@uwaterloo.ca>
>> Committed: Thu Jun 21 06:38:50 2018 -0400
>>
>> ----------------------------------------------------------------------
>> .../AbstractElasticsearchTable.java | 12 +
>> .../elasticsearch/ElasticsearchProject.java | 61 ++-
>> elasticsearch2/pom.xml | 6 +
>> .../Elasticsearch2Enumerator.java | 12 +-
>> .../elasticsearch2/Elasticsearch2Schema.java | 16 +-
>> .../elasticsearch2/Elasticsearch2Table.java | 9 +-
>> .../ElasticSearch2AdapterTest.java | 395 ++++++++++++++++++
>> .../elasticsearch2/EmbeddedElasticNode.java | 147 +++++++
>> .../elasticsearch2/EmbeddedElasticRule.java | 97 +++++
>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
>> .../calcite/test/Elasticsearch2AdapterIT.java | 270 -------------
>> .../resources/elasticsearch-zips-model.json | 50 ---
>> .../src/test/resources/zips-mini.json | 149 +++++++
>> elasticsearch5/pom.xml | 31 ++
>> .../elasticsearch5/Elasticsearch5Schema.java | 17 +-
>> .../elasticsearch5/Elasticsearch5Table.java | 11 +-
>> .../ElasticSearch5AdapterTest.java | 399 +++++++++++++++++++
>> .../elasticsearch5/EmbeddedElasticNode.java | 153 +++++++
>> .../elasticsearch5/EmbeddedElasticRule.java | 98 +++++
>> .../org/apache/calcite/test/ElasticChecker.java | 49 +++
>> .../calcite/test/Elasticsearch5AdapterIT.java | 270 -------------
>> .../resources/elasticsearch-zips-model.json | 50 ---
>> elasticsearch5/src/test/resources/log4j2.xml | 16 +
>> .../src/test/resources/zips-mini.json | 149 +++++++
>> pom.xml | 20 +-
>> 25 files changed, 1866 insertions(+), 670 deletions(-)
>> ----------------------------------------------------------------------
>>
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>> ----------------------------------------------------------------------
>> diff --git a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>> index 0980469..8cc5933 100644
>> --- a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>> +++ b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/AbstractElasticsearchTable.java
>> @@ -75,6 +75,18 @@ public abstract class AbstractElasticsearchTable extends AbstractQueryableTable
>> relOptTable, this, null);
>> }
>>
>> + /**
>> + * In ES 5.x scripted fields start with {@code params._source.foo} while in ES2.x
>> + * {@code _source.foo}. Helper method to build correct query based on runtime version of elastic.
>> + *
>> + * @see <a href="https://github.com/elastic/elasticsearch/issues/20068">_source variable</a>
>> + * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-fields.html">Scripted Fields</a>
>> + */
>> + protected String scriptedFieldPrefix() {
>> + // this is default pattern starting 5.x
>> + return "params._source";
>> + }
>> +
>> /** Executes a "find" operation on the underlying type.
>> *
>> * <p>For example,
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>> ----------------------------------------------------------------------
>> diff --git a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>> index b42abd7..961c8b0 100644
>> --- a/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>> +++ b/core/src/main/java/org/apache/calcite/adapter/elasticsearch/ElasticsearchProject.java
>> @@ -27,11 +27,15 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery;
>> import org.apache.calcite.rel.type.RelDataType;
>> import org.apache.calcite.rex.RexNode;
>> import org.apache.calcite.util.Pair;
>> -import org.apache.calcite.util.Util;
>> +
>> +import com.google.common.base.Function;
>> +import com.google.common.collect.Lists;
>>
>> import java.util.ArrayList;
>> import java.util.List;
>>
>> +import javax.annotation.Nullable;
>> +
>> /**
>> * Implementation of {@link org.apache.calcite.rel.core.Project}
>> * relational expression in Elasticsearch.
>> @@ -57,41 +61,60 @@ public class ElasticsearchProject extends Project implements ElasticsearchRel {
>> implementor.visitChild(0, getInput());
>>
>> final List<String> inFields =
>> - ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
>> + ElasticsearchRules.elasticsearchFieldNames(getInput().getRowType());
>> final ElasticsearchRules.RexToElasticsearchTranslator translator =
>> - new ElasticsearchRules.RexToElasticsearchTranslator(
>> - (JavaTypeFactory) getCluster().getTypeFactory(), inFields);
>> + new ElasticsearchRules.RexToElasticsearchTranslator(
>> + (JavaTypeFactory) getCluster().getTypeFactory(), inFields);
>>
>> - final List<String> findItems = new ArrayList<>();
>> - final List<String> scriptFieldItems = new ArrayList<>();
>> + final List<String> fields = new ArrayList<>();
>> + final List<String> scriptFields = new ArrayList<>();
>> for (Pair<RexNode, String> pair: getNamedProjects()) {
>> final String name = pair.right;
>> final String expr = pair.left.accept(translator);
>>
>> if (expr.equals("\"" + name + "\"")) {
>> - findItems.add(ElasticsearchRules.quote(name));
>> + fields.add(name);
>> } else if (expr.matches("\"literal\":.+")) {
>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
>> - + ":{\"script\": "
>> - + expr.split(":")[1] + "}");
>> + scriptFields.add(ElasticsearchRules.quote(name)
>> + + ":{\"script\": "
>> + + expr.split(":")[1] + "}");
>> } else {
>> - scriptFieldItems.add(ElasticsearchRules.quote(name)
>> - + ":{\"script\":\"params._source."
>> - + expr.replaceAll("\"", "") + "\"}");
>> + scriptFields.add(ElasticsearchRules.quote(name)
>> + + ":{\"script\":"
>> + // _source (ES2) vs params._source (ES5)
>> + + "\"" + implementor.elasticsearchTable.scriptedFieldPrefix() + "."
>> + + expr.replaceAll("\"", "") + "\"}");
>> + }
>> + }
>> +
>> + StringBuilder query = new StringBuilder();
>> + if (scriptFields.isEmpty()) {
>> + List<String> newList = Lists.transform(fields, new Function<String, String>() {
>> + @Nullable
>> + @Override public String apply(@Nullable String input) {
>> + return ElasticsearchRules.quote(input);
>> + }
>> + });
>> +
>> + final String findString = String.join(", ", newList);
>> + query.append("\"_source\" : [").append(findString).append("]");
>> + } else {
>> + // if scripted fields are present, ES ignores _source attribute
>> + for (String field: fields) {
>> + scriptFields.add(ElasticsearchRules.quote(field) + ":{\"script\": "
>> + // _source (ES2) vs params._source (ES5)
>> + + "\"" + implementor.elasticsearchTable.scriptedFieldPrefix() + "."
>> + + field + "\"}");
>> }
>> + query.append("\"script_fields\": {" + String.join(", ", scriptFields) + "}");
>> }
>> - final String findString = Util.toString(findItems, "", ", ", "");
>> - final String scriptFieldString = "\"script_fields\": {"
>> - + Util.toString(scriptFieldItems, "", ", ", "") + "}";
>> - final String fieldString = "\"_source\" : [" + findString + "]"
>> - + ", " + scriptFieldString;
>>
>> for (String opfield : implementor.list) {
>> if (opfield.startsWith("\"_source\"")) {
>> implementor.list.remove(opfield);
>> }
>> }
>> - implementor.add(fieldString);
>> + implementor.add(query.toString());
>> }
>> }
>>
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/pom.xml
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/pom.xml b/elasticsearch2/pom.xml
>> index f24622c..6fbee03 100644
>> --- a/elasticsearch2/pom.xml
>> +++ b/elasticsearch2/pom.xml
>> @@ -73,6 +73,12 @@ limitations under the License.
>> <version>${elasticsearch-java-driver.version}</version>
>> </dependency>
>> <dependency>
>> + <!-- Lang groovy dependency is needed for testing with embedded ES (scripted fields like loc[0]) -->
>> + <groupId>org.elasticsearch.module</groupId>
>> + <artifactId>lang-groovy</artifactId>
>> + <scope>test</scope>
>> + </dependency>
>> + <dependency>
>> <groupId>com.carrotsearch</groupId>
>> <artifactId>hppc</artifactId>
>> <version>${hppc.version}</version>
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>> index 84370ab..c3d2ac0 100644
>> --- a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>> +++ b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Enumerator.java
>> @@ -26,6 +26,7 @@ import org.elasticsearch.search.SearchHit;
>> import java.util.Date;
>> import java.util.Iterator;
>> import java.util.List;
>> +import java.util.Locale;
>> import java.util.Map;
>>
>> /**
>> @@ -101,15 +102,18 @@ public class Elasticsearch2Enumerator implements Enumerator<Object> {
>> private static Function1<SearchHit, Object[]> listGetter(
>> final List<Map.Entry<String, Class>> fields) {
>> return new Function1<SearchHit, Object[]>() {
>> - public Object[] apply(SearchHit searchHitFields) {
>> + public Object[] apply(SearchHit hit) {
>> Object[] objects = new Object[fields.size()];
>> for (int i = 0; i < fields.size(); i++) {
>> final Map.Entry<String, Class> field = fields.get(i);
>> final String name = field.getKey();
>> - if (searchHitFields.fields().isEmpty()) {
>> - objects[i] = convert(searchHitFields.getSource().get(name), field.getValue());
>> + if (hit.fields().isEmpty()) {
>> + objects[i] = convert(hit.getSource().get(name), field.getValue());
>> + } else if (hit.fields().containsKey(name)) {
>> + objects[i] = convert(hit.field(name).getValue(), field.getValue());
>> } else {
>> - objects[i] = convert(searchHitFields.field(name).getValue(), field.getValue());
>> + throw new IllegalStateException(
>> + String.format(Locale.getDefault(), "No result for %s", field));
>> }
>> }
>> return objects;
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>> index 668402b..46e3fc5 100644
>> --- a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>> +++ b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Schema.java
>> @@ -22,6 +22,9 @@ import org.apache.calcite.schema.impl.AbstractSchema;
>>
>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
>>
>> +import com.google.common.annotations.VisibleForTesting;
>> +import com.google.common.base.Preconditions;
>> +
>> import com.google.common.collect.ImmutableList;
>> import com.google.common.collect.ImmutableMap;
>>
>> @@ -86,6 +89,16 @@ public class Elasticsearch2Schema extends AbstractSchema
>> }
>> }
>>
>> + /**
>> + * Allows schema to be instantiated from existing elastic search client.
>> + * This constructor is used in tests.
>> + */
>> + @VisibleForTesting
>> + Elasticsearch2Schema(Client client, String index) {
>> + this.client = Preconditions.checkNotNull(client, "client");
>> + this.index = Preconditions.checkNotNull(index, "index");
>> + }
>> +
>> @Override protected Map<String, Table> getTableMap() {
>> final ImmutableMap.Builder<String, Table> builder = ImmutableMap.builder();
>>
>> @@ -120,7 +133,8 @@ public class Elasticsearch2Schema extends AbstractSchema
>>
>> final List<DiscoveryNode> nodes = ImmutableList.copyOf(transportClient.connectedNodes());
>> if (nodes.isEmpty()) {
>> - throw new RuntimeException("Cannot connect to any elasticsearch nodes");
>> + throw new IllegalStateException("Cannot connect to any elasticsearch node: "
>> + + transportNodes);
>> }
>>
>> client = transportClient;
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>> index 636aa5f..2928835 100644
>> --- a/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>> +++ b/elasticsearch2/src/main/java/org/apache/calcite/adapter/elasticsearch2/Elasticsearch2Table.java
>> @@ -45,8 +45,15 @@ public class Elasticsearch2Table extends AbstractElasticsearchTable {
>> this.client = client;
>> }
>>
>> + /**
>> + * ES version 2.x. To access document attributes ES2 uses {@code _source.foo} syntax.
>> + */
>> + @Override protected String scriptedFieldPrefix() {
>> + return "_source";
>> + }
>> +
>> @Override protected Enumerable<Object> find(String index, List<String> ops,
>> - List<Map.Entry<String, Class>> fields) {
>> + List<Map.Entry<String, Class>> fields) {
>> final String dbName = index;
>>
>> final String queryString = "{" + Util.toString(ops, "", ", ", "") + "}";
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>> new file mode 100644
>> index 0000000..287e094
>> --- /dev/null
>> +++ b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/ElasticSearch2AdapterTest.java
>> @@ -0,0 +1,395 @@
>> +/*
>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>> + * contributor license agreements. See the NOTICE file distributed with
>> + * this work for additional information regarding copyright ownership.
>> + * The ASF licenses this file to you under the Apache License, Version 2.0
>> + * (the "License"); you may not use this file except in compliance with
>> + * the License. You may obtain a copy of the License at
>> + *
>> + * http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing, software
>> + * distributed under the License is distributed on an "AS IS" BASIS,
>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> + * See the License for the specific language governing permissions and
>> + * limitations under the License.
>> + */
>> +package org.apache.calcite.adapter.elasticsearch2;
>> +
>> +import org.apache.calcite.jdbc.CalciteConnection;
>> +import org.apache.calcite.schema.SchemaPlus;
>> +import org.apache.calcite.schema.impl.ViewTable;
>> +import org.apache.calcite.schema.impl.ViewTableMacro;
>> +import org.apache.calcite.test.CalciteAssert;
>> +import org.apache.calcite.test.ElasticChecker;
>> +
>> +import com.google.common.io.LineProcessor;
>> +import com.google.common.io.Resources;
>> +
>> +import org.elasticsearch.action.bulk.BulkItemResponse;
>> +import org.elasticsearch.action.bulk.BulkRequestBuilder;
>> +import org.elasticsearch.action.bulk.BulkResponse;
>> +import org.elasticsearch.common.xcontent.XContentBuilder;
>> +import org.elasticsearch.common.xcontent.XContentFactory;
>> +
>> +import org.junit.BeforeClass;
>> +import org.junit.ClassRule;
>> +import org.junit.Test;
>> +
>> +import java.io.IOException;
>> +import java.nio.charset.StandardCharsets;
>> +import java.sql.Connection;
>> +import java.sql.DriverManager;
>> +import java.sql.SQLException;
>> +import java.util.Arrays;
>> +import java.util.Collections;
>> +import java.util.Locale;
>> +
>> +/**
>> + * Set of tests for ES adapter. Uses real instance via {@link EmbeddedElasticRule}. Document
>> + * source is local {@code zips-mini.json} file (located in the classpath).
>> + */
>> +public class ElasticSearch2AdapterTest {
>> +
>> + @ClassRule //init once for all tests
>> + public static final EmbeddedElasticRule NODE = EmbeddedElasticRule.create();
>> +
>> + private static final String ZIPS = "zips";
>> +
>> + /**
>> + * Used to create {@code zips} index and insert some data
>> + */
>> + @BeforeClass
>> + public static void setupInstance() throws Exception {
>> + // define mapping so fields are searchable (term query)
>> + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
>> + .startObject("properties")
>> + .startObject("city").field("type", "string")
>> + .field("index", "not_analyzed").endObject()
>> + .startObject("state").field("type", "string")
>> + .field("index", "not_analyzed").endObject()
>> + .startObject("pop").field("type", "long").endObject()
>> + .endObject()
>> + .endObject();
>> +
>> + // create index
>> + NODE.client().admin().indices()
>> + .prepareCreate(ZIPS)
>> + .addMapping(ZIPS, mapping)
>> + .get();
>> +
>> + BulkRequestBuilder bulk = NODE.client().prepareBulk().setRefresh(true);
>> +
>> + // load records from file
>> + Resources.readLines(ElasticSearch2AdapterTest.class.getResource("/zips-mini.json"),
>> + StandardCharsets.UTF_8, new LineProcessor<Void>() {
>> + @Override public boolean processLine(String line) throws IOException {
>> + line = line.replaceAll("_id", "id"); // _id is a reserved attribute in ES
>> + bulk.add(NODE.client().prepareIndex(ZIPS, ZIPS).setSource(line));
>> + return true;
>> + }
>> +
>> + @Override public Void getResult() {
>> + return null;
>> + }
>> + });
>> +
>> + if (bulk.numberOfActions() == 0) {
>> + throw new IllegalStateException("No records to be indexed");
>> + }
>> +
>> + BulkResponse response = bulk.execute().get();
>> +
>> + if (response.hasFailures()) {
>> + throw new IllegalStateException(
>> + String.format(Locale.getDefault(), "Failed to populate %s:\n%s", NODE.httpAddress(),
>> + Arrays.stream(response.getItems()).filter(BulkItemResponse::isFailed)
>> + .map(BulkItemResponse::getFailureMessage).findFirst().orElse("<unknown>")));
>> + }
>> +
>> + }
>> +
>> + private CalciteAssert.ConnectionFactory newConnectionFactory() {
>> + return new CalciteAssert.ConnectionFactory() {
>> + @Override public Connection createConnection() throws SQLException {
>> + final Connection connection = DriverManager.getConnection("jdbc:calcite:");
>> + final SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema();
>> +
>> + root.add("elastic", new Elasticsearch2Schema(NODE.client(), ZIPS));
>> +
>> + // add calcite view programmatically
>> + final String viewSql = "select cast(_MAP['city'] AS varchar(20)) AS \"city\", "
>> + + " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n"
>> + + " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n"
>> + + " cast(_MAP['pop'] AS integer) AS \"pop\", "
>> + + " cast(_MAP['state'] AS varchar(2)) AS \"state\", "
>> + + " cast(_MAP['id'] AS varchar(5)) AS \"id\" "
>> + + "from \"elastic\".\"zips\"";
>> +
>> + ViewTableMacro macro = ViewTable.viewMacro(root, viewSql,
>> + Collections.singletonList("elastic"), Arrays.asList("elastic", "view"), false);
>> + root.add("ZIPS", macro);
>> +
>> + return connection;
>> + }
>> + };
>> + }
>> +
>> + private CalciteAssert.AssertThat calciteAssert() {
>> + return CalciteAssert.that()
>> + .with(newConnectionFactory());
>> + }
>> +
>> + /**
>> + * Tests using calcite view
>> + */
>> + @Test
>> + public void view() throws Exception {
>> + calciteAssert()
>> + .query("select * from zips where \"city\" = 'BROOKLYN'")
>> + .returns("city=BROOKLYN; longitude=-73.956985; latitude=40.646694; "
>> + + "pop=111396; state=NY; id=11226\n")
>> + .returnsCount(1);
>> + }
>> +
>> + @Test
>> + public void emptyResult() {
>> + CalciteAssert.that()
>> + .with(newConnectionFactory())
>> + .query("select * from zips limit 0")
>> + .returnsCount(0);
>> +
>> + CalciteAssert.that()
>> + .with(newConnectionFactory())
>> + .query("select * from \"elastic\".\"zips\" where _MAP['Foo'] = '_MISSING_'")
>> + .returnsCount(0);
>> + }
>> +
>> + @Test
>> + public void basic() throws Exception {
>> + CalciteAssert.that()
>> + .with(newConnectionFactory())
>> + .query("select * from \"elastic\".\"zips\" where _MAP['city'] = 'BROOKLYN'")
>> + .returnsCount(1);
>> +
>> + CalciteAssert.that()
>> + .with(newConnectionFactory())
>> + .query("select * from \"elastic\".\"zips\" where"
>> + + " _MAP['city'] in ('BROOKLYN', 'WASHINGTON')")
>> + .returnsCount(2);
>> +
>> + // lower-case
>> + CalciteAssert.that()
>> + .with(newConnectionFactory())
>> + .query("select * from \"elastic\".\"zips\" where "
>> + + "_MAP['city'] in ('brooklyn', 'Brooklyn', 'BROOK') ")
>> + .returnsCount(0);
>> +
>> + // missing field
>> + CalciteAssert.that()
>> + .with(newConnectionFactory())
>> + .query("select * from \"elastic\".\"zips\" where _MAP['CITY'] = 'BROOKLYN'")
>> + .returnsCount(0);
>> +
>> + // limit works
>> + CalciteAssert.that()
>> + .with(newConnectionFactory())
>> + .query("select * from \"elastic\".\"zips\" limit 42")
>> + .returnsCount(42);
>> +
>> + }
>> +
>> + @Test public void testSort() {
>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>> + + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
>> + + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
>> +
>> + calciteAssert()
>> + .query("select * from zips order by \"state\"")
>> + .returnsCount(10)
>> + .explainContains(explain);
>> + }
>> +
>> + @Test public void testSortLimit() {
>> + final String sql = "select \"state\", \"id\" from zips\n"
>> + + "order by \"state\", \"id\" offset 2 rows fetch next 3 rows only";
>> + calciteAssert()
>> + .query(sql)
>> + .returnsUnordered("state=AK; id=99801",
>> + "state=AL; id=35215",
>> + "state=AL; id=35401")
>> + .queryContains(
>> + ElasticChecker.elasticsearchChecker(
>> + "\"_source\" : [\"state\", \"id\"]",
>> + "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]",
>> + "\"from\": 2",
>> + "\"size\": 3"));
>> + }
>> +
>> +
>> +
>> + @Test public void testOffsetLimit() {
>> + final String sql = "select \"state\", \"id\" from zips\n"
>> + + "offset 2 fetch next 3 rows only";
>> + calciteAssert()
>> + .query(sql)
>> + .runs()
>> + .queryContains(
>> + ElasticChecker.elasticsearchChecker(
>> + "\"from\": 2",
>> + "\"size\": 3",
>> + "\"_source\" : [\"state\", \"id\"]"));
>> + }
>> +
>> + @Test public void testLimit() {
>> + final String sql = "select \"state\", \"id\" from zips\n"
>> + + "fetch next 3 rows only";
>> +
>> + calciteAssert()
>> + .query(sql)
>> + .runs()
>> + .queryContains(
>> + ElasticChecker.elasticsearchChecker(
>> + "\"size\": 3",
>> + "\"_source\" : [\"state\", \"id\"]"));
>> + }
>> +
>> + @Test public void testFilterSort() {
>> + final String sql = "select * from zips\n"
>> + + "where \"state\" = 'CA' and \"id\" >= '70000'\n"
>> + + "order by \"state\", \"id\"";
>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>> + + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC], dir1=[ASC])\n"
>> + + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>> + + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA'), >=(CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", '70000'))])\n"
>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
>> + calciteAssert()
>> + .query(sql)
>> + .returnsOrdered("city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; "
>> + + "pop=96074; state=CA; id=90011",
>> + "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; "
>> + + "pop=99568; state=CA; id=90201",
>> + "city=NORWALK; longitude=-118.081767; latitude=33.90564; "
>> + + "pop=94188; state=CA; id=90650")
>> + .queryContains(
>> + ElasticChecker.elasticsearchChecker("\"query\" : "
>> + + "{\"constant_score\":{\"filter\":{\"bool\":"
>> + + "{\"must\":[{\"term\":{\"state\":\"CA\"}},"
>> + + "{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
>> + "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>> + + "\"city\":{\"script\": \"_source.city\"}, "
>> + + "\"pop\":{\"script\": \"_source.pop\"}, "
>> + + "\"state\":{\"script\": \"_source.state\"}, "
>> + + "\"id\":{\"script\": \"_source.id\"}}",
>> + "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]"))
>> + .explainContains(explain);
>> + }
>> +
>> + @Test public void testFilterSortDesc() {
>> + final String sql = "select * from zips\n"
>> + + "where \"pop\" BETWEEN 95000 AND 100000\n"
>> + + "order by \"state\" desc, \"pop\"";
>> + calciteAssert()
>> + .query(sql)
>> + .limit(4)
>> + .returnsOrdered(
>> + "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011",
>> + "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201");
>> + }
>> +
>> + @Test public void testFilterRedundant() {
>> + final String sql = "select * from zips\n"
>> + + "where \"state\" > 'CA' and \"state\" < 'AZ' and \"state\" = 'OK'";
>> + calciteAssert()
>> + .query(sql)
>> + .runs()
>> + .queryContains(
>> + ElasticChecker.elasticsearchChecker(""
>> + + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":"
>> + + "{\"must\":[{\"term\":{\"state\":\"OK\"}}]}}}}",
>> + "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>> + + "\"city\":{\"script\": \"_source.city\"}, "
>> + + "\"pop\":{\"script\": \"_source.pop\"}, \"state\":{\"script\": \"_source.state\"}, "
>> + + "\"id\":{\"script\": \"_source.id\"}}"
>> + ));
>> + }
>> +
>> + @Test public void testInPlan() {
>> + final String[] searches = {
>> + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
>> + + "[{\"bool\":{\"must\":[{\"term\":{\"pop\":96074}}]}},{\"bool\":{\"must\":[{\"term\":"
>> + + "{\"pop\":99568}}]}}]}}}}",
>> + "\"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, "
>> + + "\"latitude\":{\"script\":\"_source.loc[1]\"}, "
>> + + "\"city\":{\"script\": \"_source.city\"}, "
>> + + "\"pop\":{\"script\": \"_source.pop\"}, "
>> + + "\"state\":{\"script\": \"_source.state\"}, "
>> + + "\"id\":{\"script\": \"_source.id\"}}"
>> + };
>> +
>> + calciteAssert()
>> + .query("select * from zips where \"pop\" in (96074, 99568)")
>> + .returnsUnordered(
>> + "city=BELL GARDENS; longitude=-118.17205; latitude=33.969177; pop=99568; state=CA; id=90201",
>> + "city=LOS ANGELES; longitude=-118.258189; latitude=34.007856; pop=96074; state=CA; id=90011"
>> + )
>> + .queryContains(ElasticChecker.elasticsearchChecker(searches));
>> + }
>> +
>> + @Test public void testZips() {
>> + calciteAssert()
>> + .query("select \"state\", \"city\" from zips")
>> + .returnsCount(10);
>> + }
>> +
>> + @Test public void testProject() {
>> + final String sql = "select \"state\", \"city\", 0 as \"zero\"\n"
>> + + "from zips\n"
>> + + "order by \"state\", \"city\"";
>> +
>> + calciteAssert()
>> + .query(sql)
>> + .limit(2)
>> + .returnsUnordered("state=AK; city=ANCHORAGE; zero=0",
>> + "state=AK; city=FAIRBANKS; zero=0")
>> + .queryContains(
>> + ElasticChecker.elasticsearchChecker("\"script_fields\": "
>> + + "{\"zero\":{\"script\": \"0\"}, "
>> + + "\"state\":{\"script\": \"_source.state\"}, "
>> + + "\"city\":{\"script\": \"_source.city\"}}",
>> + "\"sort\": [ {\"state\": \"asc\"}, {\"city\": \"asc\"}]"));
>> + }
>> +
>> + @Test public void testFilter() {
>> + final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>> + + " ElasticsearchProject(state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>> + + " ElasticsearchFilter(condition=[=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA')])\n"
>> + + " ElasticsearchTableScan(table=[[elastic, zips]])";
>> + calciteAssert()
>> + .query("select \"state\", \"city\" from zips where \"state\" = 'CA'")
>> + .limit(3)
>> + .returnsUnordered("state=CA; city=BELL GARDENS",
>> + "state=CA; city=LOS ANGELES",
>> + "state=CA; city=NORWALK")
>> + .explainContains(explain);
>> + }
>> +
>> + @Test public void testFilterReversed() {
>> + calciteAssert()
>> + .query("select \"state\", \"city\" from zips where 'WI' < \"state\" order by \"city\"")
>> + .limit(2)
>> + .returnsUnordered("state=WV; city=BECKLEY",
>> + "state=WY; city=CHEYENNE");
>> + calciteAssert()
>> + .query("select \"state\", \"city\" from zips where \"state\" > 'WI' order by \"city\"")
>> + .limit(2)
>> + .returnsUnordered("state=WV; city=BECKLEY",
>> + "state=WY; city=CHEYENNE");
>> + }
>> +
>> +}
>> +
>> +// End ElasticSearch2AdapterTest.java
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>> new file mode 100644
>> index 0000000..4474add
>> --- /dev/null
>> +++ b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticNode.java
>> @@ -0,0 +1,147 @@
>> +/*
>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>> + * contributor license agreements. See the NOTICE file distributed with
>> + * this work for additional information regarding copyright ownership.
>> + * The ASF licenses this file to you under the Apache License, Version 2.0
>> + * (the "License"); you may not use this file except in compliance with
>> + * the License. You may obtain a copy of the License at
>> + *
>> + * http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing, software
>> + * distributed under the License is distributed on an "AS IS" BASIS,
>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> + * See the License for the specific language governing permissions and
>> + * limitations under the License.
>> + */
>> +package org.apache.calcite.adapter.elasticsearch2;
>> +
>> +import com.google.common.base.Preconditions;
>> +import com.google.common.io.Files;
>> +
>> +import org.elasticsearch.Version;
>> +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
>> +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
>> +import org.elasticsearch.client.Client;
>> +import org.elasticsearch.common.settings.Settings;
>> +import org.elasticsearch.common.transport.TransportAddress;
>> +import org.elasticsearch.node.Node;
>> +import org.elasticsearch.node.internal.InternalSettingsPreparer;
>> +import org.elasticsearch.plugins.Plugin;
>> +import org.elasticsearch.script.groovy.GroovyPlugin;
>> +
>> +import java.io.File;
>> +import java.util.Arrays;
>> +import java.util.Collection;
>> +import java.util.Collections;
>> +
>> +/**
>> + * Represents a single elastic search node which can run embedded in a java application.
>> + * Intended for unit and integration tests. Settings and plugins are crafted for Calcite.
>> + */
>> +class EmbeddedElasticNode implements AutoCloseable {
>> +
>> + private final LocalNode node;
>> + private volatile boolean isStarted;
>> +
>> + private EmbeddedElasticNode(LocalNode node) {
>> + this.node = Preconditions.checkNotNull(node, "node");
>> + }
>> +
>> + /**
>> + * Having separate class to expose (protected) constructor which allows to install
>> + * different plugins. In our case it is {@code GroovyPlugin} for scripted fields like
>> + * {@code loc[0]} or {@code loc[1]['foo']}.
>> + */
>> + private static class LocalNode extends Node {
>> + private LocalNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
>> + super(InternalSettingsPreparer.prepareEnvironment(settings, null),
>> + Version.CURRENT,
>> + classpathPlugins);
>> + }
>> + }
>> +
>> + /**
>> + * Creates an instance with existing settings
>> + */
>> + private static EmbeddedElasticNode create(Settings settings) {
>> + // ensure GroovyPlugin is installed or otherwise scripted fields would not work
>> + LocalNode node = new LocalNode(settings, Collections.singleton(GroovyPlugin.class));
>> + return new EmbeddedElasticNode(node);
>> + }
>> +
>> + /**
>> + * Creates elastic node as single member of a cluster. Node will not be started
>> + * unless {@link #start()} is explicitly called.
>> + */
>> + public static EmbeddedElasticNode create() {
>> + File data = Files.createTempDir();
>> + data.deleteOnExit();
>> + File home = Files.createTempDir();
>> + home.deleteOnExit();
>> +
>> + Settings settings = Settings.builder()
>> + .put("node.name", "fake-elastic")
>> + .put("path.home", home.getAbsolutePath())
>> + .put("path.data", data.getAbsolutePath())
>> + .put("script.inline", true) // requires GroovyPlugin
>> + .put("script.indexed", true) // requires GroovyPlugin
>> + .put("cluster.routing.allocation.disk.threshold_enabled", false)
>> + .put("node.local", true)
>> + .put("node.data", true)
>> + .put("network.host", "localhost")
>> + .build();
>> +
>> + return create(settings);
>> + }
>> +
>> + /**
>> + * Starts current node
>> + */
>> + public void start() {
>> + Preconditions.checkState(!isStarted, "already started");
>> + node.start();
>> + this.isStarted = true;
>> + }
>> +
>> + /**
>> + * Returns current address to connect to with HTTP client.
>> + */
>> + public TransportAddress httpAddress() {
>> + Preconditions.checkState(isStarted, "node is not started");
>> +
>> + NodesInfoResponse response = client().admin().cluster().prepareNodesInfo()
>> + .execute().actionGet();
>> + if (response.getNodes().length != 1) {
>> + throw new IllegalStateException("Expected single node but got "
>> + + response.getNodes().length);
>> + }
>> + NodeInfo node = response.getNodes()[0];
>> + return node.getHttp().address().boundAddresses()[0];
>> + }
>> +
>> + /**
>> + * Exposes elastic
>> + * <a href="https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html">transport client</a>
>> + *
>> + * (use of HTTP client is preferred).
>> + */
>> + public Client client() {
>> + Preconditions.checkState(isStarted, "node is not started");
>> + return node.client();
>> + }
>> +
>> + @Override public void close() throws Exception {
>> + node.close();
>> + // cleanup data dirs
>> + File data = new File(node.settings().get("path.data"));
>> + File home = new File(node.settings().get("path.home"));
>> + for (File file: Arrays.asList(data, home)) {
>> + if (file.exists()) {
>> + file.delete();
>> + }
>> + }
>> + }
>> +}
>> +
>> +// End EmbeddedElasticNode.java
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>> new file mode 100644
>> index 0000000..a633078
>> --- /dev/null
>> +++ b/elasticsearch2/src/test/java/org/apache/calcite/adapter/elasticsearch2/EmbeddedElasticRule.java
>> @@ -0,0 +1,97 @@
>> +/*
>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>> + * contributor license agreements. See the NOTICE file distributed with
>> + * this work for additional information regarding copyright ownership.
>> + * The ASF licenses this file to you under the Apache License, Version 2.0
>> + * (the "License"); you may not use this file except in compliance with
>> + * the License. You may obtain a copy of the License at
>> + *
>> + * http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing, software
>> + * distributed under the License is distributed on an "AS IS" BASIS,
>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> + * See the License for the specific language governing permissions and
>> + * limitations under the License.
>> + */
>> +package org.apache.calcite.adapter.elasticsearch2;
>> +
>> +import com.google.common.base.Preconditions;
>> +
>> +import org.elasticsearch.client.Client;
>> +import org.elasticsearch.common.transport.TransportAddress;
>> +import org.junit.rules.ExternalResource;
>> +
>> +/**
>> + * Used to initialize a single elastic node. For performance reasons (node startup costs),
>> + * same instance is usually shared across multiple tests.
>> + *
>> + * This rule should be used as follows:
>> + * <pre>
>> + * {@code
>> + *
>> + * public class MyTest {
>> + * @literal @ClassRule
>> + * public static final ElasticSearchRule RULE = ElasticSearchRule.create();
>> + *
>> + * @literal @BeforeClass
>> + * public void setup() {
>> + * // ... populate instance
>> + * }
>> + *
>> + * @literal @Test
>> + * public void myTest() {
>> + * TransportAddress address = RULE.httpAddress();
>> + * // ....
>> + * }
>> + * }
>> + * }
>> + * </pre>
>> + *
>> + * @see ExternalResource
>> + */
>> +class EmbeddedElasticRule extends ExternalResource {
>> +
>> + private final EmbeddedElasticNode node;
>> +
>> + private EmbeddedElasticRule(EmbeddedElasticNode resource) {
>> + this.node = Preconditions.checkNotNull(resource, "resource");
>> + }
>> +
>> + @Override protected void before() throws Throwable {
>> + node.start();
>> + }
>> +
>> + @Override protected void after() {
>> + try {
>> + node.close();
>> + } catch (Exception e) {
>> + throw new RuntimeException(e);
>> + }
>> + }
>> +
>> + /**
>> + * Factory method to create this rule.
>> + */
>> + public static EmbeddedElasticRule create() {
>> + return new EmbeddedElasticRule(EmbeddedElasticNode.create());
>> + }
>> +
>> + /**
>> + * Exposes current ES transport client.
>> + */
>> + Client client() {
>> + return node.client();
>> + }
>> +
>> + /**
>> + * HTTP address for rest clients (can be ES native or any other).
>> + */
>> + TransportAddress httpAddress() {
>> + return node.httpAddress();
>> + }
>> +
>> +
>> +}
>> +
>> +// End EmbeddedElasticRule.java
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>> new file mode 100644
>> index 0000000..21fc491
>> --- /dev/null
>> +++ b/elasticsearch2/src/test/java/org/apache/calcite/test/ElasticChecker.java
>> @@ -0,0 +1,49 @@
>> +/*
>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>> + * contributor license agreements. See the NOTICE file distributed with
>> + * this work for additional information regarding copyright ownership.
>> + * The ASF licenses this file to you under the Apache License, Version 2.0
>> + * (the "License"); you may not use this file except in compliance with
>> + * the License. You may obtain a copy of the License at
>> + *
>> + * http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing, software
>> + * distributed under the License is distributed on an "AS IS" BASIS,
>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> + * See the License for the specific language governing permissions and
>> + * limitations under the License.
>> + */
>> +package org.apache.calcite.test;
>> +
>> +import com.google.common.base.Function;
>> +
>> +import java.util.List;
>> +
>> +import javax.annotation.Nullable;
>> +
>> +/**
>> + * Internal util methods for ElasticSearch tests
>> + */
>> +public class ElasticChecker {
>> +
>> + private ElasticChecker() {}
>> +
>> +
>> + /** Returns a function that checks that a particular Elasticsearch pipeline is
>> + * generated to implement a query. */
>> + public static Function<List, Void> elasticsearchChecker(final String... strings) {
>> + return new Function<List, Void>() {
>> + @Nullable
>> + @Override public Void apply(@Nullable List actual) {
>> + Object[] actualArray = actual == null || actual.isEmpty() ? null
>> + : ((List) actual.get(0)).toArray();
>> + CalciteAssert.assertArrayEqual("expected Elasticsearch query not found", strings,
>> + actualArray);
>> + return null;
>> + }
>> + };
>> + }
>> +}
>> +
>> +// End ElasticChecker.java
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java b/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>> deleted file mode 100644
>> index 4e0c2b6..0000000
>> --- a/elasticsearch2/src/test/java/org/apache/calcite/test/Elasticsearch2AdapterIT.java
>> +++ /dev/null
>> @@ -1,270 +0,0 @@
>> -/*
>> - * Licensed to the Apache Software Foundation (ASF) under one or more
>> - * contributor license agreements. See the NOTICE file distributed with
>> - * this work for additional information regarding copyright ownership.
>> - * The ASF licenses this file to you under the Apache License, Version 2.0
>> - * (the "License"); you may not use this file except in compliance with
>> - * the License. You may obtain a copy of the License at
>> - *
>> - * http://www.apache.org/licenses/LICENSE-2.0
>> - *
>> - * Unless required by applicable law or agreed to in writing, software
>> - * distributed under the License is distributed on an "AS IS" BASIS,
>> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> - * See the License for the specific language governing permissions and
>> - * limitations under the License.
>> - */
>> -package org.apache.calcite.test;
>> -
>> -import org.apache.calcite.util.Util;
>> -
>> -import com.google.common.base.Function;
>> -import com.google.common.collect.ImmutableMap;
>> -
>> -import org.junit.Test;
>> -
>> -import java.util.List;
>> -import javax.annotation.Nullable;
>> -
>> -/**
>> - * Tests for the {@code org.apache.calcite.adapter.elasticsearch2} package.
>> - *
>> - * <p>Before calling this test, you need to populate Elasticsearch, as follows:
>> - *
>> - * <blockquote><code>
>> - * git clone https://github.com/vlsi/calcite-test-dataset<br>
>> - * cd calcite-test-dataset<br>
>> - * mvn install
>> - * </code></blockquote>
>> - *
>> - * <p>This will create a virtual machine with Elasticsearch and the "zips" test
>> - * dataset.
>> - */
>> -public class Elasticsearch2AdapterIT {
>> - /**
>> - * Whether to run Elasticsearch tests. Enabled by default, however test is only
>> - * included if "it" profile is activated ({@code -Pit}). To disable,
>> - * specify {@code -Dcalcite.test.elasticsearch=false} on the Java command line.
>> - */
>> - private static final boolean ENABLED = Util.getBooleanProperty("calcite.test.elasticsearch",
>> - true);
>> -
>> - /** Connection factory based on the "zips-es" model. */
>> - private static final ImmutableMap<String, String> ZIPS = ImmutableMap.of("model",
>> - Elasticsearch2AdapterIT.class.getResource("/elasticsearch-zips-model.json").getPath());
>> -
>> - /** Whether to run this test. */
>> - private boolean enabled() {
>> - return ENABLED;
>> - }
>> -
>> - /** Returns a function that checks that a particular Elasticsearch pipeline is
>> - * generated to implement a query. */
>> - private static Function<List, Void> elasticsearchChecker(final String... strings) {
>> - return new Function<List, Void>() {
>> - @Nullable
>> - @Override public Void apply(@Nullable List actual) {
>> - Object[] actualArray = actual == null || actual.isEmpty() ? null
>> - : ((List) actual.get(0)).toArray();
>> - CalciteAssert.assertArrayEqual("expected Elasticsearch query not found", strings,
>> - actualArray);
>> - return null;
>> - }
>> - };
>> - }
>> -
>> - @Test public void testSort() {
>> - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>> - + " ElasticsearchSort(sort0=[$4], dir0=[ASC])\n"
>> - + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>> - + " ElasticsearchTableScan(table=[[elasticsearch_raw, zips]])";
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query("select * from zips order by \"state\"")
>> - .returnsCount(10)
>> - .explainContains(explain);
>> - }
>> -
>> - @Test public void testSortLimit() {
>> - final String sql = "select \"state\", \"id\" from zips\n"
>> - + "order by \"state\", \"id\" offset 2 rows fetch next 3 rows only";
>> - CalciteAssert.that()
>> - .with(ZIPS)
>> - .query(sql)
>> - .returnsUnordered("state=AK; id=99503",
>> - "state=AK; id=99504",
>> - "state=AK; id=99505")
>> - .queryContains(
>> - elasticsearchChecker(
>> - "\"fields\" : [\"state\", \"id\"], \"script_fields\": {}",
>> - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]",
>> - "\"from\": 2",
>> - "\"size\": 3"));
>> - }
>> -
>> - @Test public void testOffsetLimit() {
>> - final String sql = "select \"state\", \"id\" from zips\n"
>> - + "offset 2 fetch next 3 rows only";
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query(sql)
>> - .runs()
>> - .queryContains(
>> - elasticsearchChecker(
>> - "\"from\": 2",
>> - "\"size\": 3",
>> - "\"fields\" : [\"state\", \"id\"], \"script_fields\": {}"));
>> - }
>> -
>> - @Test public void testLimit() {
>> - final String sql = "select \"state\", \"id\" from zips\n"
>> - + "fetch next 3 rows only";
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query(sql)
>> - .runs()
>> - .queryContains(
>> - elasticsearchChecker(
>> - "\"size\": 3",
>> - "\"fields\" : [\"state\", \"id\"], \"script_fields\": {}"));
>> - }
>> -
>> - @Test public void testFilterSort() {
>> - final String sql = "select * from zips\n"
>> - + "where \"city\" = 'SPRINGFIELD' and \"id\" >= '70000'\n"
>> - + "order by \"state\", \"id\"";
>> - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>> - + " ElasticsearchSort(sort0=[$4], sort1=[$5], dir0=[ASC], dir1=[ASC])\n"
>> - + " ElasticsearchProject(city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], longitude=[CAST(ITEM(ITEM($0, 'loc'), 0)):FLOAT], latitude=[CAST(ITEM(ITEM($0, 'loc'), 1)):FLOAT], pop=[CAST(ITEM($0, 'pop')):INTEGER], state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], id=[CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>> - + " ElasticsearchFilter(condition=[AND(=(CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'SPRINGFIELD'), >=(CAST(ITEM($0, 'id')):VARCHAR(5) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", '70000'))])\n"
>> - + " ElasticsearchTableScan(table=[[elasticsearch_raw, zips]])";
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query(sql)
>> - .returnsOrdered(
>> - "city=SPRINGFIELD; longitude=-92.54567; latitude=35.274879; pop=752; state=AR; id=72157",
>> - "city=SPRINGFIELD; longitude=-102.617322; latitude=37.406727; pop=1992; state=CO; id=81073",
>> - "city=SPRINGFIELD; longitude=-90.577479; latitude=30.415738; pop=5597; state=LA; id=70462",
>> - "city=SPRINGFIELD; longitude=-123.015259; latitude=44.06106; pop=32384; state=OR; id=97477",
>> - "city=SPRINGFIELD; longitude=-122.917108; latitude=44.056056; pop=27521; state=OR; id=97478")
>> - .queryContains(
>> - elasticsearchChecker("\"query\" : {\"constant_score\":{\"filter\":{\"bool\":"
>> - + "{\"must\":[{\"term\":{\"city\":\"springfield\"}},{\"range\":{\"id\":{\"gte\":\"70000\"}}}]}}}}",
>> - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"], \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, \"latitude\":{\"script\":\"_source.loc[1]\"}}",
>> - "\"sort\": [ {\"state\": \"asc\"}, {\"id\": \"asc\"}]"))
>> - .explainContains(explain);
>> - }
>> -
>> - @Test public void testFilterSortDesc() {
>> - final String sql = "select * from zips\n"
>> - + "where \"pop\" BETWEEN 20000 AND 20100\n"
>> - + "order by \"state\" desc, \"pop\"";
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query(sql)
>> - .limit(4)
>> - .returnsOrdered(
>> - "city=SHERIDAN; longitude=-106.964795; latitude=44.78486; pop=20025; state=WY; id=82801",
>> - "city=MOUNTLAKE TERRAC; longitude=-122.304036; latitude=47.793061; pop=20059; state=WA; id=98043",
>> - "city=FALMOUTH; longitude=-77.404537; latitude=38.314557; pop=20039; state=VA; id=22405",
>> - "city=FORT WORTH; longitude=-97.318409; latitude=32.725551; pop=20012; state=TX; id=76104");
>> - }
>> -
>> - @Test public void testFilterRedundant() {
>> - final String sql = "select * from zips\n"
>> - + "where \"state\" > 'CA' and \"state\" < 'AZ' and \"state\" = 'OK'";
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query(sql)
>> - .runs()
>> - .queryContains(
>> - elasticsearchChecker(""
>> - + "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":"
>> - + "{\"must\":[{\"term\":{\"state\":\"ok\"}}]}}}}",
>> - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"], \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, \"latitude\":{\"script\":\"_source.loc[1]\"}}"));
>> - }
>> -
>> - @Test public void testInPlan() {
>> - final String[] searches = {
>> - "\"query\" : {\"constant_score\":{\"filter\":{\"bool\":{\"should\":"
>> - + "[{\"bool\":{\"must\":[{\"term\":{\"pop\":20012}}]}},{\"bool\":{\"must\":[{\"term\":"
>> - + "{\"pop\":15590}}]}}]}}}}",
>> - "\"fields\" : [\"city\", \"pop\", \"state\", \"id\"], \"script_fields\": {\"longitude\":{\"script\":\"_source.loc[0]\"}, \"latitude\":{\"script\":\"_source.loc[1]\"}}"
>> - };
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query("select * from zips where \"pop\" in (20012, 15590)")
>> - .returnsUnordered(
>> - "city=COVINA; longitude=-117.884285; latitude=34.08596; pop=15590; state=CA; id=91723",
>> - "city=ARLINGTON; longitude=-97.091987; latitude=32.654752; pop=15590; state=TX; id=76018",
>> - "city=CROFTON; longitude=-76.680166; latitude=39.011163; pop=15590; state=MD; id=21114",
>> - "city=FORT WORTH; longitude=-97.318409; latitude=32.725551; pop=20012; state=TX; id=76104",
>> - "city=DINUBA; longitude=-119.39087; latitude=36.534931; pop=20012; state=CA; id=93618")
>> - .queryContains(elasticsearchChecker(searches));
>> - }
>> -
>> - @Test public void testZips() {
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query("select \"state\", \"city\" from zips")
>> - .returnsCount(10);
>> - }
>> -
>> - @Test public void testProject() {
>> - final String sql = "select \"state\", \"city\", 0 as \"zero\"\n"
>> - + "from zips\n"
>> - + "order by \"state\", \"city\"";
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query(sql)
>> - .limit(2)
>> - .returnsUnordered("state=AK; city=ELMENDORF AFB; zero=0",
>> - "state=AK; city=EIELSON AFB; zero=0")
>> - .queryContains(
>> - elasticsearchChecker("\"sort\": [ {\"state\": \"asc\"}, {\"city\": \"asc\"}]",
>> - "\"fields\" : [\"state\", \"city\"], \"script_fields\": {\"zero\":{\"script\": \"0\"}}"));
>> - }
>> -
>> - @Test public void testFilter() {
>> - final String explain = "PLAN=ElasticsearchToEnumerableConverter\n"
>> - + " ElasticsearchProject(state=[CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"], city=[CAST(ITEM($0, 'city')):VARCHAR(20) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\"])\n"
>> - + " ElasticsearchFilter(condition=[=(CAST(ITEM($0, 'state')):VARCHAR(2) CHARACTER SET \"ISO-8859-1\" COLLATE \"ISO-8859-1$en_US$primary\", 'CA')])\n"
>> - + " ElasticsearchTableScan(table=[[elasticsearch_raw, zips]])";
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query("select \"state\", \"city\" from zips where \"state\" = 'CA'")
>> - .limit(2)
>> - .returnsUnordered("state=CA; city=LOS ANGELES",
>> - "state=CA; city=LOS ANGELES")
>> - .explainContains(explain);
>> - }
>> -
>> - @Test public void testFilterReversed() {
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query("select \"state\", \"city\" from zips where 'WI' < \"state\"")
>> - .limit(2)
>> - .returnsUnordered("state=WV; city=WELCH",
>> - "state=WV; city=HANOVER");
>> - CalciteAssert.that()
>> - .enable(enabled())
>> - .with(ZIPS)
>> - .query("select \"state\", \"city\" from zips where \"state\" > 'WI'")
>> - .limit(2)
>> - .returnsUnordered("state=WV; city=WELCH",
>> - "state=WV; city=HANOVER");
>> - }
>> -}
>> -
>> -// End Elasticsearch2AdapterIT.java
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/resources/elasticsearch-zips-model.json
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/src/test/resources/elasticsearch-zips-model.json b/elasticsearch2/src/test/resources/elasticsearch-zips-model.json
>> deleted file mode 100644
>> index 6c00a9c..0000000
>> --- a/elasticsearch2/src/test/resources/elasticsearch-zips-model.json
>> +++ /dev/null
>> @@ -1,50 +0,0 @@
>> -/*
>> - * Licensed to the Apache Software Foundation (ASF) under one or more
>> - * contributor license agreements. See the NOTICE file distributed with
>> - * this work for additional information regarding copyright ownership.
>> - * The ASF licenses this file to you under the Apache License, Version 2.0
>> - * (the License); you may not use this file except in compliance with
>> - * the License. You may obtain a copy of the License at
>> - *
>> - * http://www.apache.org/licenses/LICENSE-2.0
>> - *
>> - * Unless required by applicable law or agreed to in writing, software
>> - * distributed under the License is distributed on an AS IS BASIS,
>> - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> - * See the License for the specific language governing permissions and
>> - * limitations under the License.
>> - */
>> -{
>> - "version": "1.0",
>> - "defaultSchema": "elasticsearch",
>> - "schemas": [
>> - {
>> - "type": "custom",
>> - "name": "elasticsearch_raw",
>> - "factory": "org.apache.calcite.adapter.elasticsearch2.Elasticsearch2SchemaFactory",
>> - "operand": {
>> - "coordinates": "{'127.0.0.1': 9300}",
>> - "userConfig": "{'bulk.flush.max.actions': 10, 'bulk.flush.max.size.mb': 1}",
>> - "index": "usa"
>> - }
>> - },
>> - {
>> - "name": "elasticsearch",
>> - "tables": [
>> - {
>> - "name": "ZIPS",
>> - "type": "view",
>> - "sql": [
>> - "select cast(_MAP['city'] AS varchar(20)) AS \"city\",\n",
>> - " cast(_MAP['loc'][0] AS float) AS \"longitude\",\n",
>> - " cast(_MAP['loc'][1] AS float) AS \"latitude\",\n",
>> - " cast(_MAP['pop'] AS integer) AS \"pop\",\n",
>> - " cast(_MAP['state'] AS varchar(2)) AS \"state\",\n",
>> - " cast(_MAP['id'] AS varchar(5)) AS \"id\"\n",
>> - "from \"elasticsearch_raw\".\"zips\""
>> - ]
>> - }
>> - ]
>> - }
>> - ]
>> -}
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch2/src/test/resources/zips-mini.json
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch2/src/test/resources/zips-mini.json b/elasticsearch2/src/test/resources/zips-mini.json
>> new file mode 100644
>> index 0000000..d70eadc
>> --- /dev/null
>> +++ b/elasticsearch2/src/test/resources/zips-mini.json
>> @@ -0,0 +1,149 @@
>> +{ "_id" : "01701", "city" : "FRAMINGHAM", "loc" : [ -71.42548600000001, 42.300665 ], "pop" : 65046, "state" : "MA" }
>> +{ "_id" : "02154", "city" : "NORTH WALTHAM", "loc" : [ -71.236497, 42.382492 ], "pop" : 57871, "state" : "MA" }
>> +{ "_id" : "02401", "city" : "BROCKTON", "loc" : [ -71.03434799999999, 42.081571 ], "pop" : 59498, "state" : "MA" }
>> +{ "_id" : "02840", "city" : "MIDDLETOWN", "loc" : [ -71.30347999999999, 41.504502 ], "pop" : 47687, "state" : "RI" }
>> +{ "_id" : "02860", "city" : "PAWTUCKET", "loc" : [ -71.39071300000001, 41.872873 ], "pop" : 45442, "state" : "RI" }
>> +{ "_id" : "02895", "city" : "NORTH SMITHFIELD", "loc" : [ -71.513683, 41.99948 ], "pop" : 53733, "state" : "RI" }
>> +{ "_id" : "03060", "city" : "NASHUA", "loc" : [ -71.466684, 42.756395 ], "pop" : 41438, "state" : "NH" }
>> +{ "_id" : "03103", "city" : "MANCHESTER", "loc" : [ -71.449325, 42.965563 ], "pop" : 36613, "state" : "NH" }
>> +{ "_id" : "03301", "city" : "CONCORD", "loc" : [ -71.527734, 43.218525 ], "pop" : 34035, "state" : "NH" }
>> +{ "_id" : "04240", "city" : "LEWISTON", "loc" : [ -70.191619, 44.098538 ], "pop" : 40173, "state" : "ME" }
>> +{ "_id" : "04401", "city" : "BANGOR", "loc" : [ -68.791839, 44.824199 ], "pop" : 40434, "state" : "ME" }
>> +{ "_id" : "05301", "city" : "BRATTLEBORO", "loc" : [ -72.593322, 42.857353 ], "pop" : 17522, "state" : "VT" }
>> +{ "_id" : "05401", "city" : "BURLINGTON", "loc" : [ -73.219875, 44.484023 ], "pop" : 39127, "state" : "VT" }
>> +{ "_id" : "05701", "city" : "RUTLAND", "loc" : [ -72.97077299999999, 43.614131 ], "pop" : 22576, "state" : "VT" }
>> +{ "_id" : "06010", "city" : "BRISTOL", "loc" : [ -72.930193, 41.682293 ], "pop" : 60670, "state" : "CT" }
>> +{ "_id" : "06450", "city" : "MERIDEN", "loc" : [ -72.799734, 41.533396 ], "pop" : 59441, "state" : "CT" }
>> +{ "_id" : "06902", "city" : "STAMFORD", "loc" : [ -73.53742800000001, 41.052552 ], "pop" : 54605, "state" : "CT" }
>> +{ "_id" : "07002", "city" : "BAYONNE", "loc" : [ -74.119169, 40.666399 ], "pop" : 61444, "state" : "NJ" }
>> +{ "_id" : "07087", "city" : "WEEHAWKEN", "loc" : [ -74.030558, 40.768153 ], "pop" : 69646, "state" : "NJ" }
>> +{ "_id" : "07111", "city" : "IRVINGTON", "loc" : [ -74.23127100000001, 40.7261 ], "pop" : 60986, "state" : "NJ" }
>> +{ "_id" : "10021", "city" : "NEW YORK", "loc" : [ -73.958805, 40.768476 ], "pop" : 106564, "state" : "NY" }
>> +{ "_id" : "11226", "city" : "BROOKLYN", "loc" : [ -73.956985, 40.646694 ], "pop" : 111396, "state" : "NY" }
>> +{ "_id" : "11373", "city" : "JACKSON HEIGHTS", "loc" : [ -73.878551, 40.740388 ], "pop" : 88241, "state" : "NY" }
>> +{ "_id" : "17042", "city" : "CLEONA", "loc" : [ -76.425895, 40.335912 ], "pop" : 61993, "state" : "PA" }
>> +{ "_id" : "18042", "city" : "FORKS TOWNSHIP", "loc" : [ -75.23582, 40.6867 ], "pop" : 65784, "state" : "PA" }
>> +{ "_id" : "19143", "city" : "PHILADELPHIA", "loc" : [ -75.228819, 39.944815 ], "pop" : 80454, "state" : "PA" }
>> +{ "_id" : "19711", "city" : "NEWARK", "loc" : [ -75.737534, 39.701129 ], "pop" : 50573, "state" : "DE" }
>> +{ "_id" : "19720", "city" : "MANOR", "loc" : [ -75.589938, 39.67703 ], "pop" : 46906, "state" : "DE" }
>> +{ "_id" : "19901", "city" : "DOVER", "loc" : [ -75.535983, 39.156639 ], "pop" : 46005, "state" : "DE" }
>> +{ "_id" : "20011", "city" : "WASHINGTON", "loc" : [ -77.020251, 38.951786 ], "pop" : 62924, "state" : "DC" }
>> +{ "_id" : "20301", "city" : "PENTAGON", "loc" : [ -77.038196, 38.891019 ], "pop" : 21, "state" : "DC" }
>> +{ "_id" : "21061", "city" : "GLEN BURNIE", "loc" : [ -76.61886199999999, 39.158968 ], "pop" : 75692, "state" : "MD" }
>> +{ "_id" : "21207", "city" : "GWYNN OAK", "loc" : [ -76.734064, 39.329628 ], "pop" : 76002, "state" : "MD" }
>> +{ "_id" : "21215", "city" : "BALTIMORE", "loc" : [ -76.67939699999999, 39.344572 ], "pop" : 74402, "state" : "MD" }
>> +{ "_id" : "22901", "city" : "CHARLOTTESVILLE", "loc" : [ -78.490869, 38.054752 ], "pop" : 62708, "state" : "VA" }
>> +{ "_id" : "23464", "city" : "VIRGINIA BEACH", "loc" : [ -76.175909, 36.797772 ], "pop" : 67276, "state" : "VA" }
>> +{ "_id" : "23602", "city" : "NEWPORT NEWS", "loc" : [ -76.53212499999999, 37.131684 ], "pop" : 68525, "state" : "VA" }
>> +{ "_id" : "25801", "city" : "BECKLEY", "loc" : [ -81.206084, 37.793214 ], "pop" : 45196, "state" : "WV" }
>> +{ "_id" : "26003", "city" : "ELM GROVE", "loc" : [ -80.685126, 40.072736 ], "pop" : 49136, "state" : "WV" }
>> +{ "_id" : "26505", "city" : "STAR CITY", "loc" : [ -79.95422499999999, 39.633858 ], "pop" : 70185, "state" : "WV" }
>> +{ "_id" : "27292", "city" : "LEXINGTON", "loc" : [ -80.262049, 35.82306 ], "pop" : 69179, "state" : "NC" }
>> +{ "_id" : "28677", "city" : "STATESVILLE", "loc" : [ -80.894009, 35.799022 ], "pop" : 52895, "state" : "NC" }
>> +{ "_id" : "29150", "city" : "OSWEGO", "loc" : [ -80.32100800000001, 33.928199 ], "pop" : 46394, "state" : "SC" }
>> +{ "_id" : "29501", "city" : "FLORENCE", "loc" : [ -79.772786, 34.18375 ], "pop" : 66990, "state" : "SC" }
>> +{ "_id" : "29801", "city" : "AIKEN", "loc" : [ -81.71942900000001, 33.553024 ], "pop" : 51233, "state" : "SC" }
>> +{ "_id" : "30032", "city" : "DECATUR", "loc" : [ -84.263165, 33.740825 ], "pop" : 56056, "state" : "GA" }
>> +{ "_id" : "30906", "city" : "PEACH ORCHARD", "loc" : [ -82.038358, 33.402024 ], "pop" : 58646, "state" : "GA" }
>> +{ "_id" : "32216", "city" : "JACKSONVILLE", "loc" : [ -81.547387, 30.293907 ], "pop" : 58867, "state" : "FL" }
>> +{ "_id" : "33012", "city" : "HIALEAH", "loc" : [ -80.30589999999999, 25.865395 ], "pop" : 73194, "state" : "FL" }
>> +{ "_id" : "33311", "city" : "FORT LAUDERDALE", "loc" : [ -80.172786, 26.142104 ], "pop" : 65378, "state" : "FL" }
>> +{ "_id" : "35215", "city" : "CENTER POINT", "loc" : [ -86.693197, 33.635447 ], "pop" : 43862, "state" : "AL" }
>> +{ "_id" : "35401", "city" : "TUSCALOOSA", "loc" : [ -87.56266599999999, 33.196891 ], "pop" : 42124, "state" : "AL" }
>> +{ "_id" : "35901", "city" : "SOUTHSIDE", "loc" : [ -86.010279, 33.997248 ], "pop" : 44165, "state" : "AL" }
>> +{ "_id" : "37042", "city" : "CLARKSVILLE", "loc" : [ -87.418621, 36.585315 ], "pop" : 43296, "state" : "TN" }
>> +{ "_id" : "37211", "city" : "NASHVILLE", "loc" : [ -86.72403799999999, 36.072486 ], "pop" : 51478, "state" : "TN" }
>> +{ "_id" : "38109", "city" : "MEMPHIS", "loc" : [ -90.073238, 35.042538 ], "pop" : 60508, "state" : "TN" }
>> +{ "_id" : "39180", "city" : "VICKSBURG", "loc" : [ -90.85065, 32.325824 ], "pop" : 46968, "state" : "MS" }
>> +{ "_id" : "39401", "city" : "HATTIESBURG", "loc" : [ -89.306471, 31.314553 ], "pop" : 41866, "state" : "MS" }
>> +{ "_id" : "39440", "city" : "LAUREL", "loc" : [ -89.13115500000001, 31.705444 ], "pop" : 45040, "state" : "MS" }
>> +{ "_id" : "40214", "city" : "LOUISVILLE", "loc" : [ -85.77802699999999, 38.159318 ], "pop" : 42198, "state" : "KY" }
>> +{ "_id" : "40216", "city" : "SHIVELY", "loc" : [ -85.831771, 38.186138 ], "pop" : 41719, "state" : "KY" }
>> +{ "_id" : "40601", "city" : "HATTON", "loc" : [ -84.88061, 38.192831 ], "pop" : 46563, "state" : "KY" }
>> +{ "_id" : "44035", "city" : "ELYRIA", "loc" : [ -82.10508799999999, 41.372353 ], "pop" : 66674, "state" : "OH" }
>> +{ "_id" : "44060", "city" : "MENTOR", "loc" : [ -81.342133, 41.689468 ], "pop" : 60109, "state" : "OH" }
>> +{ "_id" : "44107", "city" : "EDGEWATER", "loc" : [ -81.79714300000001, 41.482654 ], "pop" : 59702, "state" : "OH" }
>> +{ "_id" : "46360", "city" : "MICHIGAN CITY", "loc" : [ -86.869899, 41.698031 ], "pop" : 55392, "state" : "IN" }
>> +{ "_id" : "47130", "city" : "JEFFERSONVILLE", "loc" : [ -85.735885, 38.307767 ], "pop" : 56543, "state" : "IN" }
>> +{ "_id" : "47906", "city" : "WEST LAFAYETTE", "loc" : [ -86.923661, 40.444025 ], "pop" : 54702, "state" : "IN" }
>> +{ "_id" : "48180", "city" : "TAYLOR", "loc" : [ -83.267269, 42.231738 ], "pop" : 70811, "state" : "MI" }
>> +{ "_id" : "48185", "city" : "WESTLAND", "loc" : [ -83.374908, 42.318882 ], "pop" : 84712, "state" : "MI" }
>> +{ "_id" : "48227", "city" : "DETROIT", "loc" : [ -83.193732, 42.388303 ], "pop" : 68390, "state" : "MI" }
>> +{ "_id" : "50010", "city" : "AMES", "loc" : [ -93.639398, 42.029859 ], "pop" : 52105, "state" : "IA" }
>> +{ "_id" : "50317", "city" : "PLEASANT HILL", "loc" : [ -93.549446, 41.612499 ], "pop" : 39883, "state" : "IA" }
>> +{ "_id" : "52001", "city" : "DUBUQUE", "loc" : [ -90.68191400000001, 42.514977 ], "pop" : 41934, "state" : "IA" }
>> +{ "_id" : "53209", "city" : "MILWAUKEE", "loc" : [ -87.947834, 43.118765 ], "pop" : 51008, "state" : "WI" }
>> +{ "_id" : "54401", "city" : "WAUSAU", "loc" : [ -89.633955, 44.963433 ], "pop" : 51083, "state" : "WI" }
>> +{ "_id" : "54901", "city" : "OSHKOSH", "loc" : [ -88.54363499999999, 44.021962 ], "pop" : 57187, "state" : "WI" }
>> +{ "_id" : "55106", "city" : "SAINT PAUL", "loc" : [ -93.048817, 44.968384 ], "pop" : 47905, "state" : "MN" }
>> +{ "_id" : "55112", "city" : "NEW BRIGHTON", "loc" : [ -93.199691, 45.074129 ], "pop" : 44128, "state" : "MN" }
>> +{ "_id" : "55337", "city" : "BURNSVILLE", "loc" : [ -93.275283, 44.76086 ], "pop" : 51421, "state" : "MN" }
>> +{ "_id" : "57103", "city" : "SIOUX FALLS", "loc" : [ -96.686415, 43.537386 ], "pop" : 32508, "state" : "SD" }
>> +{ "_id" : "57401", "city" : "ABERDEEN", "loc" : [ -98.485642, 45.466109 ], "pop" : 28786, "state" : "SD" }
>> +{ "_id" : "57701", "city" : "ROCKERVILLE", "loc" : [ -103.200259, 44.077041 ], "pop" : 45328, "state" : "SD" }
>> +{ "_id" : "58103", "city" : "FARGO", "loc" : [ -96.812252, 46.856406 ], "pop" : 38483, "state" : "ND" }
>> +{ "_id" : "58501", "city" : "BISMARCK", "loc" : [ -100.774755, 46.823448 ], "pop" : 36602, "state" : "ND" }
>> +{ "_id" : "58701", "city" : "MINOT", "loc" : [ -101.298476, 48.22914 ], "pop" : 42195, "state" : "ND" }
>> +{ "_id" : "59102", "city" : "BILLINGS", "loc" : [ -108.572662, 45.781265 ], "pop" : 40121, "state" : "MT" }
>> +{ "_id" : "59601", "city" : "HELENA", "loc" : [ -112.021283, 46.613066 ], "pop" : 40102, "state" : "MT" }
>> +{ "_id" : "59801", "city" : "MISSOULA", "loc" : [ -114.025207, 46.856274 ], "pop" : 33811, "state" : "MT" }
>> +{ "_id" : "60623", "city" : "CHICAGO", "loc" : [ -87.7157, 41.849015 ], "pop" : 112047, "state" : "IL" }
>> +{ "_id" : "60634", "city" : "NORRIDGE", "loc" : [ -87.796054, 41.945213 ], "pop" : 69160, "state" : "IL" }
>> +{ "_id" : "60650", "city" : "CICERO", "loc" : [ -87.76008, 41.84776 ], "pop" : 67670, "state" : "IL" }
>> +{ "_id" : "63031", "city" : "FLORISSANT", "loc" : [ -90.340097, 38.806865 ], "pop" : 52659, "state" : "MO" }
>> +{ "_id" : "63116", "city" : "SAINT LOUIS", "loc" : [ -90.26254299999999, 38.581356 ], "pop" : 49014, "state" : "MO" }
>> +{ "_id" : "63136", "city" : "JENNINGS", "loc" : [ -90.260189, 38.738878 ], "pop" : 54994, "state" : "MO" }
>> +{ "_id" : "66502", "city" : "MANHATTAN", "loc" : [ -96.585776, 39.193757 ], "pop" : 50178, "state" : "KS" }
>> +{ "_id" : "67212", "city" : "WICHITA", "loc" : [ -97.438344, 37.700683 ], "pop" : 41349, "state" : "KS" }
>> +{ "_id" : "67401", "city" : "BAVARIA", "loc" : [ -97.60878700000001, 38.823802 ], "pop" : 45208, "state" : "KS" }
>> +{ "_id" : "68104", "city" : "OMAHA", "loc" : [ -95.999888, 41.29186 ], "pop" : 35325, "state" : "NE" }
>> +{ "_id" : "68502", "city" : "LINCOLN", "loc" : [ -96.693763, 40.789282 ], "pop" : 27576, "state" : "NE" }
>> +{ "_id" : "68847", "city" : "KEARNEY", "loc" : [ -99.077883, 40.713608 ], "pop" : 28674, "state" : "NE" }
>> +{ "_id" : "70072", "city" : "MARRERO", "loc" : [ -90.110462, 29.859756 ], "pop" : 58905, "state" : "LA" }
>> +{ "_id" : "70117", "city" : "NEW ORLEANS", "loc" : [ -90.03124, 29.970298 ], "pop" : 56494, "state" : "LA" }
>> +{ "_id" : "70560", "city" : "NEW IBERIA", "loc" : [ -91.819959, 30.001027 ], "pop" : 56105, "state" : "LA" }
>> +{ "_id" : "72032", "city" : "CONWAY", "loc" : [ -92.423574, 35.084199 ], "pop" : 43236, "state" : "AR" }
>> +{ "_id" : "72076", "city" : "GRAVEL RIDGE", "loc" : [ -92.13043500000001, 34.881985 ], "pop" : 37428, "state" : "AR" }
>> +{ "_id" : "72401", "city" : "JONESBORO", "loc" : [ -90.69652600000001, 35.833016 ], "pop" : 53532, "state" : "AR" }
>> +{ "_id" : "73034", "city" : "EDMOND", "loc" : [ -97.47983499999999, 35.666483 ], "pop" : 43814, "state" : "OK" }
>> +{ "_id" : "73505", "city" : "LAWTON", "loc" : [ -98.455234, 34.617939 ], "pop" : 45542, "state" : "OK" }
>> +{ "_id" : "74801", "city" : "SHAWNEE", "loc" : [ -96.931321, 35.34907 ], "pop" : 40076, "state" : "OK" }
>> +{ "_id" : "78207", "city" : "SAN ANTONIO", "loc" : [ -98.52596699999999, 29.422855 ], "pop" : 58355, "state" : "TX" }
>> +{ "_id" : "78521", "city" : "BROWNSVILLE", "loc" : [ -97.461236, 25.922103 ], "pop" : 79463, "state" : "TX" }
>> +{ "_id" : "78572", "city" : "ALTON", "loc" : [ -98.342647, 26.24153 ], "pop" : 67604, "state" : "TX" }
>> +{ "_id" : "80123", "city" : "BOW MAR", "loc" : [ -105.07766, 39.596854 ], "pop" : 59418, "state" : "CO" }
>> +{ "_id" : "80221", "city" : "FEDERAL HEIGHTS", "loc" : [ -105.007985, 39.840562 ], "pop" : 54069, "state" : "CO" }
>> +{ "_id" : "80631", "city" : "GARDEN CITY", "loc" : [ -104.704756, 40.413968 ], "pop" : 53905, "state" : "CO" }
>> +{ "_id" : "82001", "city" : "CHEYENNE", "loc" : [ -104.796234, 41.143719 ], "pop" : 33107, "state" : "WY" }
>> +{ "_id" : "82070", "city" : "LARAMIE", "loc" : [ -105.581146, 41.312907 ], "pop" : 29327, "state" : "WY" }
>> +{ "_id" : "82716", "city" : "GILLETTE", "loc" : [ -105.497442, 44.282009 ], "pop" : 25968, "state" : "WY" }
>> +{ "_id" : "83301", "city" : "TWIN FALLS", "loc" : [ -114.469265, 42.556495 ], "pop" : 34539, "state" : "ID" }
>> +{ "_id" : "83704", "city" : "BOISE", "loc" : [ -116.295099, 43.633001 ], "pop" : 40912, "state" : "ID" }
>> +{ "_id" : "83814", "city" : "COEUR D ALENE", "loc" : [ -116.784976, 47.692841 ], "pop" : 33589, "state" : "ID" }
>> +{ "_id" : "84118", "city" : "KEARNS", "loc" : [ -111.98521, 40.652759 ], "pop" : 55999, "state" : "UT" }
>> +{ "_id" : "84120", "city" : "WEST VALLEY CITY", "loc" : [ -112.009783, 40.68708 ], "pop" : 52854, "state" : "UT" }
>> +{ "_id" : "84604", "city" : "PROVO", "loc" : [ -111.654906, 40.260681 ], "pop" : 43841, "state" : "UT" }
>> +{ "_id" : "85023", "city" : "PHOENIX", "loc" : [ -112.111838, 33.632383 ], "pop" : 54668, "state" : "AZ" }
>> +{ "_id" : "85204", "city" : "MESA", "loc" : [ -111.789554, 33.399168 ], "pop" : 55180, "state" : "AZ" }
>> +{ "_id" : "85364", "city" : "YUMA", "loc" : [ -114.642362, 32.701507 ], "pop" : 57131, "state" : "AZ" }
>> +{ "_id" : "87501", "city" : "POJOAQUE VALLEY", "loc" : [ -105.974818, 35.702472 ], "pop" : 51715, "state" : "NM" }
>> +{ "_id" : "88001", "city" : "LAS CRUCES", "loc" : [ -106.746034, 32.321641 ], "pop" : 57502, "state" : "NM" }
>> +{ "_id" : "88201", "city" : "ROSWELL", "loc" : [ -104.525857, 33.388504 ], "pop" : 53644, "state" : "NM" }
>> +{ "_id" : "89031", "city" : "NORTH LAS VEGAS", "loc" : [ -115.124832, 36.206228 ], "pop" : 48113, "state" : "NV" }
>> +{ "_id" : "89115", "city" : "LAS VEGAS", "loc" : [ -115.067062, 36.215818 ], "pop" : 51532, "state" : "NV" }
>> +{ "_id" : "89502", "city" : "RENO", "loc" : [ -119.776395, 39.497239 ], "pop" : 38332, "state" : "NV" }
>> +{ "_id" : "90011", "city" : "LOS ANGELES", "loc" : [ -118.258189, 34.007856 ], "pop" : 96074, "state" : "CA" }
>> +{ "_id" : "90201", "city" : "BELL GARDENS", "loc" : [ -118.17205, 33.969177 ], "pop" : 99568, "state" : "CA" }
>> +{ "_id" : "90650", "city" : "NORWALK", "loc" : [ -118.081767, 33.90564 ], "pop" : 94188, "state" : "CA" }
>> +{ "_id" : "96734", "city" : "KAILUA", "loc" : [ -157.744781, 21.406262 ], "pop" : 53403, "state" : "HI" }
>> +{ "_id" : "96744", "city" : "KANEOHE", "loc" : [ -157.811543, 21.422819 ], "pop" : 55236, "state" : "HI" }
>> +{ "_id" : "96818", "city" : "HONOLULU", "loc" : [ -157.926925, 21.353173 ], "pop" : 62915, "state" : "HI" }
>> +{ "_id" : "97005", "city" : "BEAVERTON", "loc" : [ -122.805395, 45.475035 ], "pop" : 46660, "state" : "OR" }
>> +{ "_id" : "97206", "city" : "PORTLAND", "loc" : [ -122.59727, 45.483995 ], "pop" : 43134, "state" : "OR" }
>> +{ "_id" : "97301", "city" : "SALEM", "loc" : [ -122.979692, 44.926039 ], "pop" : 48007, "state" : "OR" }
>> +{ "_id" : "98031", "city" : "KENT", "loc" : [ -122.193184, 47.388004 ], "pop" : 50515, "state" : "WA" }
>> +{ "_id" : "98059", "city" : "RENTON", "loc" : [ -122.151178, 47.467383 ], "pop" : 48197, "state" : "WA" }
>> +{ "_id" : "98310", "city" : "BREMERTON", "loc" : [ -122.629913, 47.601916 ], "pop" : 49057, "state" : "WA" }
>> +{ "_id" : "99504", "city" : "ANCHORAGE", "loc" : [ -149.74467, 61.203696 ], "pop" : 32383, "state" : "AK" }
>> +{ "_id" : "99709", "city" : "FAIRBANKS", "loc" : [ -147.846917, 64.85437 ], "pop" : 23238, "state" : "AK" }
>> +{ "_id" : "99801", "city" : "JUNEAU", "loc" : [ -134.529429, 58.362767 ], "pop" : 24947, "state" : "AK" }
>> \ No newline at end of file
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch5/pom.xml
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch5/pom.xml b/elasticsearch5/pom.xml
>> index 21d60f1..0fe9df9 100644
>> --- a/elasticsearch5/pom.xml
>> +++ b/elasticsearch5/pom.xml
>> @@ -78,6 +78,35 @@ limitations under the License.
>> <version>${elasticsearch5-java-driver.version}</version>
>> </dependency>
>> <dependency>
>> + <groupId>org.elasticsearch.plugin</groupId>
>> + <artifactId>transport-netty3-client</artifactId>
>> + <version>${elasticsearch5-java-driver.version}</version>
>> + <scope>test</scope>
>> + </dependency>
>> + <dependency>
>> + <!-- Elastic search doesn't export painless script artifact to maven central.
>> + Using 3rd party version (codelibs)
>> + This JAR is used only in tests
>> + https://discuss.elastic.co/t/5-0-0-using-painless-in-esintegtestcase/64447/13
>> + -->
>> + <groupId>org.codelibs.elasticsearch.module</groupId>
>> + <artifactId>lang-painless</artifactId>
>> + <version>${elasticsearch5-java-driver.version}</version>
>> + <scope>test</scope>
>> + </dependency>
>> + <dependency>
>> + <groupId>org.apache.logging.log4j</groupId>
>> + <artifactId>log4j-api</artifactId>
>> + <version>${log4j2.version}</version>
>> + <scope>test</scope>
>> + </dependency>
>> + <dependency>
>> + <groupId>org.apache.logging.log4j</groupId>
>> + <artifactId>log4j-core</artifactId>
>> + <version>${log4j2.version}</version>
>> + <scope>test</scope>
>> + </dependency>
>> + <dependency>
>> <groupId>com.carrotsearch</groupId>
>> <artifactId>hppc</artifactId>
>> <version>${hppc.version}</version>
>> @@ -110,6 +139,8 @@ limitations under the License.
>> <ignoredUnusedDeclaredDependency>org.apache.calcite.avatica:avatica</ignoredUnusedDeclaredDependency>
>> <ignoredUnusedDeclaredDependency>org.slf4j:slf4j-api</ignoredUnusedDeclaredDependency>
>> <ignoredUnusedDeclaredDependency>org.slf4j:slf4j-log4j12</ignoredUnusedDeclaredDependency>
>> + <ignoredUnusedDeclaredDependency>org.apache.logging.log4j:log4j-api</ignoredUnusedDeclaredDependency>
>> + <ignoredUnusedDeclaredDependency>org.apache.logging.log4j:log4j-core</ignoredUnusedDeclaredDependency>
>> </ignoredUnusedDeclaredDependencies>
>> </configuration>
>> </execution>
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Schema.java
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Schema.java b/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Schema.java
>> index 6e16f2f..ebb96c8 100644
>> --- a/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Schema.java
>> +++ b/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Schema.java
>> @@ -22,6 +22,8 @@ import org.apache.calcite.schema.impl.AbstractSchema;
>>
>> import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
>>
>> +import com.google.common.annotations.VisibleForTesting;
>> +import com.google.common.base.Preconditions;
>> import com.google.common.collect.ImmutableList;
>> import com.google.common.collect.ImmutableMap;
>>
>> @@ -87,6 +89,17 @@ public class Elasticsearch5Schema extends AbstractSchema
>> }
>> }
>>
>> + /**
>> + * Allows schema to be instantiated from existing elastic search client.
>> + * This constructor is used in tests.
>> + */
>> + @VisibleForTesting
>> + Elasticsearch5Schema(Client client, String index) {
>> + this.client = Preconditions.checkNotNull(client, "client");
>> + this.index = Preconditions.checkNotNull(index, "index");
>> + }
>> +
>> +
>> @Override protected Map<String, Table> getTableMap() {
>> final ImmutableMap.Builder<String, Table> builder = ImmutableMap.builder();
>>
>> @@ -123,8 +136,10 @@ public class Elasticsearch5Schema extends AbstractSchema
>>
>> final List<DiscoveryNode> nodes =
>> ImmutableList.copyOf(transportClient.connectedNodes());
>> +
>> if (nodes.isEmpty()) {
>> - throw new RuntimeException("Cannot connect to any elasticsearch nodes");
>> + throw new IllegalStateException("Cannot connect to any elasticsearch node from "
>> + + transportNodes);
>> }
>>
>> client = transportClient;
>>
>> http://git-wip-us.apache.org/repos/asf/calcite/blob/c12cb4b0/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Table.java
>> ----------------------------------------------------------------------
>> diff --git a/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Table.java b/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Table.java
>> index 175e3cc..66e3edf 100644
>> --- a/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Table.java
>> +++ b/elasticsearch5/src/main/java/org/apache/calcite/adapter/elasticsearch5/Elasticsearch5Table.java
>> @@ -25,15 +25,19 @@ import org.apache.calcite.linq4j.function.Function1;
>> import org.apache.calcite.util.Util;
>>
>> import org.elasticsearch.client.Client;
>> +import org.elasticsearch.common.settings.Settings;
>> import org.elasticsearch.common.xcontent.NamedXContentRegistry;
>> import org.elasticsearch.common.xcontent.XContent;
>> import org.elasticsearch.common.xcontent.XContentParser;
>> import org.elasticsearch.common.xcontent.json.JsonXContent;
>> import org.elasticsearch.index.query.QueryParseContext;
>> import org.elasticsearch.search.SearchHit;
>> +import org.elasticsearch.search.SearchModule;
>> import org.elasticsearch.search.builder.SearchSourceBuilder;
>>
>> import java.io.IOException;
>> +import java.io.UncheckedIOException;
>> +import java.util.Collections;
>> import java.util.Iterator;
>> import java.util.List;
>> import java.util.Map;
>> @@ -61,13 +65,16 @@ public class Elasticsearch5Table extends AbstractElasticsearchTable {
>> searchSourceBuilder = new SearchSourceBuilder();
>> } else {
>> String queryString = "{" + Util.toString(ops, "", ", ", "") + "}";
>> - NamedXContentRegistry xContentRegistry = NamedXContentRegistry.EMPTY;
>> XContent xContent = JsonXContent.jsonXContent;
>> + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
>> + NamedXContentRegistry xContentRegistry =
>> + new NamedXContentRegistry(searchModule.getNamedXContents());
>> +
>> try (XContentParser parser = xContent.createParser(xContentRegistry, queryString)) {
>> final QueryParseContext queryParseContext = new QueryParseContext(parser);
>> searchSourceBuilder = SearchSourceBuilder.fromXContent(queryParseContext);
>> } catch (IOException ex) {
>> - throw new RuntimeException(ex);
>> + throw new UncheckedIOException(ex);
>> }
>> }
>> final Function1<SearchHit, Object> getter = Elasticsearch5Enumerator.getter(fields);
>>
>