You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2015/08/11 14:20:49 UTC
[26/50] [abbrv] incubator-lens git commit: LENS-252 : Add elastic
search driver
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESCriteriaVisitor.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESCriteriaVisitor.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESCriteriaVisitor.java
new file mode 100644
index 0000000..fbc606b
--- /dev/null
+++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESCriteriaVisitor.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.es.translator.impl;
+
+import java.util.List;
+
+import org.apache.lens.driver.es.ESDriverConfig;
+import org.apache.lens.driver.es.exceptions.InvalidQueryException;
+import org.apache.lens.driver.es.translator.ASTCriteriaVisitor;
+
+import org.apache.commons.lang.Validate;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+
+public final class ESCriteriaVisitor implements ASTCriteriaVisitor {
+
+ private final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance;
+
+ private ObjectNode node = jsonNodeFactory.objectNode();
+
+ @Override
+ public void visitLogicalOp(String logicalOp, List<ASTCriteriaVisitor> visitedSubTrees) {
+ final ArrayNode subTrees = jsonNodeFactory.arrayNode();
+ for (ASTCriteriaVisitor criteriaVisitor : visitedSubTrees) {
+ subTrees.add(((ESCriteriaVisitor) criteriaVisitor).node);
+ }
+ node.put(ESDriverConfig.LOGICAL_OPS.get(logicalOp), subTrees);
+ }
+
+ @Override
+ public void visitUnaryLogicalOp(String logicalOp, ASTCriteriaVisitor visitedSubTree) {
+ node.put(ESDriverConfig.LOGICAL_OPS.get(logicalOp), ((ESCriteriaVisitor) visitedSubTree).node);
+ }
+
+ @Override
+ public void visitPredicate(String predicateOp, String leftColCanonical, List<String> rightExps)
+ throws InvalidQueryException {
+ final String leftCol = visitColumn(leftColCanonical);
+ String elasticPredicateOp = ESDriverConfig.PREDICATES.get(predicateOp);
+ if (elasticPredicateOp.equals(ESDriverConfig.TERM)) {
+ final ObjectNode termNode = jsonNodeFactory.objectNode();
+ termNode.put(leftCol, removeSingleQuotesFromLiteral(rightExps.get(0)));
+ node.put(ESDriverConfig.TERM, termNode);
+ } else if (elasticPredicateOp.equals(ESDriverConfig.TERMS)) {
+ final ObjectNode termsNode = jsonNodeFactory.objectNode();
+ final ArrayNode arrayNode = jsonNodeFactory.arrayNode();
+ for (String right : rightExps) {
+ arrayNode.add(removeSingleQuotesFromLiteral(right));
+ }
+ termsNode.put(leftCol, arrayNode);
+ node.put(ESDriverConfig.TERMS, termsNode);
+ } else if (ESDriverConfig.RANGE_PREDICATES.containsValue(elasticPredicateOp)) {
+ final ObjectNode rangeNode = jsonNodeFactory.objectNode();
+ final ObjectNode rangeInnerNode = jsonNodeFactory.objectNode();
+ if (predicateOp.equals("between")) {
+ Validate.isTrue(rightExps.size() == 2);
+ rangeInnerNode.put("gt", removeSingleQuotesFromLiteral(rightExps.get(0)));
+ rangeInnerNode.put("lt", removeSingleQuotesFromLiteral(rightExps.get(1)));
+ } else {
+ rangeInnerNode.put(elasticPredicateOp, removeSingleQuotesFromLiteral(rightExps.get(0)));
+ }
+ rangeNode.put(leftCol, rangeInnerNode);
+ node.put(ESDriverConfig.RANGE, rangeNode);
+ } else {
+ throw new InvalidQueryException("No handlers for the registered predicate" + predicateOp);
+ }
+ }
+
+ public ObjectNode getNode() {
+ return node;
+ }
+
+ private static String visitColumn(String cannonicalColName) {
+ final String[] colParts = cannonicalColName.split("\\.");
+ return colParts[colParts.length - 1];
+ }
+
+ private static String removeSingleQuotesFromLiteral(String rightExp) {
+ return rightExp.replaceAll("^\'|\'$", "");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESCriteriaVisitorFactory.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESCriteriaVisitorFactory.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESCriteriaVisitorFactory.java
new file mode 100644
index 0000000..04b773d
--- /dev/null
+++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESCriteriaVisitorFactory.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.es.translator.impl;
+
+import org.apache.lens.driver.es.translator.CriteriaVisitorFactory;
+
+public final class ESCriteriaVisitorFactory implements CriteriaVisitorFactory {
+
+ @Override
+ public ESCriteriaVisitor getInstance() {
+ return new ESCriteriaVisitor();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESTermVisitor.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESTermVisitor.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESTermVisitor.java
new file mode 100644
index 0000000..49aa0d4
--- /dev/null
+++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/translator/impl/ESTermVisitor.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.es.translator.impl;
+
+import static org.apache.lens.driver.es.ESDriverConfig.*;
+
+import org.apache.lens.driver.es.ESDriverConfig;
+import org.apache.lens.driver.es.ESQuery;
+import org.apache.lens.driver.es.translator.ESVisitor;
+
+import org.apache.commons.lang3.Validate;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * The ESTermVisitor for constructing elastic search queries from ASTNode
+ */
+public class ESTermVisitor extends ESVisitor {
+
+ final ArrayNode fields = JSON_NODE_FACTORY.arrayNode();
+ final ArrayNode sorts = JSON_NODE_FACTORY.arrayNode();
+
+ public ESTermVisitor(ESDriverConfig config) {
+ super(config);
+ queryType = ESQuery.QueryType.TERM;
+ }
+
+ @Override
+ public void visitSimpleSelect(String column, String alias) {
+ column = visitColumn(column);
+ final String aliasName = alias == null ? column : alias;
+ Validate.isTrue(!querySchema.contains(aliasName), "Ambiguous alias '" + aliasName + "'");
+ querySchema.add(aliasName);
+ selectedColumnNames.add(column);
+ fields.add(column);
+ }
+
+ @Override
+ public void visitAggregation(String aggregationType, String columnName, String alias) {
+ throw new UnsupportedOperationException("Valid groups have to be specified for aggregation");
+ }
+
+ @Override
+ public void visitGroupBy(String colName) {
+ throw new UnsupportedOperationException("Group bys are not specified in a term query");
+ }
+
+ @Override
+ public void visitLimit(int limit) {
+ if (this.limit==-1 || limit < config.getMaxLimit()) {
+ this.limit = limit;
+ }
+ }
+
+ @Override
+ public void visitOrderBy(String colName, OrderBy orderBy) {
+ colName = visitColumn(colName);
+ final ObjectNode sortNode = JSON_NODE_FACTORY.objectNode();
+ sortNode.put(colName, ORDER_BYS.get(orderBy));
+ sorts.add(sortNode);
+ }
+
+ @Override
+ public void completeVisit() {
+ queryNode.put(FROM, DEFAULT_TERM_QUERY_OFFSET);
+ final int initialBatch = limit != -1 && limit < config.getTermFetchSize()
+ ?
+ limit
+ :
+ config.getTermFetchSize();
+ queryNode.put(SIZE, initialBatch);
+ queryNode.put(FIELDS, fields);
+ queryNode.put(TERM_SORT, sorts);
+ queryNode.put(ESDriverConfig.QUERY_TIME_OUT_STRING, config.getQueryTimeOutMs());
+ queryNode.put(FILTER, criteriaNode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/main/resources/esdriver-default.xml
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/main/resources/esdriver-default.xml b/lens-driver-es/src/main/resources/esdriver-default.xml
new file mode 100644
index 0000000..5bd56b3
--- /dev/null
+++ b/lens-driver-es/src/main/resources/esdriver-default.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+
+ <property>
+ <name>lens.driver.es.term.fetch.size</name>
+ <description>Fetch (buffer) size for document look up queries</description>
+ <value>10000</value>
+ </property>
+ <property>
+ <name>lens.driver.es.query.timeout.millis</name>
+ <description>Query timeout</description>
+ <value>10000</value>
+ </property>
+ <property>
+ <name>lens.driver.es.max.row.size</name>
+ <description>max rows for es document look up queries, non existent or -1 refers no limit</description>
+ <value>-1</value>
+ </property>
+ <property>
+ <name>lens.driver.es.aggr.bucket.size</name>
+ <description>Max cardinality of group by (higher value means higher resource usage at server end)</description>
+ <value>6</value>
+ </property>
+ <property>
+ <name>lens.driver.es.jest.servers</name>
+ <description>List of http servers, will be used on a round robin basis</description>
+ <value>http://localhost:9200,http://localhost:4200</value>
+ </property>
+ <property>
+ <name>lens.driver.es.jest.max.conn</name>
+ <description>max connections</description>
+ <value>20</value>
+ </property>
+ <property>
+ <name>lens.driver.es.client.class</name>
+ <description>Choice of client class, default is JestClientImpl</description>
+ <value>org.apache.lens.driver.es.client.jest.JestClientImpl</value>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java
new file mode 100644
index 0000000..f453416
--- /dev/null
+++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.es;
+
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.testng.annotations.BeforeTest;
+
+public abstract class ESDriverTest {
+
+ protected Configuration config = new Configuration();
+ protected ESDriverConfig esDriverConfig;
+ protected ESDriver driver = new ESDriver();
+ protected MockClientES mockClientES;
+
+ @BeforeTest
+ public void beforeTest() throws LensException {
+ initializeConfig(config);
+ esDriverConfig = new ESDriverConfig(config);
+ driver.configure(config);
+ mockClientES = (MockClientES) driver.getESClient();
+ }
+
+ protected abstract void initializeConfig(Configuration config);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/test/java/org/apache/lens/driver/es/MockClientES.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/MockClientES.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/MockClientES.java
new file mode 100644
index 0000000..77300f9
--- /dev/null
+++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/MockClientES.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.es;
+
+import java.util.List;
+
+import org.apache.lens.api.query.ResultRow;
+import org.apache.lens.driver.es.client.ESClient;
+import org.apache.lens.driver.es.client.ESResultSet;
+import org.apache.lens.server.api.driver.LensResultSetMetadata;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.service.cli.ColumnDescriptor;
+import org.apache.hive.service.cli.Type;
+import org.apache.hive.service.cli.TypeDescriptor;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+
+class MockClientES extends ESClient {
+
+ private static final ImmutableMap<String, ResultSetProvider> QUERY_RESULTS_MAP;
+
+ private interface ResultSetProvider {
+ ESResultSet getResultSet();
+ }
+
+ static {
+ ImmutableMap.Builder<String, ResultSetProvider> queryResultsMapBuilder = ImmutableMap.builder();
+ queryResultsMapBuilder.put(
+ "{\"from\":0,\"size\":1,\"fields\":[\"col1\"],\"sort\":[],\"timeout\":10000,\"filter\":{\"match_all\":{}}}",
+ new ResultSetProvider() {
+ @Override
+ public ESResultSet getResultSet() {
+ return new ESResultSet(
+ 1,
+ Lists.newArrayList(
+ new ResultRow(Lists.<Object>newArrayList("v1"))
+ ),
+ new LensResultSetMetadata() {
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ return Lists.newArrayList(
+ new ColumnDescriptor("col1", "", new TypeDescriptor(Type.STRING_TYPE), 0)
+ );
+ }
+ });
+ }
+ });
+ queryResultsMapBuilder.put(
+ "{\"from\":1,\"size\":1,\"fields\":[\"col1\"],\"sort\":[],\"timeout\":10000,\"filter\":{\"match_all\":{}}}",
+ new ResultSetProvider() {
+ @Override
+ public ESResultSet getResultSet() {
+ return new ESResultSet(
+ 1,
+ Lists.newArrayList(
+ new ResultRow(Lists.<Object>newArrayList("v1"))
+ ),
+ new LensResultSetMetadata() {
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ return Lists.newArrayList(
+ new ColumnDescriptor("col1", "", new TypeDescriptor(Type.STRING_TYPE), 0)
+ );
+ }
+ });
+ }
+ });
+ queryResultsMapBuilder.put(
+ "{\"from\":2,\"size\":1,\"fields\":[\"col1\"],\"sort\":[],\"timeout\":10000,\"filter\":{\"match_all\":{}}}",
+ new ResultSetProvider() {
+ @Override
+ public ESResultSet getResultSet() {
+ return new ESResultSet(
+ 1,
+ Lists.newArrayList(
+ new ResultRow(Lists.<Object>newArrayList("v1"))
+ ),
+ new LensResultSetMetadata() {
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ return Lists.newArrayList(
+ new ColumnDescriptor("col1", "", new TypeDescriptor(Type.STRING_TYPE), 0)
+ );
+ }
+ });
+ }
+ });
+ queryResultsMapBuilder.put(
+ "{\"from\":3,\"size\":1,\"fields\":[\"col1\"],\"sort\":[],\"timeout\":10000,\"filter\":{\"match_all\":{}}}",
+ new ResultSetProvider() {
+ @Override
+ public ESResultSet getResultSet() {
+ return new ESResultSet(
+ 0,
+ Lists.<ResultRow>newArrayList(),
+ new LensResultSetMetadata() {
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ return Lists.newArrayList(
+ new ColumnDescriptor("col1", "", new TypeDescriptor(Type.STRING_TYPE), 0)
+ );
+ }
+ });
+ }
+ });
+ QUERY_RESULTS_MAP = queryResultsMapBuilder.build();
+ }
+
+
+ public MockClientES(ESDriverConfig esDriverConfig, Configuration conf) {
+ super(esDriverConfig, conf);
+ }
+
+ @Override
+ protected ESResultSet executeImpl(ESQuery esQuery) {
+ return QUERY_RESULTS_MAP.get(esQuery.getQuery()).getResultSet();
+ }
+
+ @Override
+ public String explain(ESQuery esQuery) {
+ return QUERY_RESULTS_MAP.containsKey(esQuery.getQuery())
+ ?
+ "{}"
+ :
+ null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/test/java/org/apache/lens/driver/es/QueryTranslationTest.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/QueryTranslationTest.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/QueryTranslationTest.java
new file mode 100644
index 0000000..be33764
--- /dev/null
+++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/QueryTranslationTest.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.es;
+
+import java.io.IOException;
+
+import org.apache.lens.cube.parse.BetweenTimeRangeWriter;
+import org.apache.lens.cube.parse.CubeQueryConfUtil;
+import org.apache.lens.driver.es.translator.ESVisitor;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.testng.Assert;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import com.google.common.collect.ImmutableList;
+
+public class QueryTranslationTest extends ESDriverTest {
+
+ private static class ValidQuery {
+ private final String name;
+ private final String hql;
+ private final JsonNode expectedJson;
+
+ @JsonCreator
+ ValidQuery(@JsonProperty("name") String name,
+ @JsonProperty("hql") String hql,
+ @JsonProperty("expectedJson") JsonNode expectedJson) {
+ this.name = name;
+ this.hql = hql;
+ this.expectedJson = expectedJson;
+ }
+ }
+
+ private static class InvalidQuery {
+ private final String name;
+ private final String hql;
+
+ @JsonCreator
+ InvalidQuery(@JsonProperty("name") String name,
+ @JsonProperty("hql") String hql) {
+ this.name = name;
+ this.hql = hql;
+ }
+ }
+
+ private static <T> T loadResource(String resourcePath, Class<T> type) {
+ try {
+ return OBJECT_MAPPER.readValue(
+ QueryTranslationTest.class.getClassLoader()
+ .getResourceAsStream(resourcePath),
+ type
+ );
+ } catch (IOException e) {
+ throw new RuntimeException("FATAL! Cannot initialize test resource : " + resourcePath);
+ }
+ }
+
+ public static final String VALID_QUERIES_RESOURCE_PATH = "valid-queries.data";
+ public static final String INVALID_QUERIES_RESOURCE_PATH = "invalid-queries.data";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ private static final ImmutableList<ValidQuery> VALID_QUERIES;
+ private static final ImmutableList<InvalidQuery> IN_VALID_QUERIES;
+ static {
+ VALID_QUERIES = ImmutableList.copyOf(loadResource(VALID_QUERIES_RESOURCE_PATH, ValidQuery[].class));
+ IN_VALID_QUERIES = ImmutableList.copyOf(loadResource(INVALID_QUERIES_RESOURCE_PATH, InvalidQuery[].class));
+ }
+
+ @BeforeTest
+ @Override
+ public void beforeTest() throws LensException {
+ super.beforeTest();
+ }
+
+ @Override
+ protected void initializeConfig(Configuration config) {
+ config.setInt(ESDriverConfig.TERM_FETCH_SIZE_KEY, 10000);
+ config.setInt(ESDriverConfig.QUERY_TIME_OUT_LENS_KEY, 10000);
+ config.setInt(ESDriverConfig.MAX_ROW_SIZE_KEY, -1);
+ config.setInt(ESDriverConfig.AGGR_BUCKET_SIZE_LENS_KEY, 100);
+ config.setStrings(ESDriverConfig.CLIENT_CLASS_KEY, MockClientES.class.getCanonicalName());
+ config.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false);
+ config.setStrings(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "es_storage");
+ config.setStrings(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, BetweenTimeRangeWriter.class.getCanonicalName());
+ config.setStrings(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT, "yyyy-MM-dd'T'HH:mm:ss");
+ }
+
+ @Test
+ public void testQueryTranslation() throws LensException {
+ for(final ValidQuery query : VALID_QUERIES) {
+ Assert.assertEquals(
+ ESVisitor.rewrite(esDriverConfig, query.hql).getQuery(),
+ query.expectedJson.toString(),
+ "Test case '" + query.name + "' failed."
+ );
+ }
+ }
+
+ @Test
+ public void testInvalidQueries() {
+ for(final InvalidQuery invalidQuery : IN_VALID_QUERIES) {
+ try {
+ ESVisitor.rewrite(esDriverConfig, invalidQuery.hql);
+ Assert.fail("The invalid query"+ invalidQuery.name +"did not suffer any exception");
+ } catch (Throwable e) {
+ continue;
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java
new file mode 100644
index 0000000..82835b1
--- /dev/null
+++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ResultSetTransformationTest.java
@@ -0,0 +1,573 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.es;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lens.api.query.ResultRow;
+import org.apache.lens.cube.parse.CubeQueryConfUtil;
+import org.apache.lens.driver.es.client.ESResultSet;
+import org.apache.lens.driver.es.client.jest.JestResultSetTransformer;
+import org.apache.lens.server.api.driver.LensResultSetMetadata;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.service.cli.ColumnDescriptor;
+import org.apache.hive.service.cli.Type;
+import org.apache.hive.service.cli.TypeDescriptor;
+
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+import junit.framework.Assert;
+
+public class ResultSetTransformationTest extends ESDriverTest {
+
+ private static final ImmutableMap<Result, ESResultSet> VALID_TRANSFORMATIONS;
+ private static final ImmutableMap<Result, ESResultSet> IN_VALID_TRANSFORMATIONS;
+ private static final JsonParser JSON_PARSER = new JsonParser();
+
+ static {
+ ImmutableMap.Builder<Result, ESResultSet> reponsesBuilder = ImmutableMap.builder();
+
+ /**
+ * Sample term query result transformation
+ */
+ reponsesBuilder.put(
+ new Result(
+ Lists.newArrayList("col1_alias", "col2"),
+ Lists.newArrayList("col1", "col2"),
+ (JsonObject) JSON_PARSER.parse("{\n"
+ + " \"took\": 653,\n"
+ + " \"timed_out\": false,\n"
+ + " \"_shards\": {\n"
+ + " \"total\": 5,\n"
+ + " \"successful\": 5,\n"
+ + " \"failed\": 0\n"
+ + " },\n"
+ + " \"hits\": {\n"
+ + " \"total\": 100,\n"
+ + " \"max_score\": 1,\n"
+ + " \"hits\": [\n"
+ + " {\n"
+ + " \"_index\": \"index\",\n"
+ + " \"_type\": \"type\",\n"
+ + " \"_id\": \"12345\",\n"
+ + " \"_score\": 1,\n"
+ + " \"fields\": {\n"
+ + " \"col1\": [\n"
+ + " \"val1\"\n"
+ + " ],\n"
+ + " \"col2\": [\n"
+ + " \"val2\"\n"
+ + " ]\n"
+ + " }\n"
+ + " },\n"
+ + " {\n"
+ + " \"_index\": \"index\",\n"
+ + " \"_type\": \"type\",\n"
+ + " \"_id\": \"123456\",\n"
+ + " \"_score\": 1,\n"
+ + " \"fields\": {\n"
+ + " \"col1\": [\n"
+ + " \"val3\"\n"
+ + " ],\n"
+ + " \"col2\": [\n"
+ + " \"val4\"\n"
+ + " ]\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + "}"),
+ ESQuery.QueryType.AGGR
+ ),
+ new ESResultSet(
+ 2,
+ Lists.newArrayList(
+ new ResultRow(Lists.<Object>newArrayList("val1", "val2")),
+ new ResultRow(Lists.<Object>newArrayList("val3", "val4"))
+ ),
+ new LensResultSetMetadata() {
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ return Lists.newArrayList(
+ new ColumnDescriptor("col1_alias", "", new TypeDescriptor(Type.STRING_TYPE), 0),
+ new ColumnDescriptor("col2", "", new TypeDescriptor(Type.STRING_TYPE), 1)
+ );
+ }
+ })
+ );
+
+ /**
+ * Sample aggregate query transformation
+ */
+ reponsesBuilder.put(
+ new Result(
+ Lists.newArrayList("col1", "col2", "aggr_col"),
+ Lists.newArrayList("col1", "col2", "aggr_col"),
+ (JsonObject) JSON_PARSER.parse("{\n"
+ + " \"took\": 3356,\n"
+ + " \"timed_out\": false,\n"
+ + " \"_shards\": {\n"
+ + " \"total\": 5,\n"
+ + " \"successful\": 5,\n"
+ + " \"failed\": 0\n"
+ + " },\n"
+ + " \"hits\": {\n"
+ + " \"total\": 100,\n"
+ + " \"max_score\": 0,\n"
+ + " \"hits\": []\n"
+ + " },\n"
+ + " \"aggregations\": {\n"
+ + " \"filter_wrapper\": {\n"
+ + " \"doc_count\": 100,\n"
+ + " \"col1\": {\n"
+ + " \"doc_count_error_upper_bound\": 333,\n"
+ + " \"sum_other_doc_count\": 18017550,\n"
+ + " \"buckets\": [\n"
+ + " {\n"
+ + " \"key\": \"g1v1\",\n"
+ + " \"doc_count\": 14099915,\n"
+ + " \"col2\": {\n"
+ + " \"doc_count_error_upper_bound\": 0,\n"
+ + " \"sum_other_doc_count\": 0,\n"
+ + " \"buckets\": [\n"
+ + " {\n"
+ + " \"key\": \"g2v1\",\n"
+ + " \"doc_count\": 10432335,\n"
+ + " \"aggr_col\": {\n"
+ + " \"value\": 1\n"
+ + " }\n"
+ + " },\n"
+ + " {\n"
+ + " \"key\": \"g2v2\",\n"
+ + " \"doc_count\": 2,\n"
+ + " \"aggr_col\": {\n"
+ + " \"value\": 2\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + " },\n"
+ + " {\n"
+ + " \"key\": \"g1v2\",\n"
+ + " \"doc_count\": 8608107,\n"
+ + " \"col2\": {\n"
+ + " \"doc_count_error_upper_bound\": 0,\n"
+ + " \"sum_other_doc_count\": 0,\n"
+ + " \"buckets\": [\n"
+ + " {\n"
+ + " \"key\": \"g2v3\",\n"
+ + " \"doc_count\": 3,\n"
+ + " \"aggr_col\": {\n"
+ + " \"value\": 3\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + " }\n"
+ + " }\n"
+ + "}"),
+ ESQuery.QueryType.AGGR
+ ),
+ new ESResultSet(
+ 3,
+ Lists.newArrayList(
+ new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v1", "1")),
+ new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v2", "2")),
+ new ResultRow(Lists.<Object>newArrayList("g1v2", "g2v3", "3"))
+ ),
+ new LensResultSetMetadata() {
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ return Lists.newArrayList(
+ new ColumnDescriptor("col1", "", new TypeDescriptor(Type.STRING_TYPE), 0),
+ new ColumnDescriptor("col2", "", new TypeDescriptor(Type.STRING_TYPE), 1),
+ new ColumnDescriptor("aggr_col", "", new TypeDescriptor(Type.STRING_TYPE), 2)
+ );
+ }
+ })
+ );
+
+ reponsesBuilder.put(
+ new Result(
+ Lists.newArrayList("col1", "aggr_col", "col2"),
+ Lists.newArrayList("col1", "aggr_col", "col2"),
+ (JsonObject) JSON_PARSER.parse("{\n"
+ + " \"took\": 3356,\n"
+ + " \"timed_out\": false,\n"
+ + " \"_shards\": {\n"
+ + " \"total\": 5,\n"
+ + " \"successful\": 5,\n"
+ + " \"failed\": 0\n"
+ + " },\n"
+ + " \"hits\": {\n"
+ + " \"total\": 100,\n"
+ + " \"max_score\": 0,\n"
+ + " \"hits\": []\n"
+ + " },\n"
+ + " \"aggregations\": {\n"
+ + " \"filter_wrapper\": {\n"
+ + " \"doc_count\": 100,\n"
+ + " \"col1\": {\n"
+ + " \"doc_count_error_upper_bound\": 333,\n"
+ + " \"sum_other_doc_count\": 18017550,\n"
+ + " \"buckets\": [\n"
+ + " {\n"
+ + " \"key\": \"g1v1\",\n"
+ + " \"doc_count\": 14099915,\n"
+ + " \"col2\": {\n"
+ + " \"doc_count_error_upper_bound\": 0,\n"
+ + " \"sum_other_doc_count\": 0,\n"
+ + " \"buckets\": [\n"
+ + " {\n"
+ + " \"key\": \"g2v1\",\n"
+ + " \"doc_count\": 10432335,\n"
+ + " \"aggr_col\": {\n"
+ + " \"value\": 1\n"
+ + " }\n"
+ + " },\n"
+ + " {\n"
+ + " \"key\": \"g2v2\",\n"
+ + " \"doc_count\": 2,\n"
+ + " \"aggr_col\": {\n"
+ + " \"value\": 2\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + " },\n"
+ + " {\n"
+ + " \"key\": \"g1v2\",\n"
+ + " \"doc_count\": 8608107,\n"
+ + " \"col2\": {\n"
+ + " \"doc_count_error_upper_bound\": 0,\n"
+ + " \"sum_other_doc_count\": 0,\n"
+ + " \"buckets\": [\n"
+ + " {\n"
+ + " \"key\": \"g2v3\",\n"
+ + " \"doc_count\": 3,\n"
+ + " \"aggr_col\": {\n"
+ + " \"value\": 3\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + " }\n"
+ + " }\n"
+ + "}"),
+ ESQuery.QueryType.AGGR
+ ),
+ new ESResultSet(
+ 3,
+ Lists.newArrayList(
+ new ResultRow(Lists.<Object>newArrayList("g1v1", "1", "g2v1")),
+ new ResultRow(Lists.<Object>newArrayList("g1v1", "2", "g2v2")),
+ new ResultRow(Lists.<Object>newArrayList("g1v2", "3", "g2v3"))
+ ),
+ new LensResultSetMetadata() {
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ return Lists.newArrayList(
+ new ColumnDescriptor("col1", "", new TypeDescriptor(Type.STRING_TYPE), 0),
+ new ColumnDescriptor("aggr_col", "", new TypeDescriptor(Type.STRING_TYPE), 1),
+ new ColumnDescriptor("col2", "", new TypeDescriptor(Type.STRING_TYPE), 2)
+ );
+ }
+ })
+ );
+
+
+ VALID_TRANSFORMATIONS = reponsesBuilder.build();
+
+ ImmutableMap.Builder<Result, ESResultSet> invalidResponsesBuilder = ImmutableMap.builder();
+ /**
+ * invalid aliases
+ */
+ invalidResponsesBuilder.put(
+ new Result(
+ Lists.newArrayList("col1", "col2"),
+ Lists.newArrayList("col1", "col2"),
+ (JsonObject) JSON_PARSER.parse("{\n"
+ + " \"took\": 653,\n"
+ + " \"timed_out\": false,\n"
+ + " \"_shards\": {\n"
+ + " \"total\": 5,\n"
+ + " \"successful\": 5,\n"
+ + " \"failed\": 0\n"
+ + " },\n"
+ + " \"hits\": {\n"
+ + " \"total\": 100,\n"
+ + " \"max_score\": 1,\n"
+ + " \"hits\": [\n"
+ + " {\n"
+ + " \"_index\": \"index\",\n"
+ + " \"_type\": \"type\",\n"
+ + " \"_id\": \"12345\",\n"
+ + " \"_score\": 1,\n"
+ + " \"fields\": {\n"
+ + " \"col1\": [\n"
+ + " \"val1\"\n"
+ + " ],\n"
+ + " \"col2\": [\n"
+ + " \"val2\"\n"
+ + " ]\n"
+ + " }\n"
+ + " },\n"
+ + " {\n"
+ + " \"_index\": \"index\",\n"
+ + " \"_type\": \"type\",\n"
+ + " \"_id\": \"123456\",\n"
+ + " \"_score\": 1,\n"
+ + " \"fields\": {\n"
+ + " \"col1\": [\n"
+ + " \"val3\"\n"
+ + " ],\n"
+ + " \"col2\": [\n"
+ + " \"val4\"\n"
+ + " ]\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + "}"),
+ ESQuery.QueryType.AGGR
+ ),
+ new ESResultSet(
+ 2,
+ Lists.newArrayList(
+ new ResultRow(Lists.<Object>newArrayList("val1", "val2")),
+ new ResultRow(Lists.<Object>newArrayList("val3", "val4"))
+ ),
+ new LensResultSetMetadata() {
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ return Lists.newArrayList(
+ new ColumnDescriptor("col1_alias", "", new TypeDescriptor(Type.STRING_TYPE), 0),
+ new ColumnDescriptor("col2", "", new TypeDescriptor(Type.STRING_TYPE), 1)
+ );
+ }
+ })
+ );
+
+ /**
+ * Invalid aggregate transformations, missing aliases
+ */
+ invalidResponsesBuilder.put(
+ new Result(
+ Lists.newArrayList("col1", "aggr_col"),
+ Lists.newArrayList("col1", "aggr_col"),
+ (JsonObject) JSON_PARSER.parse("{\n"
+ + " \"took\": 3356,\n"
+ + " \"timed_out\": false,\n"
+ + " \"_shards\": {\n"
+ + " \"total\": 5,\n"
+ + " \"successful\": 5,\n"
+ + " \"failed\": 0\n"
+ + " },\n"
+ + " \"hits\": {\n"
+ + " \"total\": 100,\n"
+ + " \"max_score\": 0,\n"
+ + " \"hits\": []\n"
+ + " },\n"
+ + " \"aggregations\": {\n"
+ + " \"filter_wrapper\": {\n"
+ + " \"doc_count\": 100,\n"
+ + " \"col1\": {\n"
+ + " \"doc_count_error_upper_bound\": 333,\n"
+ + " \"sum_other_doc_count\": 18017550,\n"
+ + " \"buckets\": [\n"
+ + " {\n"
+ + " \"key\": \"g1v1\",\n"
+ + " \"doc_count\": 14099915,\n"
+ + " \"col2\": {\n"
+ + " \"doc_count_error_upper_bound\": 0,\n"
+ + " \"sum_other_doc_count\": 0,\n"
+ + " \"buckets\": [\n"
+ + " {\n"
+ + " \"key\": \"g2v1\",\n"
+ + " \"doc_count\": 10432335,\n"
+ + " \"aggr_col\": {\n"
+ + " \"value\": 1\n"
+ + " }\n"
+ + " },\n"
+ + " {\n"
+ + " \"key\": \"g2v2\",\n"
+ + " \"doc_count\": 2,\n"
+ + " \"aggr_col\": {\n"
+ + " \"value\": 2\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + " },\n"
+ + " {\n"
+ + " \"key\": \"g1v2\",\n"
+ + " \"doc_count\": 8608107,\n"
+ + " \"col2\": {\n"
+ + " \"doc_count_error_upper_bound\": 0,\n"
+ + " \"sum_other_doc_count\": 0,\n"
+ + " \"buckets\": [\n"
+ + " {\n"
+ + " \"key\": \"g2v3\",\n"
+ + " \"doc_count\": 3,\n"
+ + " \"aggr_col\": {\n"
+ + " \"value\": 3\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + " }\n"
+ + " ]\n"
+ + " }\n"
+ + " }\n"
+ + " }\n"
+ + "}"),
+ ESQuery.QueryType.AGGR
+ ),
+ new ESResultSet(
+ 3,
+ Lists.newArrayList(
+ new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v1", "1")),
+ new ResultRow(Lists.<Object>newArrayList("g1v1", "g2v2", "2")),
+ new ResultRow(Lists.<Object>newArrayList("g1v2", "g2v3", "3"))
+ ),
+ new LensResultSetMetadata() {
+ @Override
+ public List<ColumnDescriptor> getColumns() {
+ return Lists.newArrayList(
+ new ColumnDescriptor("col1", "", new TypeDescriptor(Type.STRING_TYPE), 0),
+ new ColumnDescriptor("col2", "", new TypeDescriptor(Type.STRING_TYPE), 1),
+ new ColumnDescriptor("aggr_col", "", new TypeDescriptor(Type.STRING_TYPE), 2)
+ );
+ }
+ })
+ );
+ IN_VALID_TRANSFORMATIONS = invalidResponsesBuilder.build();
+
+ }
+
+ @BeforeTest
+ @Override
+ public void beforeTest() throws LensException {
+ super.beforeTest();
+ }
+
+ @Override
+ protected void initializeConfig(Configuration config) {
+ config.setInt(ESDriverConfig.TERM_FETCH_SIZE_KEY, 10000);
+ config.setInt(ESDriverConfig.QUERY_TIME_OUT_LENS_KEY, 10000);
+ config.setInt(ESDriverConfig.MAX_ROW_SIZE_KEY, -1);
+ config.setInt(ESDriverConfig.AGGR_BUCKET_SIZE_LENS_KEY, 100);
+ config.setStrings(ESDriverConfig.CLIENT_CLASS_KEY, MockClientES.class.getCanonicalName());
+ config.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false);
+ config.setStrings(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "es_storage");
+ }
+
+ private void assertResultsAreEqual(ESResultSet resultSet1, ESResultSet resultSet2) {
+ final Collection<ColumnDescriptor> columns1 = resultSet1.getMetadata().getColumns();
+ final Collection<ColumnDescriptor> columns2 = resultSet2.getMetadata().getColumns();
+ Assert.assertEquals(columns1.size(), columns2.size());
+ final Iterator<ColumnDescriptor> iterator1 = columns1.iterator();
+ final Iterator<ColumnDescriptor> iterator2 = columns2.iterator();
+ while (iterator1.hasNext()) {
+ final ColumnDescriptor column1 = iterator1.next();
+ final ColumnDescriptor column2 = iterator2.next();
+ Assert.assertEquals("Column aliases are different! " + column1.getName() + " " + column1.getName(),
+ column1.getName(), column2.getName());
+ Assert.assertEquals("Column positions are different! " + column1.getName() + " " + column1.getName(),
+ column1.getOrdinalPosition(), column2.getOrdinalPosition());
+ Assert.assertEquals("Column types are different! " + column1.getName() + " " + column1.getName(),
+ column1.getType(), column2.getType());
+ }
+
+ Assert.assertEquals(resultSet1.size(), resultSet2.size());
+ while (resultSet1.hasNext()) {
+ final ResultRow row1 = resultSet1.next();
+ final ResultRow row2 = resultSet2.next();
+ Assert.assertEquals("Row length is different", row1.getValues().size(), row2.getValues().size());
+ Iterator<Object> values1 = row1.getValues().iterator();
+ Iterator<Object> values2 = row2.getValues().iterator();
+ while (values1.hasNext()) {
+ Assert.assertEquals("Values are different", values1.next(), values2.next());
+ }
+ }
+
+ }
+
+ @Test
+ public void testTransformations() {
+ for (Map.Entry<Result, ESResultSet> entry : VALID_TRANSFORMATIONS.entrySet()) {
+ final Result rawResult = entry.getKey();
+ ESResultSet resultSet =
+ JestResultSetTransformer.transformFrom(rawResult.object, rawResult.schema, rawResult.cols);
+ assertResultsAreEqual(resultSet, entry.getValue());
+ }
+ }
+
+ @Test
+ public void testInvalidTranformations() {
+ for (Map.Entry<Result, ESResultSet> entry : IN_VALID_TRANSFORMATIONS.entrySet()) {
+ boolean failed = false;
+ try {
+ final Result rawResult = entry.getKey();
+ ESResultSet resultSet =
+ JestResultSetTransformer.transformFrom(rawResult.object, rawResult.schema, rawResult.cols);
+ assertResultsAreEqual(resultSet, entry.getValue());
+ failed = true;
+ throw new RuntimeException("Result sets are equal - ");
+ } catch (Throwable e) {
+ if (failed) {
+ Assert.fail("Results sets are equal Expected - not equal" + e.getMessage());
+ }
+ }
+ }
+ }
+
+ static class Result {
+ final List<String> schema;
+ final List<String> cols;
+ final JsonObject object;
+ final ESQuery.QueryType queryType;
+
+ Result(List<String> schema, List<String> cols, JsonObject object, ESQuery.QueryType queryType) {
+ this.schema = schema;
+ this.cols = cols;
+ this.object = object;
+ this.queryType = queryType;
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/test/java/org/apache/lens/driver/es/ScrollingQueryTest.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ScrollingQueryTest.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ScrollingQueryTest.java
new file mode 100644
index 0000000..ea84d8c
--- /dev/null
+++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ScrollingQueryTest.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.driver.es;
+
+import java.util.List;
+
+import org.apache.lens.api.query.ResultRow;
+import org.apache.lens.cube.parse.CubeQueryConfUtil;
+import org.apache.lens.driver.es.client.ESResultSet;
+import org.apache.lens.driver.es.translator.ESVisitor;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.testng.Assert;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import com.beust.jcommander.internal.Lists;
+
+public class ScrollingQueryTest extends ESDriverTest{
+
+ public ScrollingQueryTest() {
+ super();
+ }
+
+ @Override
+ protected void initializeConfig(Configuration config) {
+ config.setInt(ESDriverConfig.TERM_FETCH_SIZE_KEY, 1);
+ config.setInt(ESDriverConfig.QUERY_TIME_OUT_LENS_KEY, 10000);
+ config.setInt(ESDriverConfig.MAX_ROW_SIZE_KEY, -1);
+ config.setInt(ESDriverConfig.AGGR_BUCKET_SIZE_LENS_KEY, 100);
+ config.setStrings(ESDriverConfig.CLIENT_CLASS_KEY, MockClientES.class.getCanonicalName());
+ config.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false);
+ config.setStrings(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "es_storage");
+ }
+
+ @BeforeTest
+ @Override
+ public void beforeTest() throws LensException {
+ super.beforeTest();
+ }
+
+ @Test
+ protected void testScrollQueryExactLimit() throws LensException {
+ ESResultSet resultSet =
+ mockClientES.execute(ESVisitor.rewrite(esDriverConfig, "select col1 from index.type limit 3"));
+ final List<ResultRow> rows = Lists.newArrayList();
+ while (resultSet.hasNext()) {
+ rows.add(resultSet.next());
+ }
+ Assert.assertEquals(rows.size(), 3, "Streaming failed!!!");
+ }
+
+ @Test
+ protected void testScrollQueryLesserLimit() throws LensException {
+ ESResultSet resultSet =
+ mockClientES.execute(ESVisitor.rewrite(esDriverConfig, "select col1 from index.type limit 2"));
+ final List<ResultRow> rows = Lists.newArrayList();
+ while (resultSet.hasNext()) {
+ rows.add(resultSet.next());
+ }
+ Assert.assertEquals(rows.size(), 2, "Streaming failed!!!");
+ }
+
+ @Test
+ protected void testScrollQueryMoreLimit() throws LensException {
+ ESResultSet resultSet =
+ mockClientES.execute(ESVisitor.rewrite(esDriverConfig, "select col1 from index.type limit 4"));
+ final List<ResultRow> rows = Lists.newArrayList();
+ while (resultSet.hasNext()) {
+ rows.add(resultSet.next());
+ }
+ Assert.assertEquals(rows.size(), 3, "Streaming failed!!!");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/test/resources/invalid-queries.data
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/resources/invalid-queries.data b/lens-driver-es/src/test/resources/invalid-queries.data
new file mode 100644
index 0000000..8756a51
--- /dev/null
+++ b/lens-driver-es/src/test/resources/invalid-queries.data
@@ -0,0 +1,18 @@
+[
+ {
+ "name" : "Ambiguous column selection",
+ "hql" : "select r, count(d), sum(z), r from a.x group by r"
+ },
+ {
+ "name" : "Order by cannot be used with group by as of now",
+ "hql" : "select count(d), sum(z), r from a.x group by r order by z"
+ },
+ {
+ "name" : "Group by col not selected",
+ "hql" : "select count(d), sum(z) from a.x group by r"
+ },
+ {
+ "name" : "Invalid columns in select and group by",
+ "hql" : "select a, b from a.x group by r"
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-driver-es/src/test/resources/valid-queries.data
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/resources/valid-queries.data b/lens-driver-es/src/test/resources/valid-queries.data
new file mode 100644
index 0000000..982325c
--- /dev/null
+++ b/lens-driver-es/src/test/resources/valid-queries.data
@@ -0,0 +1,67 @@
+[
+ {
+ "name" : "Basic simple select",
+ "hql" : "select col1 from index.type",
+ "expectedJson" : {"from":0,"size":10000,"fields":["col1"],"sort":[],"timeout":10000,"filter":{"match_all":{}}}
+ },
+ {
+ "name" : "Basic simple select with limit",
+ "hql" : "select col1 from index.type limit 10",
+ "expectedJson" : {"from":0,"size":10,"fields":["col1"],"sort":[],"timeout":10000,"filter":{"match_all":{}}}
+ },
+ {
+ "name" : "Basic simple select with limit and simple criteria",
+ "hql" : "select col1 from index.type where col2=5 limit 10 ",
+ "expectedJson" : {"from":0,"size":10,"fields":["col1"],"sort":[],"timeout":10000,"filter":{"term":{"col2":"5"}}}
+ },
+ {
+ "name" : "Basic simple select with limit and simple criteria - string literal",
+ "hql" : "select col1 from index.type where col2='z' limit 10 ",
+ "expectedJson" : {"from":0,"size":10,"fields":["col1"],"sort":[],"timeout":10000,"filter":{"term":{"col2":"z"}}}
+ },
+ {
+ "name" : "Basic simple select with limit and complex criteria",
+ "hql" : "select col1 from index.type where !(col1=5 and col2='p' and col3='wer' or (col4='9')) limit 10",
+ "expectedJson" : {"from":0,"size":10,"fields":["col1"],"sort":[],"timeout":10000,"filter":{"not":{"or":[{"and":[{"and":[{"term":{"col1":"5"}},{"term":{"col2":"p"}}]},{"term":{"col3":"wer"}}]},{"term":{"col4":"9"}}]}}}
+ },
+ {
+ "name" : "Table level Aggregation without group by",
+ "hql" : "select count(col1) from index.type",
+ "expectedJson" : {"size":0,"timeout":10000,"aggs":{"filter_wrapper":{"filter":{"match_all":{}},"aggs":{"col1":{"value_count":{"field":"col1"}}}}}}
+ },
+ {
+ "name" : "Table level Aggregation without group by and with complex criteria",
+ "hql" : "select count(col1) from index.type where !(col1=5 and col2='p' and col3='wer' or (col4='9'))",
+ "expectedJson" : {"size":0,"timeout":10000,"aggs":{"filter_wrapper":{"filter":{"not":{"or":[{"and":[{"and":[{"term":{"col1":"5"}},{"term":{"col2":"p"}}]},{"term":{"col3":"wer"}}]},{"term":{"col4":"9"}}]}},"aggs":{"col1":{"value_count":{"field":"col1"}}}}}}
+ },
+ {
+ "name" : "Table level Aggregation - multiple cols without group by and with complex criteria",
+ "hql" : "select count(col1), max(col2) from index.type where !(col1=5 and col2='p' and col3='wer' or (col4='9'))",
+ "expectedJson" : {"size":0,"timeout":10000,"aggs":{"filter_wrapper":{"filter":{"not":{"or":[{"and":[{"and":[{"term":{"col1":"5"}},{"term":{"col2":"p"}}]},{"term":{"col3":"wer"}}]},{"term":{"col4":"9"}}]}},"aggs":{"col1":{"value_count":{"field":"col1"}},"col2":{"max":{"field":"col2"}}}}}}
+ },
+ {
+ "name" : "Group by with aggregation",
+ "hql" : "select col2, count(col1) from index.type group by col2",
+ "expectedJson" : {"size":0,"timeout":10000,"aggs":{"filter_wrapper":{"aggs":{"col2":{"terms":{"field":"col2","size":100},"aggs":{"col1":{"value_count":{"field":"col1"}}}}},"filter":{"match_all":{}}}}}
+ },
+ {
+ "name" : "Group by with aggregation and complex criteria",
+ "hql" : "select col2, count(col1) from index.type where !(col1=5 and col2='p' and col3='wer' or (col4='9')) group by col2",
+ "expectedJson" : {"size":0,"timeout":10000,"aggs":{"filter_wrapper":{"aggs":{"col2":{"terms":{"field":"col2","size":100},"aggs":{"col1":{"value_count":{"field":"col1"}}}}},"filter":{"not":{"or":[{"and":[{"and":[{"term":{"col1":"5"}},{"term":{"col2":"p"}}]},{"term":{"col3":"wer"}}]},{"term":{"col4":"9"}}]}}}}}
+ },
+ {
+ "name" : "Group by with aggregation and complex criteria, select order jumbled",
+ "hql" : "select count(col1), col2 from index.type where !(col1=5 and col2='p' and col3='wer' or (col4='9')) group by col2",
+ "expectedJson" : {"size":0,"timeout":10000,"aggs":{"filter_wrapper":{"aggs":{"col2":{"terms":{"field":"col2","size":100},"aggs":{"col1":{"value_count":{"field":"col1"}}}}},"filter":{"not":{"or":[{"and":[{"and":[{"term":{"col1":"5"}},{"term":{"col2":"p"}}]},{"term":{"col3":"wer"}}]},{"term":{"col4":"9"}}]}}}}}
+ },
+ {
+ "name" : "Group by with multiple aggregation and complex criteria",
+ "hql" : "select col2, max(col3), count(col1) from index.type where !(col1=5 and col2='p' and col3='wer' or (col4='9')) group by col2",
+ "expectedJson" : {"size":0,"timeout":10000,"aggs":{"filter_wrapper":{"aggs":{"col2":{"terms":{"field":"col2","size":100},"aggs":{"col3":{"max":{"field":"col3"}},"col1":{"value_count":{"field":"col1"}}}}},"filter":{"not":{"or":[{"and":[{"and":[{"term":{"col1":"5"}},{"term":{"col2":"p"}}]},{"term":{"col3":"wer"}}]},{"term":{"col4":"9"}}]}}}}}
+ },
+ {
+ "name" : "Group by with multiple aggregation and complex criteria, select order jumbled",
+ "hql" : "select count(col1), col2, max(col3) from index.type where !(col1=5 and col2='p' and col3='wer' or (col4='9')) group by col2",
+ "expectedJson" : {"size":0,"timeout":10000,"aggs":{"filter_wrapper":{"aggs":{"col2":{"terms":{"field":"col2","size":100},"aggs":{"col1":{"value_count":{"field":"col1"}},"col3":{"max":{"field":"col3"}}}}},"filter":{"not":{"or":[{"and":[{"and":[{"term":{"col1":"5"}},{"term":{"col2":"p"}}]},{"term":{"col3":"wer"}}]},{"term":{"col4":"9"}}]}}}}}
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 4732f6d..720825a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -403,6 +403,16 @@ public final class LensConfConstants {
public static final String NATIVE_TABLE_NAME = METASTORE_PFX + "native.table.name";
/**
+ * The Constant ES_INDEX_NAME.
+ */
+ public static final String ES_INDEX_NAME = METASTORE_PFX + "es.index.name";
+
+ /**
+ * The Constant ES_TYPE_NAME.
+ */
+ public static final String ES_TYPE_NAME = METASTORE_PFX + "es.type.name";
+
+ /**
* Gets the service impl conf key.
*
* @param sName the s name
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/lens-server/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index 35e7718..5174607 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -73,6 +73,11 @@
</dependency>
<dependency>
<groupId>org.apache.lens</groupId>
+ <artifactId>lens-driver-es</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lens</groupId>
<artifactId>lens-driver-jdbc</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 99f8c30..3c98077 100644
--- a/pom.xml
+++ b/pom.xml
@@ -48,6 +48,11 @@
<spark.version>1.3.0</spark.version>
<hive.version>0.13.3-inm</hive.version>
+ <!-- jest client for elasticsearch -->
+ <jest.version>0.1.5</jest.version>
+ <httpclient.version>4.4</httpclient.version>
+ <httpcore.version>4.4</httpcore.version>
+
<!-- common library/framework versions -->
<slf4j.version>1.7.6</slf4j.version>
<log4j.version>1.2.16</log4j.version>
@@ -60,6 +65,7 @@
<typesafe.config.version>1.2.1</typesafe.config.version>
<jackson.asl.version>1.9.13</jackson.asl.version>
+ <jackson.guava.version>2.3.3</jackson.guava.version>
<opencsv.version>2.3</opencsv.version>
<jsch.version>0.1.42</jsch.version>
@@ -1220,6 +1226,33 @@
<artifactId>jsch</artifactId>
<version>${jsch.version}</version>
</dependency>
+
+ <!-- jest elastic search deps -->
+ <dependency>
+ <groupId>io.searchbox</groupId>
+ <artifactId>jest</artifactId>
+ <version>${jest.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-guava</artifactId>
+ <version>${jackson.guava.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcore.version}</version>
+ </dependency>
+
+
</dependencies>
</dependencyManagement>
@@ -1232,6 +1265,7 @@
<module>lens-query-lib</module>
<module>lens-driver-hive</module>
<module>lens-driver-jdbc</module>
+ <module>lens-driver-es</module>
<module>lens-server</module>
<module>lens-client</module>
<module>lens-cli</module>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/src/site/apt/admin/config-server.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config-server.apt b/src/site/apt/admin/config-server.apt
index 9349bfc..141f2b3 100644
--- a/src/site/apt/admin/config-server.apt
+++ b/src/site/apt/admin/config-server.apt
@@ -33,14 +33,17 @@ Configuring lens server
The supported drivers are configured through classnames in this release. The configuration
for each driver can be specified in their site.xml file. For example, HiveDriver's
- configuration should be specified in hive-site.xml. Supported drivers in this version
- are org.apache.lens.driver.hive.HiveDriver and org.apache.lens.driver.jdbc.JDBCDriver.
- The configuration parameters and
- their detailed description and default values for HiveDriver
- are available {{{./hivedriver-config.html} here}}
- and for JDBCDriver available {{{./jdbcdriver-config.html} here}}.
- The overridden configuration for HiveDriver and JDBCDriver can be part of hivedriver-site.xml
- and jdbc-driver-site.xml respectively.
+ configuration should be specified in hivedriver-site.xml. Supported drivers in this version
+ are org.apache.lens.driver.hive.HiveDriver, org.apache.lens.driver.jdbc.JDBCDriver
+ and org.apache.lens.driver.es.ESDriver.
+ The configuration parameters and their description are explained in their respective docs listed below
+ HiveDriver - {{{./hivedriver-config.html} here}}
+ JDBCDriver - {{{./jdbcdriver-config.html} here}}
+ ESDriver - {{{./esdriver-config.html} here}}.
+ For overridding configurations, use the appropriate site.xml as listed below
+ hive - hivedriver-site.xml
+ jdbc - jdbc-driver-site.xml
+ es - esdriver-site.xml
* Session configuration
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bffa78c9/src/site/apt/admin/esdriver-config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/esdriver-config.apt b/src/site/apt/admin/esdriver-config.apt
new file mode 100644
index 0000000..ce952ae
--- /dev/null
+++ b/src/site/apt/admin/esdriver-config.apt
@@ -0,0 +1,41 @@
+~~
+~~ Licensed to the Apache Software Foundation (ASF) under one
+~~ or more contributor license agreements. See the NOTICE file
+~~ distributed with this work for additional information
+~~ regarding copyright ownership. The ASF licenses this file
+~~ to you under the Apache License, Version 2.0 (the
+~~ "License"); you may not use this file except in compliance
+~~ with the License. You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing,
+~~ software distributed under the License is distributed on an
+~~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+~~ KIND, either express or implied. See the License for the
+~~ specific language governing permissions and limitations
+~~ under the License.
+~~
+
+ES driver configuration
+
+===
+
+*--+--+---+--+
+|<<No.>>|<<Property Name>>|<<Default Value>>|<<Description>>|
+*--+--+---+--+
+|1|lens.driver.es.aggr.bucket.size|6|Max cardinality of group by (higher value means higher resource usage at server end)|
+*--+--+---+--+
+|2|lens.driver.es.client.class|org.apache.lens.driver.es.client.jest.JestClientImpl|Choice of client class, default is JestClientImpl|
+*--+--+---+--+
+|3|lens.driver.es.jest.max.conn|20|max connections|
+*--+--+---+--+
+|4|lens.driver.es.jest.servers|http://localhost:9200,http://localhost:4200|List of http servers, will be used on a round robin basis|
+*--+--+---+--+
+|5|lens.driver.es.max.row.size|-1|max rows for es document look up queries, non existent or -1 refers no limit|
+*--+--+---+--+
+|6|lens.driver.es.query.timeout.millis|10000|Query timeout|
+*--+--+---+--+
+|7|lens.driver.es.term.fetch.size|10000|Fetch (buffer) size for document look up queries|
+*--+--+---+--+
+The configuration parameters and their default values