You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ap...@apache.org on 2018/01/12 04:45:47 UTC
atlas git commit: ATLAS-2356: Optional where clause.
Repository: atlas
Updated Branches:
refs/heads/master ecf8095ff -> f28d0f54d
ATLAS-2356: Optional where clause.
Signed-off-by: apoorvnaik <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f28d0f54
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f28d0f54
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f28d0f54
Branch: refs/heads/master
Commit: f28d0f54d74a0c6c715d79070986547e6d03d269
Parents: ecf8095
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Thu Jan 11 14:45:29 2018 -0800
Committer: apoorvnaik <ap...@apache.org>
Committed: Thu Jan 11 20:45:32 2018 -0800
----------------------------------------------------------------------
.../atlas/discovery/EntityDiscoveryService.java | 15 +-
.../java/org/apache/atlas/query/AtlasDSL.java | 4 +-
.../java/org/apache/atlas/query/DSLVisitor.java | 68 +++++--
.../atlas/query/GremlinQueryComposer.java | 46 +++--
.../java/org/apache/atlas/query/Lookup.java | 5 +
.../apache/atlas/query/RegistryBasedLookup.java | 5 +
.../org/apache/atlas/query/DSLQueriesTest.java | 86 ++++-----
.../atlas/query/GremlinQueryComposerTest.java | 185 ++++++++++---------
8 files changed, 238 insertions(+), 176 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/f28d0f54/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index ba00782..b691706 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -35,7 +35,6 @@ import org.apache.atlas.model.profile.AtlasUserSavedSearch;
import org.apache.atlas.query.AtlasDSL;
import org.apache.atlas.query.GremlinQuery;
import org.apache.atlas.query.QueryParams;
-import org.apache.atlas.query.antlr4.AtlasDSLParser;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper;
@@ -68,7 +67,6 @@ import javax.script.Bindings;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.util.*;
-import java.util.stream.Stream;
import static org.apache.atlas.AtlasErrorCode.CLASSIFICATION_NOT_FOUND;
import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED;
@@ -897,27 +895,16 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Override
public String getDslQueryUsingTypeNameClassification(String query, String typeName, String classification) {
- final String whereDSLKeyword = "where";
- final String[] keywords = new String[]{whereDSLKeyword, "isa", "is", "limit", "orderby", "has"};
- final String whereFormat = whereDSLKeyword + " %s";
-
String queryStr = query == null ? "" : query;
if (StringUtils.isNotEmpty(typeName)) {
- if(StringUtils.isNotEmpty(query)) {
- String s = query.toLowerCase();
- if(!Stream.of(keywords).anyMatch(x -> s.startsWith(x))) {
- queryStr = String.format(whereFormat, query);
- }
- }
-
queryStr = escapeTypeName(typeName) + " " + queryStr;
}
if (StringUtils.isNotEmpty(classification)) {
// isa works with a type name only - like hive_column isa PII; it doesn't work with more complex query
if (StringUtils.isEmpty(query)) {
- queryStr += String.format("%s %s %s", queryStr, "isa", classification);
+ queryStr += (" isa " + classification);
}
}
return queryStr;
http://git-wip-us.apache.org/repos/asf/atlas/blob/f28d0f54/repository/src/main/java/org/apache/atlas/query/AtlasDSL.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/query/AtlasDSL.java b/repository/src/main/java/org/apache/atlas/query/AtlasDSL.java
index b771447..7817f91 100644
--- a/repository/src/main/java/org/apache/atlas/query/AtlasDSL.java
+++ b/repository/src/main/java/org/apache/atlas/query/AtlasDSL.java
@@ -17,6 +17,7 @@
*/
package org.apache.atlas.query;
+import com.google.common.annotations.VisibleForTesting;
import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
@@ -53,7 +54,8 @@ public class AtlasDSL {
return RESERVED_KEYWORDS.contains(word);
}
- private static AtlasDSLParser.QueryContext parse(String queryStr) throws AtlasBaseException {
+ @VisibleForTesting
+ static AtlasDSLParser.QueryContext parse(String queryStr) throws AtlasBaseException {
AtlasDSLParser.QueryContext ret;
try {
http://git-wip-us.apache.org/repos/asf/atlas/blob/f28d0f54/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java b/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java
index 75be85f..9a213a3 100644
--- a/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java
+++ b/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java
@@ -24,9 +24,9 @@ import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
+
+import static org.apache.atlas.query.antlr4.AtlasDSLParser.RULE_whereClause;
public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
private static final Logger LOG = LoggerFactory.getLogger(DSLVisitor.class);
@@ -34,6 +34,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
private static final String AND = "AND";
private static final String OR = "OR";
+ private Set<Integer> visitedRuleIndexes = new HashSet<>();
private final GremlinQueryComposer gremlinQueryComposer;
public DSLVisitor(GremlinQueryComposer gremlinQueryComposer) {
@@ -41,12 +42,27 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
}
@Override
+ public Void visitSpaceDelimitedQueries(SpaceDelimitedQueriesContext ctx) {
+ addVisitedRule(ctx.getRuleIndex());
+ return super.visitSpaceDelimitedQueries(ctx);
+ }
+
+ @Override
+ public Void visitCommaDelimitedQueries(CommaDelimitedQueriesContext ctx) {
+ addVisitedRule(ctx.getRuleIndex());
+ return super.visitCommaDelimitedQueries(ctx);
+ }
+
+ @Override
public Void visitIsClause(IsClauseContext ctx) {
if (LOG.isDebugEnabled()) {
LOG.debug("=> DSLVisitor.visitIsClause({})", ctx);
}
- gremlinQueryComposer.addFromIsA(ctx.arithE().getText(), ctx.identifier().getText());
+ if(!hasVisitedRule(RULE_whereClause)) {
+ gremlinQueryComposer.addFromIsA(ctx.arithE().getText(), ctx.identifier().getText());
+ }
+
return super.visitIsClause(ctx);
}
@@ -56,7 +72,10 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
LOG.debug("=> DSLVisitor.visitHasClause({})", ctx);
}
- gremlinQueryComposer.addFromProperty(ctx.arithE().getText(), ctx.identifier().getText());
+ if(!hasVisitedRule(RULE_whereClause)) {
+ gremlinQueryComposer.addFromProperty(ctx.arithE().getText(), ctx.identifier().getText());
+ }
+
return super.visitHasClause(ctx);
}
@@ -140,10 +159,8 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
LOG.debug("=> DSLVisitor.visitWhereClause({})", ctx);
}
-
- // The first expr shouldn't be processed if there are following exprs
+ addVisitedRule(ctx.getRuleIndex());
ExprContext expr = ctx.expr();
-
processExpr(expr, gremlinQueryComposer);
return super.visitWhereClause(ctx);
}
@@ -170,6 +187,19 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
}
@Override
+ public Void visitSingleQrySrc(SingleQrySrcContext ctx) {
+ if (!hasVisitedRule(RULE_whereClause)) {
+ if (ctx.fromExpression() == null) {
+ if (ctx.expr() != null && gremlinQueryComposer.hasFromClause()) {
+ processExpr(ctx.expr(), gremlinQueryComposer);
+ }
+ }
+ }
+
+ return super.visitSingleQrySrc(ctx);
+ }
+
+ @Override
public Void visitGroupByExpression(GroupByExpressionContext ctx) {
if (LOG.isDebugEnabled()) {
LOG.debug("=> DSLVisitor.visitGroupByExpression({})", ctx);
@@ -238,11 +268,11 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
}
private void processExpr(final CompEContext compE, final GremlinQueryComposer gremlinQueryComposer) {
- if (compE != null && compE.isClause() == null && compE.hasClause() == null && compE.isClause() == null) {
+ if (compE != null && compE.isClause() == null && compE.hasClause() == null) {
ComparisonClauseContext comparisonClause = compE.comparisonClause();
// The nested expression might have ANDs/ORs
- if(comparisonClause == null) {
+ if (comparisonClause == null) {
ExprContext exprContext = compE.arithE().multiE().atomE().expr();
// Only extract comparison clause if there are no nested exprRight clauses
if (CollectionUtils.isEmpty(exprContext.exprRight())) {
@@ -254,7 +284,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
String lhs = comparisonClause.arithE(0).getText();
String op, rhs;
AtomEContext atomECtx = comparisonClause.arithE(1).multiE().atomE();
- if(atomECtx.literal() == null ||
+ if (atomECtx.literal() == null ||
(atomECtx.literal() != null && atomECtx.literal().valueArray() == null)) {
op = comparisonClause.operator().getText().toUpperCase();
rhs = comparisonClause.arithE(1).getText();
@@ -268,6 +298,14 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
processExpr(compE.arithE().multiE().atomE().expr(), gremlinQueryComposer);
}
}
+
+ if (compE != null && compE.isClause() != null) {
+ gremlinQueryComposer.addFromIsA(compE.isClause().arithE().getText(), compE.isClause().identifier().getText());
+ }
+
+ if (compE != null && compE.hasClause() != null) {
+ gremlinQueryComposer.addFromProperty(compE.hasClause().arithE().getText(), compE.hasClause().identifier().getText());
+ }
}
private String getInClause(AtomEContext atomEContext) {
@@ -281,4 +319,12 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> {
return sb.toString();
}
+
+ private void addVisitedRule(int ruleIndex) {
+ visitedRuleIndexes.add(ruleIndex);
+ }
+
+ private boolean hasVisitedRule(int ruleIndex) {
+ return visitedRuleIndexes.contains(ruleIndex);
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f28d0f54/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
index 76f31a3..fe2d6d7 100644
--- a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
+++ b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
@@ -56,16 +56,13 @@ public class GremlinQueryComposer {
private int providedOffset = DEFAULT_QUERY_RESULT_OFFSET;
private Context context;
- private static final ThreadLocal<DateFormat> DSL_DATE_FORMAT = new ThreadLocal<DateFormat>() {
- @Override
- public DateFormat initialValue() {
- DateFormat ret = new SimpleDateFormat(ISO8601_FORMAT);
+ private static final ThreadLocal<DateFormat> DSL_DATE_FORMAT = ThreadLocal.withInitial(() -> {
+ DateFormat ret = new SimpleDateFormat(ISO8601_FORMAT);
- ret.setTimeZone(TimeZone.getTimeZone("UTC"));
+ ret.setTimeZone(TimeZone.getTimeZone("UTC"));
- return ret;
- }
- };
+ return ret;
+ });
public GremlinQueryComposer(Lookup registryLookup, final AtlasDSL.QueryMetadata qmd, boolean isNestedQuery) {
this.isNestedQuery = isNestedQuery;
@@ -124,13 +121,19 @@ public class GremlinQueryComposer {
LOG.debug("addFromProperty(typeName={}, attribute={})", typeName, attribute);
}
- addFrom(typeName);
+ if(!isNestedQuery) {
+ addFrom(typeName);
+ }
+
add(GremlinClause.HAS_PROPERTY,
IdentifierHelper.getQualifiedName(lookup, context, attribute));
}
public void addFromIsA(String typeName, String traitName) {
- addFrom(typeName);
+ if (!isNestedQuery) {
+ addFrom(typeName);
+ }
+
add(GremlinClause.TRAIT, traitName);
}
@@ -284,6 +287,15 @@ public class GremlinQueryComposer {
return s;
}
+ public List<String> getErrorList() {
+ combineErrorLists();
+ return errorList;
+ }
+
+ private void combineErrorLists() {
+ errorList.addAll(context.getErrorList());
+ }
+
private String getTransformedClauses(String[] items) {
String ret;
String body = String.join(".", Stream.of(items).filter(Objects::nonNull).collect(Collectors.toList()));
@@ -406,7 +418,7 @@ public class GremlinQueryComposer {
}
private void moveToLast(GremlinClause clause) {
- int index = queryClauses.hasClause(clause);
+ int index = queryClauses.contains(clause);
if (-1 == index) {
return;
}
@@ -475,6 +487,11 @@ public class GremlinQueryComposer {
add(GremlinClause.GROUP_BY, ia.getQualifiedName());
}
+ public boolean hasFromClause() {
+ return queryClauses.contains(GremlinClause.HAS_TYPE) != -1 ||
+ queryClauses.contains(GremlinClause.HAS_TYPE_WITHIN) != -1;
+ }
+
private static class GremlinClauseValue {
private final GremlinClause clause;
private final String value;
@@ -540,7 +557,7 @@ public class GremlinQueryComposer {
return list.size();
}
- public int hasClause(GremlinClause clause) {
+ public int contains(GremlinClause clause) {
for (int i = 0; i < list.size(); i++) {
if (list.get(i).getClause() == clause)
return i;
@@ -640,5 +657,10 @@ public class GremlinQueryComposer {
aliasMap.put(alias, typeName);
}
+
+ public List<String> getErrorList() {
+ errorList.addAll(lookup.getErrorList());
+ return errorList;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f28d0f54/repository/src/main/java/org/apache/atlas/query/Lookup.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/query/Lookup.java b/repository/src/main/java/org/apache/atlas/query/Lookup.java
index d09b207..fa75cfb 100644
--- a/repository/src/main/java/org/apache/atlas/query/Lookup.java
+++ b/repository/src/main/java/org/apache/atlas/query/Lookup.java
@@ -20,6 +20,9 @@ package org.apache.atlas.query;
import org.apache.atlas.type.AtlasType;
+import java.util.Collection;
+import java.util.List;
+
public interface Lookup {
AtlasType getType(String typeName);
@@ -40,4 +43,6 @@ public interface Lookup {
String getTypeFromEdge(GremlinQueryComposer.Context context, String item);
boolean isDate(GremlinQueryComposer.Context context, String attributeName);
+
+ List<String> getErrorList();
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f28d0f54/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java b/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
index 0f4316f..b78bd2e 100644
--- a/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
+++ b/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
@@ -159,4 +159,9 @@ class RegistryBasedLookup implements Lookup {
protected void addError(String s) {
errorList.add(s);
}
+
+ @Override
+ public List<String> getErrorList() {
+ return errorList;
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f28d0f54/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java b/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
index f3afa34..5db55dc 100644
--- a/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
+++ b/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
@@ -17,19 +17,12 @@
*/
package org.apache.atlas.query;
-import org.antlr.v4.runtime.CharStreams;
-import org.antlr.v4.runtime.CommonTokenStream;
-import org.antlr.v4.runtime.TokenStream;
import org.apache.atlas.TestModules;
import org.apache.atlas.discovery.EntityDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
-import org.apache.atlas.query.antlr4.AtlasDSLLexer;
-import org.apache.atlas.query.antlr4.AtlasDSLParser;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.testng.ITestContext;
import org.testng.annotations.*;
import javax.inject.Inject;
@@ -38,13 +31,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
@@ -120,6 +106,9 @@ public class DSLQueriesTest extends BasicTestSetup {
public void comparison(String query, int expected) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
assertSearchResult(searchResult, expected);
+
+ AtlasSearchResult searchResult2 = discoveryService.searchUsingDslQuery(query.replace("where", " "), 25, 0);
+ assertSearchResult(searchResult2, expected);
}
@DataProvider(name = "basicProvider")
@@ -128,7 +117,6 @@ public class DSLQueriesTest extends BasicTestSetup {
{"from hive_db", 3},
{"hive_db", 3},
{"hive_db where hive_db.name=\"Reporting\"", 1},
- {"hive_db hive_db.name = \"Reporting\"", 3},
{"hive_db where hive_db.name=\"Reporting\" select name, owner", 1},
{"hive_db has name", 3},
{"from hive_table", 10},
@@ -155,8 +143,6 @@ public class DSLQueriesTest extends BasicTestSetup {
{"`Log Data`", 3},
{"`isa`", 0},
{"hive_table as t, sd, hive_column as c where t.name=\"sales_fact\" select c.name as colName, c.dataType as colType", 0},
- {"hive_table where name='sales_fact', db where name='Sales'", 1},
- {"hive_table where name='sales_fact', db where name='Reporting'", 0},
{"DataSet where name='sales_fact'", 1},
{"Asset where name='sales_fact'", 1}
};
@@ -164,6 +150,11 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "basicProvider")
public void basic(String query, int expected) throws AtlasBaseException {
+ queryAssert(query, expected);
+ //queryAssert(query.replace("where", " "), expected);
+ }
+
+ private void queryAssert(String query, int expected) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
assertSearchResult(searchResult, expected);
}
@@ -182,8 +173,8 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "limitProvider")
public void limit(String query, int expected, int limit, int offset) throws AtlasBaseException {
- AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, limit, offset);
- assertSearchResult(searchResult, expected);
+ queryAssert(query, expected);
+ queryAssert(query.replace("where", " "), expected);
}
@DataProvider(name = "syntaxVerifierProvider")
@@ -268,25 +259,16 @@ public class DSLQueriesTest extends BasicTestSetup {
{"hive_table where (name = \"sales_fact\" and createTime >= \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, createTime as _col_1 limit 10 offset 1", 0},
{"hive_table where (name = \"sales_fact\" and createTime >= \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, createTime as _col_1 limit 10", 1},
{"hive_table where (name = \"sales_fact\" and createTime >= \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, createTime as _col_1 limit 0 offset 1", 0},
-
- {"hive_table where name='sales_fact', db where name='Sales'", 1},
- {"hive_table where name='sales_fact', db where name='Sales' limit 10", 1},
- {"hive_table where name='sales_fact', db where name='Sales' limit 10 offset 1", 0},
- {"hive_table where name='sales_fact', db where name='Reporting'", 0},
- {"hive_table where name='sales_fact', db where name='Reporting' limit 10", 0},
- {"hive_table where name='sales_fact', db where name='Reporting' limit 10 offset 1", 0},
-
{"hive_db as d where owner = ['John ETL', 'Jane BI']", 2},
{"hive_db as d where owner = ['John ETL', 'Jane BI'] limit 10", 2},
{"hive_db as d where owner = ['John ETL', 'Jane BI'] limit 10 offset 1", 1},
-
};
}
@Test(dataProvider = "syntaxVerifierProvider")
public void syntax(String query, int expected) throws AtlasBaseException {
- AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
- assertSearchResult(searchResult, expected);
+ queryAssert(query, expected);
+ queryAssert(query.replace("where", " "), expected);
}
@DataProvider(name = "orderByProvider")
@@ -365,8 +347,8 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "orderByProvider")
public void orderBy(String query, int expected, String orderBy, boolean ascending) throws AtlasBaseException {
- AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
- assertSearchResult(searchResult, expected);
+ queryAssert(query, expected);
+ queryAssert(query.replace("where", " "), expected);
}
@DataProvider(name = "likeQueriesProvider")
@@ -376,13 +358,18 @@ public class DSLQueriesTest extends BasicTestSetup {
{"hive_db where name like \"R*\"", 1},
{"hive_db where hive_db.name like \"R???rt?*\" or hive_db.name like \"S?l?s\" or hive_db.name like\"Log*\"", 3},
{"hive_db where hive_db.name like \"R???rt?*\" and hive_db.name like \"S?l?s\" and hive_db.name like\"Log*\"", 0},
- {"hive_table where name like 'sales*', db where name like 'Sa?es'", 1},
{"hive_table where name like 'sales*' and db.name like 'Sa?es'", 1},
{"hive_table where db.name like \"Sa*\"", 4},
{"hive_table where db.name like \"Sa*\" and name like \"*dim\"", 3},
};
}
+ @Test(dataProvider = "likeQueriesProvider")
+ public void likeQueries(String query, int expected) throws AtlasBaseException {
+ queryAssert(query, expected);
+ queryAssert(query.replace("where", " "), expected);
+ }
+
@DataProvider(name = "minMaxCountProvider")
private Object[][] minMaxCountQueries() {
return new Object[][]{
@@ -499,14 +486,26 @@ public class DSLQueriesTest extends BasicTestSetup {
@Test(dataProvider = "minMaxCountProvider")
public void minMaxCount(String query, FieldValueValidator fv) throws AtlasBaseException {
+ queryAssert(query, fv);
+ queryAssert(query.replace("where", " "), fv);
+ }
+
+ private void queryAssert(String query, FieldValueValidator fv) throws AtlasBaseException {
AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
assertSearchResult(searchResult, fv);
}
- @Test(dataProvider = "likeQueriesProvider")
- public void likeQueries(String query, int expected) throws AtlasBaseException {
- AtlasSearchResult searchResult = discoveryService.searchUsingDslQuery(query, 25, 0);
- assertSearchResult(searchResult, expected);
+ private void assertSearchResult(AtlasSearchResult searchResult, FieldValueValidator expected) {
+ assertNotNull(searchResult);
+ assertNull(searchResult.getEntities());
+
+ assertEquals(searchResult.getAttributes().getName().size(), expected.getFieldNamesCount());
+ for (int i = 0; i < searchResult.getAttributes().getName().size(); i++) {
+ String s = searchResult.getAttributes().getName().get(i);
+ assertEquals(s, expected.fieldNames[i]);
+ }
+
+ assertEquals(searchResult.getAttributes().getValues().size(), expected.values.size());
}
private void assertSearchResult(AtlasSearchResult searchResult, int expected) {
@@ -523,19 +522,6 @@ public class DSLQueriesTest extends BasicTestSetup {
}
}
- private void assertSearchResult(AtlasSearchResult searchResult, FieldValueValidator expected) {
- assertNotNull(searchResult);
- assertNull(searchResult.getEntities());
-
- assertEquals(searchResult.getAttributes().getName().size(), expected.getFieldNamesCount());
- for (int i = 0; i < searchResult.getAttributes().getName().size(); i++) {
- String s = searchResult.getAttributes().getName().get(i);
- assertEquals(s, expected.fieldNames[i]);
- }
-
- assertEquals(searchResult.getAttributes().getValues().size(), expected.values.size());
- }
-
private class FieldValueValidator {
class ResultObject {
Map<String, Object> fieldValues = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/atlas/blob/f28d0f54/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java b/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
index 295509b..3fcd325 100644
--- a/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
+++ b/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
@@ -17,29 +17,22 @@
*/
package org.apache.atlas.query;
-import org.antlr.v4.runtime.CharStreams;
-import org.antlr.v4.runtime.CommonTokenStream;
-import org.antlr.v4.runtime.TokenStream;
-import org.apache.atlas.query.antlr4.AtlasDSLLexer;
+import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.query.antlr4.AtlasDSLParser;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang.StringUtils;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.assertFalse;
public class GremlinQueryComposerTest {
private List<String> errorList = new ArrayList<>();
@@ -103,6 +96,8 @@ public class GremlinQueryComposerTest {
@Test
public void valueArray() {
verify("DB where owner = ['hdfs', 'anon']", "g.V().has('__typeName', 'DB').has('DB.owner', within('hdfs','anon')).limit(25).toList()");
+ verify("DB owner = ['hdfs', 'anon']", "g.V().has('__typeName', 'DB').has('DB.owner', within('hdfs','anon')).limit(25).toList()");
+ verify("hive_db as d owner = ['hdfs', 'anon']", "g.V().has('__typeName', 'hive_db').as('d').has('hive_db.owner', within('hdfs','anon')).limit(25).toList()");
}
@Test
public void groupByMin() {
@@ -222,71 +217,90 @@ public class GremlinQueryComposerTest {
verify(String.format(queryFormat, "FALSE"), String.format(expectedFormat, "FALSE"));
}
+ @DataProvider(name = "nestedQueriesProvider")
+ private Object[][] nestedQueriesSource() {
+ return new Object[][]{
+ {"Table where name=\"sales_fact\" or name=\"testtable_1\"",
+ "g.V().has('__typeName', 'Table').or(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.name', eq(\"testtable_1\"))).limit(25).toList()"},
+ {"Table where name=\"sales_fact\" and name=\"testtable_1\"",
+ "g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.name', eq(\"testtable_1\"))).limit(25).toList()"},
+ {"Table where name=\"sales_fact\" or name=\"testtable_1\" or name=\"testtable_2\"",
+ "g.V().has('__typeName', 'Table')" +
+ ".or(" +
+ "__.has('Table.name', eq(\"sales_fact\"))," +
+ "__.has('Table.name', eq(\"testtable_1\"))," +
+ "__.has('Table.name', eq(\"testtable_2\"))" +
+ ").limit(25).toList()"},
+ {"Table where name=\"sales_fact\" and name=\"testtable_1\" and name=\"testtable_2\"",
+ "g.V().has('__typeName', 'Table')" +
+ ".and(" +
+ "__.has('Table.name', eq(\"sales_fact\"))," +
+ "__.has('Table.name', eq(\"testtable_1\"))," +
+ "__.has('Table.name', eq(\"testtable_2\"))" +
+ ").limit(25).toList()"},
+ {"Table where (name=\"sales_fact\" or name=\"testtable_1\") and name=\"testtable_2\"",
+ "g.V().has('__typeName', 'Table')" +
+ ".and(" +
+ "__.or(" +
+ "__.has('Table.name', eq(\"sales_fact\"))," +
+ "__.has('Table.name', eq(\"testtable_1\"))" +
+ ")," +
+ "__.has('Table.name', eq(\"testtable_2\")))" +
+ ".limit(25).toList()"},
+ {"Table where name=\"sales_fact\" or (name=\"testtable_1\" and name=\"testtable_2\")",
+ "g.V().has('__typeName', 'Table')" +
+ ".or(" +
+ "__.has('Table.name', eq(\"sales_fact\"))," +
+ "__.and(" +
+ "__.has('Table.name', eq(\"testtable_1\"))," +
+ "__.has('Table.name', eq(\"testtable_2\")))" +
+ ")" +
+ ".limit(25).toList()"},
+ {"Table where name=\"sales_fact\" or name=\"testtable_1\" and name=\"testtable_2\"",
+ "g.V().has('__typeName', 'Table')" +
+ ".and(" +
+ "__.or(" +
+ "__.has('Table.name', eq(\"sales_fact\"))," +
+ "__.has('Table.name', eq(\"testtable_1\"))" +
+ ")," +
+ "__.has('Table.name', eq(\"testtable_2\")))" +
+ ".limit(25).toList()"},
+ {"Table where (name=\"sales_fact\" and owner=\"Joe\") OR (name=\"sales_fact_daily_mv\" and owner=\"Joe BI\")",
+ "g.V().has('__typeName', 'Table')" +
+ ".or(" +
+ "__.and(" +
+ "__.has('Table.name', eq(\"sales_fact\"))," +
+ "__.has('Table.owner', eq(\"Joe\"))" +
+ ")," +
+ "__.and(" +
+ "__.has('Table.name', eq(\"sales_fact_daily_mv\"))," +
+ "__.has('Table.owner', eq(\"Joe BI\"))" +
+ "))" +
+ ".limit(25).toList()"},
+ {"Table where owner=\"hdfs\" or ((name=\"testtable_1\" or name=\"testtable_2\") and createdTime < \"2017-12-12T02:35:58.440Z\")",
+ "g.V().has('__typeName', 'Table').or(__.has('Table.owner', eq(\"hdfs\")),__.and(__.or(__.has('Table.name', eq(\"testtable_1\")),__.has('Table.name', eq(\"testtable_2\"))),__.has('Table.createdTime', lt('1513046158440')))).limit(25).toList()"},
+ {"hive_db where hive_db.name='Reporting' and hive_db.createTime < '2017-12-12T02:35:58.440Z'",
+ "g.V().has('__typeName', 'hive_db').and(__.has('hive_db.name', eq('Reporting')),__.has('hive_db.createTime', lt('1513046158440'))).limit(25).toList()"},
+ {"Table where db.name='Sales' and db.clusterName='cl1'",
+ "g.V().has('__typeName', 'Table').and(__.out('__Table.db').has('DB.name', eq('Sales')).dedup().in('__Table.db'),__.out('__Table.db').has('DB.clusterName', eq('cl1')).dedup().in('__Table.db')).limit(25).toList()"},
+ };
+ }
+
+ @Test(dataProvider = "nestedQueriesProvider")
+ public void nestedQueries(String query, String expectedGremlin) {
+ verify(query, expectedGremlin);
+ verify(query.replace("where", " "), expectedGremlin);
+ }
+
@Test
- public void nestedQueries() {
- verify("Table where name=\"sales_fact\" or name=\"testtable_1\"",
- "g.V().has('__typeName', 'Table').or(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.name', eq(\"testtable_1\"))).limit(25).toList()");
- verify("Table where name=\"sales_fact\" and name=\"testtable_1\"",
- "g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.name', eq(\"testtable_1\"))).limit(25).toList()");
- verify("Table where name=\"sales_fact\" or name=\"testtable_1\" or name=\"testtable_2\"",
- "g.V().has('__typeName', 'Table')" +
- ".or(" +
- "__.has('Table.name', eq(\"sales_fact\"))," +
- "__.has('Table.name', eq(\"testtable_1\"))," +
- "__.has('Table.name', eq(\"testtable_2\"))" +
- ").limit(25).toList()");
- verify("Table where name=\"sales_fact\" and name=\"testtable_1\" and name=\"testtable_2\"",
- "g.V().has('__typeName', 'Table')" +
- ".and(" +
- "__.has('Table.name', eq(\"sales_fact\"))," +
- "__.has('Table.name', eq(\"testtable_1\"))," +
- "__.has('Table.name', eq(\"testtable_2\"))" +
- ").limit(25).toList()");
- verify("Table where (name=\"sales_fact\" or name=\"testtable_1\") and name=\"testtable_2\"",
- "g.V().has('__typeName', 'Table')" +
- ".and(" +
- "__.or(" +
- "__.has('Table.name', eq(\"sales_fact\"))," +
- "__.has('Table.name', eq(\"testtable_1\"))" +
- ")," +
- "__.has('Table.name', eq(\"testtable_2\")))" +
- ".limit(25).toList()");
- verify("Table where name=\"sales_fact\" or (name=\"testtable_1\" and name=\"testtable_2\")",
- "g.V().has('__typeName', 'Table')" +
- ".or(" +
- "__.has('Table.name', eq(\"sales_fact\"))," +
- "__.and(" +
- "__.has('Table.name', eq(\"testtable_1\"))," +
- "__.has('Table.name', eq(\"testtable_2\")))" +
- ")" +
- ".limit(25).toList()");
- verify("Table where name=\"sales_fact\" or name=\"testtable_1\" and name=\"testtable_2\"",
- "g.V().has('__typeName', 'Table')" +
- ".and(" +
- "__.or(" +
- "__.has('Table.name', eq(\"sales_fact\"))," +
- "__.has('Table.name', eq(\"testtable_1\"))" +
- ")," +
- "__.has('Table.name', eq(\"testtable_2\")))" +
- ".limit(25).toList()");
- verify("Table where (name=\"sales_fact\" and owner=\"Joe\") OR (name=\"sales_fact_daily_mv\" and owner=\"Joe BI\")",
- "g.V().has('__typeName', 'Table')" +
- ".or(" +
- "__.and(" +
- "__.has('Table.name', eq(\"sales_fact\"))," +
- "__.has('Table.owner', eq(\"Joe\"))" +
- ")," +
- "__.and(" +
- "__.has('Table.name', eq(\"sales_fact_daily_mv\"))," +
- "__.has('Table.owner', eq(\"Joe BI\"))" +
- "))" +
- ".limit(25).toList()");
- verify("Table where owner=\"hdfs\" or ((name=\"testtable_1\" or name=\"testtable_2\") and createdTime < \"2017-12-12T02:35:58.440Z\")",
- "g.V().has('__typeName', 'Table').or(__.has('Table.owner', eq(\"hdfs\")),__.and(__.or(__.has('Table.name', eq(\"testtable_1\")),__.has('Table.name', eq(\"testtable_2\"))),__.has('Table.createdTime', lt('1513046158440')))).limit(25).toList()");
- verify("hive_db where hive_db.name='Reporting' and hive_db.createTime < '2017-12-12T02:35:58.440Z'",
- "g.V().has('__typeName', 'hive_db').and(__.has('hive_db.name', eq('Reporting')),__.has('hive_db.createTime', lt('1513046158440'))).limit(25).toList()");
- verify("Table where db.name='Sales' and db.clusterName='cl1'",
- "g.V().has('__typeName', 'Table').and(__.out('__Table.db').has('DB.name', eq('Sales')).dedup().in('__Table.db'),__.out('__Table.db').has('DB.clusterName', eq('cl1')).dedup().in('__Table.db')).limit(25).toList()");
+ public void hasInWhereClause() {
+ verify("Table as t where t has name and t isa Dimension",
+ "g.V().has('__typeName', 'Table').as('t').and(__.has('Table.name'),__.has('__traitNames', within('Dimension'))).limit(25).toList()");
+ verify("Table as t where t has name and t.name = 'sales_fact'",
+ "g.V().has('__typeName', 'Table').as('t').and(__.has('Table.name'),__.has('Table.name', eq('sales_fact'))).limit(25).toList()");
+ verify("Table as t where t is Dimension and t.name = 'sales_fact'",
+ "g.V().has('__typeName', 'Table').as('t').and(__.has('__traitNames', within('Dimension')),__.has('Table.name', eq('sales_fact'))).limit(25).toList()");
+ verify("Table isa 'Dimension' and t.name = 'sales_fact'", "g.V().has('__typeName', 'Table').has('__traitNames', within(''Dimension'')).limit(25).toList()");
}
@Test
@@ -305,23 +319,13 @@ public class GremlinQueryComposerTest {
}
private AtlasDSLParser.QueryContext getParsedQuery(String query) {
+ AtlasDSL.Parser parser = new AtlasDSL.Parser();
AtlasDSLParser.QueryContext queryContext = null;
- InputStream stream = new ByteArrayInputStream(query.getBytes());
- AtlasDSLLexer lexer = null;
-
try {
- lexer = new AtlasDSLLexer(CharStreams.fromStream(stream));
- } catch (IOException e) {
- assertTrue(false);
+ queryContext = parser.parse(query);
+ } catch (AtlasBaseException e) {
+ assertFalse(e != null, e.getMessage());
}
-
- TokenStream inputTokenStream = new CommonTokenStream(lexer);
- AtlasDSLParser parser = new AtlasDSLParser(inputTokenStream);
- queryContext = parser.query();
-
- assertNotNull(queryContext);
- assertNull(queryContext.exception);
-
return queryContext;
}
@@ -430,5 +434,10 @@ public class GremlinQueryComposerTest {
return attributeName.equals("createdTime") ||
attributeName.equals("createTime");
}
+
+ @Override
+ public List<String> getErrorList() {
+ return errorList;
+ }
}
}