You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ap...@apache.org on 2018/02/01 06:16:08 UTC
atlas git commit: ATLAS-2430: Addressed close for nested queries.
Repository: atlas
Updated Branches:
refs/heads/master 134d60fb2 -> ba4df832e
ATLAS-2430: Addressed close for nested queries.
Signed-off-by: apoorvnaik <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/ba4df832
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/ba4df832
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/ba4df832
Branch: refs/heads/master
Commit: ba4df832e96f75459a137f83a75d821701b857e6
Parents: 134d60f
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Wed Jan 31 22:15:56 2018 -0800
Committer: apoorvnaik <ap...@apache.org>
Committed: Wed Jan 31 22:15:56 2018 -0800
----------------------------------------------------------------------
.../atlas/query/GremlinQueryComposer.java | 93 ++++++++++----------
.../apache/atlas/query/IdentifierHelper.java | 16 ++--
.../org/apache/atlas/query/DSLQueriesTest.java | 1 +
.../atlas/query/GremlinQueryComposerTest.java | 8 ++
4 files changed, 62 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/ba4df832/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
index c2286b4..386eec0 100644
--- a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
+++ b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
@@ -50,22 +50,22 @@ public class GremlinQueryComposer {
private static final Logger LOG = LoggerFactory.getLogger(GremlinQueryComposer.class);
private static final String ISO8601_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
private static final String ISO8601_DATE_FORMAT = "yyyy-MM-dd";
-
+
private static final ThreadLocal<DateFormat[]> DSL_DATE_FORMAT = ThreadLocal.withInitial(() -> {
final String formats[] = {ISO8601_FORMAT, ISO8601_DATE_FORMAT};
- DateFormat[] dfs = new DateFormat[formats.length];
+ DateFormat[] dfs = new DateFormat[formats.length];
for (int i = 0; i < formats.length; i++) {
dfs[i] = new SimpleDateFormat(formats[i]);
dfs[i].setTimeZone(TimeZone.getTimeZone("UTC"));
- }
+ }
return dfs;
});
-
- private final String EMPTY_STRING = "";
- private final int DEFAULT_QUERY_RESULT_LIMIT = 25;
- private final int DEFAULT_QUERY_RESULT_OFFSET = 0;
- private final GremlinClauseList queryClauses = new GremlinClauseList();
- private final Set<String> attributesProcessed = new HashSet<>();
+
+ private final String EMPTY_STRING = "";
+ private final int DEFAULT_QUERY_RESULT_LIMIT = 25;
+ private final int DEFAULT_QUERY_RESULT_OFFSET = 0;
+ private final GremlinClauseList queryClauses = new GremlinClauseList();
+ private final Set<String> attributesProcessed = new HashSet<>();
private final Lookup lookup;
private final boolean isNestedQuery;
private final AtlasDSL.QueryMetadata queryMetadata;
@@ -104,12 +104,12 @@ public class GremlinQueryComposer {
LOG.debug("addFrom(typeName={})", typeName);
}
- IdentifierHelper.IdentifierMetadata ta = getIdMetadata(typeName);
+ IdentifierHelper.Info typeInfo = createInfo(typeName);
- if (context.shouldRegister(ta.get())) {
- context.registerActive(ta.get());
+ if (context.shouldRegister(typeInfo.get())) {
+ context.registerActive(typeInfo.get());
- IdentifierHelper.IdentifierMetadata ia = getIdMetadata(ta.get());
+ IdentifierHelper.Info ia = createInfo(typeInfo.get());
if (ia.isTrait()) {
add(GremlinClause.TRAIT, ia);
@@ -121,7 +121,7 @@ public class GremlinQueryComposer {
}
}
} else {
- IdentifierHelper.IdentifierMetadata ia = getIdMetadata(ta.get());
+ IdentifierHelper.Info ia = createInfo(typeInfo.get());
introduceType(ia);
}
}
@@ -135,7 +135,7 @@ public class GremlinQueryComposer {
addFrom(typeName);
}
- add(GremlinClause.HAS_PROPERTY, getIdMetadata(attribute));
+ add(GremlinClause.HAS_PROPERTY, createInfo(attribute));
}
public void addIsA(String typeName, String traitName) {
@@ -143,8 +143,8 @@ public class GremlinQueryComposer {
addFrom(typeName);
}
- IdentifierHelper.IdentifierMetadata ia = getIdMetadata(traitName);
- add(GremlinClause.TRAIT, ia);
+ IdentifierHelper.Info traitInfo = createInfo(traitName);
+ add(GremlinClause.TRAIT, traitInfo);
}
public void addWhere(String lhs, String operator, String rhs) {
@@ -152,13 +152,13 @@ public class GremlinQueryComposer {
LOG.debug("addWhere(lhs={}, operator={}, rhs={})", lhs, operator, rhs);
}
- String currentType = context.getActiveTypeName();
- IdentifierHelper.IdentifierMetadata org = null;
- IdentifierHelper.IdentifierMetadata lhsI = getIdMetadata(lhs);
+ String currentType = context.getActiveTypeName();
+ IdentifierHelper.Info org = null;
+ IdentifierHelper.Info lhsI = createInfo(lhs);
if (!lhsI.isPrimitive()) {
introduceType(lhsI);
org = lhsI;
- lhsI = getIdMetadata(lhs);
+ lhsI = createInfo(lhs);
}
if (!context.validator.isValidQualifiedName(lhsI.getQualifiedName(), lhsI.getRaw())) {
@@ -204,10 +204,6 @@ public class GremlinQueryComposer {
this.attributesProcessed.addAll(attributesProcessed);
}
- public void addProcessedAttribute(String attribute) {
- attributesProcessed.add(attribute);
- }
-
public void addSelect(SelectClauseComposer selectClauseComposer) {
process(selectClauseComposer);
@@ -215,7 +211,6 @@ public class GremlinQueryComposer {
addSelectAttrExistsCheck(selectClauseComposer);
}
-
// If the query contains orderBy and groupBy then the transformation determination is deferred to the method processing orderBy
if (!(queryMetadata.hasOrderBy() && queryMetadata.hasGroupBy())) {
addSelectTransformation(selectClauseComposer, null, false);
@@ -277,8 +272,9 @@ public class GremlinQueryComposer {
public String get() {
close();
- String items[] = getFormattedClauses(queryMetadata.needTransformation());
- String s = queryMetadata.needTransformation() ?
+ boolean mustTransform = !isNestedQuery && queryMetadata.needTransformation();
+ String items[] = getFormattedClauses(mustTransform);
+ String s = mustTransform ?
getTransformedClauses(items) :
String.join(".", items);
@@ -298,7 +294,7 @@ public class GremlinQueryComposer {
LOG.debug("addOrderBy(name={}, isDesc={})", name, isDesc);
}
- IdentifierHelper.IdentifierMetadata ia = getIdMetadata(name);
+ IdentifierHelper.Info ia = createInfo(name);
if (queryMetadata.hasSelect() && queryMetadata.hasGroupBy()) {
addSelectTransformation(this.context.selectClauseComposer, getQualifiedName(ia), isDesc);
} else if (queryMetadata.hasGroupBy()) {
@@ -314,7 +310,8 @@ public class GremlinQueryComposer {
for (DateFormat dateFormat : DSL_DATE_FORMAT.get()) {
try {
return dateFormat.parse(s).getTime();
- } catch (ParseException ignored) {}
+ } catch (ParseException ignored) {
+ }
}
context.validator.check(false, AtlasErrorCode.INVALID_DSL_INVALID_DATE, s);
@@ -326,7 +323,7 @@ public class GremlinQueryComposer {
queryClauses.contains(GremlinClause.HAS_TYPE_WITHIN) != -1;
}
- private String getQualifiedName(IdentifierHelper.IdentifierMetadata ia) {
+ private String getQualifiedName(IdentifierHelper.Info ia) {
return context.validator.isValidQualifiedName(ia.getQualifiedName(), ia.getRaw()) ?
ia.getQualifiedName() : ia.getRaw();
}
@@ -337,8 +334,8 @@ public class GremlinQueryComposer {
String[] qualifiedAttributes = selectClauseComposer.getAttributes();
if (qualifiedAttributes != null && qualifiedAttributes.length > 0) {
for (int i = 0; i < qualifiedAttributes.length; i++) {
- String qualifiedAttribute = qualifiedAttributes[i];
- IdentifierHelper.IdentifierMetadata idMetadata = getIdMetadata(qualifiedAttribute);
+ String qualifiedAttribute = qualifiedAttributes[i];
+ IdentifierHelper.Info idMetadata = createInfo(qualifiedAttribute);
// Only primitive attributes need to be checked
if (idMetadata.isPrimitive() && !selectClauseComposer.isAggregatorIdx(i) && !attributesProcessed.contains(qualifiedAttribute)) {
add(GremlinClause.HAS_PROPERTY, qualifiedAttribute);
@@ -359,7 +356,7 @@ public class GremlinQueryComposer {
}
for (int i = 0; i < scc.getItems().length; i++) {
- IdentifierHelper.IdentifierMetadata ia = getIdMetadata(scc.getItem(i));
+ IdentifierHelper.Info ia = createInfo(scc.getItem(i));
if (scc.isAggregatorWithArgument(i) && !ia.isPrimitive()) {
context.check(false, AtlasErrorCode.INVALID_DSL_SELECT_INVALID_AGG, ia.getQualifiedName());
@@ -383,7 +380,7 @@ public class GremlinQueryComposer {
scc.incrementTypesIntroduced();
scc.isSelectNoop = !ia.hasParts();
if (ia.hasParts()) {
- scc.assign(i, getQualifiedName(getIdMetadata(ia.get())), GremlinClause.INLINE_GET_PROPERTY);
+ scc.assign(i, getQualifiedName(createInfo(ia.get())), GremlinClause.INLINE_GET_PROPERTY);
}
} else {
scc.assign(i, getQualifiedName(ia), GremlinClause.INLINE_GET_PROPERTY);
@@ -396,7 +393,7 @@ public class GremlinQueryComposer {
context.validator.check(!scc.hasMixedAttributes(), AtlasErrorCode.INVALID_DSL_SELECT_ATTR_MIXING);
}
- private boolean hasNoopCondition(IdentifierHelper.IdentifierMetadata ia) {
+ private boolean hasNoopCondition(IdentifierHelper.Info ia) {
return !ia.isPrimitive() && !ia.isAttribute() && context.hasAlias(ia.getRaw());
}
@@ -519,7 +516,7 @@ public class GremlinQueryComposer {
}
}
- private boolean introduceType(IdentifierHelper.IdentifierMetadata ia) {
+ private boolean introduceType(IdentifierHelper.Info ia) {
if (ia.isReferredType()) {
add(GremlinClause.OUT, ia.getEdgeLabel());
context.registerActive(ia);
@@ -528,7 +525,7 @@ public class GremlinQueryComposer {
return ia.isReferredType();
}
- private IdentifierHelper.IdentifierMetadata getIdMetadata(String actualTypeName) {
+ private IdentifierHelper.Info createInfo(String actualTypeName) {
return IdentifierHelper.create(context, lookup, actualTypeName);
}
@@ -549,7 +546,7 @@ public class GremlinQueryComposer {
LOG.debug("addOrderByClause(name={})", name, descr);
}
- IdentifierHelper.IdentifierMetadata ia = getIdMetadata(name);
+ IdentifierHelper.Info ia = createInfo(name);
add((!descr) ? GremlinClause.ORDER_BY : GremlinClause.ORDER_BY_DESC, ia);
}
@@ -558,16 +555,16 @@ public class GremlinQueryComposer {
LOG.debug("addGroupByClause(name={})", name);
}
- IdentifierHelper.IdentifierMetadata ia = getIdMetadata(name);
+ IdentifierHelper.Info ia = createInfo(name);
add(GremlinClause.GROUP_BY, ia);
}
- private void add(GremlinClause clause, IdentifierHelper.IdentifierMetadata ia) {
- if (context != null && !context.validator.isValid(context, clause, ia)) {
+ private void add(GremlinClause clause, IdentifierHelper.Info idInfo) {
+ if (context != null && !context.validator.isValid(context, clause, idInfo)) {
return;
}
- add(clause, (ia.getQualifiedName() == null ? ia.get() : ia.getQualifiedName()));
+ add(clause, (idInfo.getQualifiedName() == null ? idInfo.get() : idInfo.getQualifiedName()));
}
private void add(GremlinClause clause, String... args) {
@@ -623,10 +620,10 @@ public class GremlinQueryComposer {
}
}
- public void registerActive(IdentifierHelper.IdentifierMetadata identifierMetadata) {
- if (validator.check(StringUtils.isNotEmpty(identifierMetadata.getTypeName()),
- AtlasErrorCode.INVALID_DSL_UNKNOWN_TYPE, identifierMetadata.getRaw())) {
- registerActive(identifierMetadata.getTypeName());
+ public void registerActive(IdentifierHelper.Info info) {
+ if (validator.check(StringUtils.isNotEmpty(info.getTypeName()),
+ AtlasErrorCode.INVALID_DSL_UNKNOWN_TYPE, info.getRaw())) {
+ registerActive(info.getTypeName());
} else {
activeType = UNKNOWN_TYPE;
}
@@ -706,7 +703,7 @@ public class GremlinQueryComposer {
this.lookup = lookup;
}
- public boolean isValid(Context ctx, GremlinClause clause, IdentifierHelper.IdentifierMetadata ia) {
+ public boolean isValid(Context ctx, GremlinClause clause, IdentifierHelper.Info ia) {
switch (clause) {
case TRAIT:
return check(ia.isTrait(), AtlasErrorCode.INVALID_DSL_UNKNOWN_CLASSIFICATION, ia.getRaw());
http://git-wip-us.apache.org/repos/asf/atlas/blob/ba4df832/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java
index 87a8dc7..20f037a 100644
--- a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java
+++ b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java
@@ -47,12 +47,12 @@ public class IdentifierHelper {
return ret;
}
- public static IdentifierMetadata create(GremlinQueryComposer.Context context,
- org.apache.atlas.query.Lookup lookup,
- String identifier) {
- IdentifierMetadata ia = new IdentifierMetadata(identifier);
- ia.update(lookup, context);
- return ia;
+ public static Info create(GremlinQueryComposer.Context context,
+ org.apache.atlas.query.Lookup lookup,
+ String identifier) {
+ Info idInfo = new Info(identifier);
+ idInfo.update(lookup, context);
+ return idInfo;
}
private static String extract(Pattern p, String s) {
@@ -106,7 +106,7 @@ public class IdentifierHelper {
}
- public static class IdentifierMetadata {
+ public static class Info {
private String raw;
private String actual;
private String[] parts;
@@ -123,7 +123,7 @@ public class IdentifierHelper {
private String qualifiedName;
private boolean isDate;
- public IdentifierMetadata(String s) {
+ public Info(String s) {
this.raw = removeQuotes(s);
this.actual = IdentifierHelper.get(raw);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/ba4df832/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java b/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
index 831a098..88ca784 100644
--- a/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
+++ b/repository/src/test/java/org/apache/atlas/query/DSLQueriesTest.java
@@ -214,6 +214,7 @@ public class DSLQueriesTest extends BasicTestSetup {
{"hive_db as db1 hive_table where (db1.name = \"Reporting\")", 0},
{"hive_table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1", 1},
{"hive_table where (name = \"sales_fact\" and createTime >= \"2014-12-11T02:35:58.440Z\" ) select name as _col_0, createTime as _col_1", 1},
+ {"hive_table where (name = \"sales_fact\" and db.name = \"Sales\") select name, createTime", 1},
{"Dimension", 5},
{"JdbcAccess", 2},
{"ETL", 5},
http://git-wip-us.apache.org/repos/asf/atlas/blob/ba4df832/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java b/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
index 6e7420f..7827a6e 100644
--- a/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
+++ b/repository/src/test/java/org/apache/atlas/query/GremlinQueryComposerTest.java
@@ -314,6 +314,14 @@ public class GremlinQueryComposerTest {
}
@Test
+ public void whereComplexAndSelect() {
+ String exSel = "def f(r){ t=[['name']]; r.each({t.add([it.value('Table.name')])}); t.unique(); }";
+ String exMain = "g.V().has('__typeName', 'Table').and(__.out('__Table.db').has('DB.name', eq(\"Reporting\")).dedup().in('__Table.db'),__.has('Table.name', eq(\"sales_fact\"))).limit(local, 25).limit(25).toList()";
+ verify("Table where db.name = \"Reporting\" and name =\"sales_fact\" select name", getExpected(exSel, exMain));
+ verify("Table where db.name = \"Reporting\" and name =\"sales_fact\"", exMain);
+ }
+
+ @Test
public void invalidQueries() {
verify("hdfs_path like h1", "");
}