You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/11/05 20:31:59 UTC
[37/40] atlas git commit: ATLAS-2251: Remove TypeSystem and related
implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/notification/src/test/java/org/apache/atlas/notification/hook/HookMessageDeserializerTest.java
----------------------------------------------------------------------
diff --git a/notification/src/test/java/org/apache/atlas/notification/hook/HookMessageDeserializerTest.java b/notification/src/test/java/org/apache/atlas/notification/hook/HookMessageDeserializerTest.java
index 49b877b..3e1c3dd 100644
--- a/notification/src/test/java/org/apache/atlas/notification/hook/HookMessageDeserializerTest.java
+++ b/notification/src/test/java/org/apache/atlas/notification/hook/HookMessageDeserializerTest.java
@@ -18,12 +18,13 @@
package org.apache.atlas.notification.hook;
+import org.apache.atlas.model.v1.instance.Referenceable;
+import org.apache.atlas.model.v1.instance.Struct;
import org.apache.atlas.notification.AbstractNotification;
import org.apache.atlas.notification.entity.EntityNotificationImplTest;
import org.apache.atlas.notification.hook.HookNotification.EntityUpdateRequest;
import org.apache.atlas.notification.hook.HookNotification.HookNotificationMessage;
-import org.apache.atlas.typesystem.Referenceable;
-import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.type.AtlasType;
import org.apache.commons.lang3.RandomStringUtils;
import org.testng.annotations.Test;
@@ -61,7 +62,7 @@ public class HookMessageDeserializerTest {
Referenceable entity = generateEntityWithTrait();
EntityUpdateRequest message = new EntityUpdateRequest("user1", entity);
- String jsonMsg = AbstractNotification.GSON.toJson(message);
+ String jsonMsg = AtlasType.toV1Json(message);
HookNotificationMessage deserializedMessage = deserializer.deserialize(jsonMsg);
assertEqualMessage(deserializedMessage, message);
@@ -79,7 +80,7 @@ public class HookMessageDeserializerTest {
assertTrue(jsonMsgList.size() == 1);
String compressedMsg = jsonMsgList.get(0);
- String uncompressedMsg = AbstractNotification.GSON.toJson(message);
+ String uncompressedMsg = AtlasType.toV1Json(message);
assertTrue(compressedMsg.length() < uncompressedMsg.length(), "Compressed message (" + compressedMsg.length() + ") should be shorter than uncompressed message (" + uncompressedMsg.length() + ")");
@@ -134,7 +135,7 @@ public class HookMessageDeserializerTest {
EntityUpdateRequest deserializedEntityUpdateRequest = (EntityUpdateRequest) deserializedMessage;
Referenceable deserializedEntity = deserializedEntityUpdateRequest.getEntities().get(0);
Referenceable entity = message.getEntities().get(0);
- String traitName = entity.getTraits().get(0);
+ String traitName = entity.getTraitNames().get(0);
assertEquals(deserializedEntity.getId(), entity.getId());
assertEquals(deserializedEntity.getTypeName(), entity.getTypeName());
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java
----------------------------------------------------------------------
diff --git a/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java b/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java
index dd3257e..786fbfe 100644
--- a/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java
+++ b/notification/src/test/java/org/apache/atlas/notification/hook/HookNotificationTest.java
@@ -17,8 +17,8 @@
*/
package org.apache.atlas.notification.hook;
-import org.apache.atlas.notification.AbstractNotification;
-import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.model.v1.instance.Referenceable;
+import org.apache.atlas.type.AtlasType;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
@@ -37,7 +37,7 @@ public class HookNotificationTest {
String user = "user";
HookNotification.EntityCreateRequest request = new HookNotification.EntityCreateRequest(user, entity1, entity2);
- String notificationJson = AbstractNotification.GSON.toJson(request);
+ String notificationJson = AtlasType.toV1Json(request);
HookNotification.HookNotificationMessage actualNotification =
HOOK_MESSAGE_DESERIALIZER.deserialize(notificationJson);
@@ -60,7 +60,7 @@ public class HookNotificationTest {
entity.set("attr", "value");
HookNotification.EntityCreateRequest request = new HookNotification.EntityCreateRequest(null, entity);
- String notificationJsonFromCode = AbstractNotification.GSON.toJson(request);
+ String notificationJsonFromCode = AtlasType.toV1Json(request);
System.out.println(notificationJsonFromCode);
//Json without user and assert that the string can be deserialised
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/GraphTransactionAdvisor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/GraphTransactionAdvisor.java b/repository/src/main/java/org/apache/atlas/GraphTransactionAdvisor.java
deleted file mode 100644
index 9751a87..0000000
--- a/repository/src/main/java/org/apache/atlas/GraphTransactionAdvisor.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas;
-
-import org.aopalliance.aop.Advice;
-import org.apache.atlas.annotation.GraphTransaction;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.aop.Pointcut;
-import org.springframework.aop.support.AbstractPointcutAdvisor;
-import org.springframework.aop.support.StaticMethodMatcherPointcut;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import java.lang.reflect.Method;
-
-@Component
-public class GraphTransactionAdvisor extends AbstractPointcutAdvisor {
- private static final Logger LOG = LoggerFactory.getLogger(GraphTransactionAdvisor.class);
-
- private final StaticMethodMatcherPointcut pointcut = new StaticMethodMatcherPointcut() {
- @Override
- public boolean matches(Method method, Class<?> targetClass) {
- boolean annotationPresent = method.isAnnotationPresent(GraphTransaction.class);
- if (annotationPresent) {
- LOG.info("GraphTransaction intercept for {}.{}", targetClass.getName(), method.getName());
- }
- return annotationPresent;
- }
- };
-
- private final GraphTransactionInterceptor interceptor;
-
- @Inject
- public GraphTransactionAdvisor(GraphTransactionInterceptor interceptor) {
- this.interceptor = interceptor;
- }
-
- @Override
- public Pointcut getPointcut() {
- return pointcut;
- }
-
- @Override
- public Advice getAdvice() {
- return interceptor;
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
deleted file mode 100644
index af7f1b4..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/DataSetLineageService.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.discovery;
-
-import org.apache.atlas.ApplicationProperties;
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasConfiguration;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.annotation.GraphTransaction;
-import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
-import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.atlas.query.GremlinQueryResult;
-import org.apache.atlas.query.InputLineageClosureQuery;
-import org.apache.atlas.query.OutputLineageClosureQuery;
-import org.apache.atlas.query.QueryParams;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.MetadataRepository;
-import org.apache.atlas.repository.graph.GraphHelper;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.typesystem.exception.EntityNotFoundException;
-import org.apache.atlas.typesystem.exception.SchemaNotFoundException;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.TypeUtils;
-import org.apache.atlas.utils.ParamChecker;
-import org.apache.commons.configuration.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-import scala.Option;
-import scala.Some;
-import scala.collection.JavaConversions;
-import scala.collection.immutable.List;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import java.util.Arrays;
-import java.util.Iterator;
-
-/**
- * Hive implementation of Lineage service interface.
- */
-@Singleton
-@Component
-public class DataSetLineageService implements LineageService {
-
- private static final Logger LOG = LoggerFactory.getLogger(DataSetLineageService.class);
-
- private static final Option<List<String>> SELECT_ATTRIBUTES =
- Some.apply(JavaConversions.asScalaBuffer(Arrays.asList(AtlasClient.NAME,
- AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)).toList());
- public static final String SELECT_INSTANCE_GUID = "__guid";
-
- public static final String DATASET_SCHEMA_QUERY_PREFIX = "atlas.lineage.schema.query.";
-
- private static final String HIVE_PROCESS_TYPE_NAME = "Process";
- private static final String HIVE_PROCESS_INPUT_ATTRIBUTE_NAME = "inputs";
- private static final String HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME = "outputs";
-
- private static final Configuration propertiesConf;
-
- static {
- try {
- propertiesConf = ApplicationProperties.get();
- } catch (AtlasException e) {
- throw new RuntimeException(e);
- }
- }
-
-
- private final AtlasGraph graph;
- private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
- private final GraphBackedDiscoveryService discoveryService;
-
- @Inject
- DataSetLineageService(MetadataRepository metadataRepository,
- GraphBackedDiscoveryService discoveryService,
- AtlasGraph atlasGraph) throws DiscoveryException {
- this.graph = atlasGraph;
- this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
- this.discoveryService = discoveryService;
- }
-
- /**
- * Return the lineage outputs graph for the given datasetName.
- *
- * @param datasetName datasetName
- * @return Outputs Graph as JSON
- */
- @Override
- @GraphTransaction
- public String getOutputsGraph(String datasetName) throws AtlasException {
- LOG.info("Fetching lineage outputs graph for datasetName={}", datasetName);
- datasetName = ParamChecker.notEmpty(datasetName, "dataset name");
- TypeUtils.Pair<String, String> typeIdPair = validateDatasetNameExists(datasetName);
- return getOutputsGraphForId(typeIdPair.right);
- }
-
- /**
- * Return the lineage inputs graph for the given tableName.
- *
- * @param tableName tableName
- * @return Inputs Graph as JSON
- */
- @Override
- @GraphTransaction
- public String getInputsGraph(String tableName) throws AtlasException {
- LOG.info("Fetching lineage inputs graph for tableName={}", tableName);
- tableName = ParamChecker.notEmpty(tableName, "table name");
- TypeUtils.Pair<String, String> typeIdPair = validateDatasetNameExists(tableName);
- return getInputsGraphForId(typeIdPair.right);
- }
-
- @Override
- @GraphTransaction
- public String getInputsGraphForEntity(String guid) throws AtlasException {
- LOG.info("Fetching lineage inputs graph for entity={}", guid);
- guid = ParamChecker.notEmpty(guid, "Entity id");
- validateDatasetExists(guid);
- return getInputsGraphForId(guid);
- }
-
- private String getInputsGraphForId(String guid) {
- InputLineageClosureQuery
- inputsQuery = new InputLineageClosureQuery(AtlasClient.DATA_SET_SUPER_TYPE, SELECT_INSTANCE_GUID,
- guid, HIVE_PROCESS_TYPE_NAME,
- HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME, Option.empty(),
- SELECT_ATTRIBUTES, true, graphPersistenceStrategy, graph);
- GremlinQueryResult result = inputsQuery.evaluate();
- return inputsQuery.graph(result).toInstanceJson();
- }
-
- @Override
- @GraphTransaction
- public String getOutputsGraphForEntity(String guid) throws AtlasException {
- LOG.info("Fetching lineage outputs graph for entity guid={}", guid);
- guid = ParamChecker.notEmpty(guid, "Entity id");
- validateDatasetExists(guid);
- return getOutputsGraphForId(guid);
- }
-
- private String getOutputsGraphForId(String guid) {
- OutputLineageClosureQuery outputsQuery =
- new OutputLineageClosureQuery(AtlasClient.DATA_SET_SUPER_TYPE, SELECT_INSTANCE_GUID, guid, HIVE_PROCESS_TYPE_NAME,
- HIVE_PROCESS_INPUT_ATTRIBUTE_NAME, HIVE_PROCESS_OUTPUT_ATTRIBUTE_NAME, Option.empty(),
- SELECT_ATTRIBUTES, true, graphPersistenceStrategy, graph);
- GremlinQueryResult result = outputsQuery.evaluate();
- return outputsQuery.graph(result).toInstanceJson();
- }
-
- /**
- * Return the schema for the given tableName.
- *
- * @param datasetName tableName
- * @return Schema as JSON
- */
- @Override
- @GraphTransaction
- public String getSchema(String datasetName) throws AtlasException {
- datasetName = ParamChecker.notEmpty(datasetName, "table name");
- LOG.info("Fetching schema for tableName={}", datasetName);
- TypeUtils.Pair<String, String> typeIdPair = validateDatasetNameExists(datasetName);
-
- return getSchemaForId(typeIdPair.left, typeIdPair.right);
- }
-
- private String getSchemaForId(String typeName, String guid) throws DiscoveryException, SchemaNotFoundException {
- String configName = DATASET_SCHEMA_QUERY_PREFIX + typeName;
- if (propertiesConf.getString(configName) != null) {
- final String schemaQuery =
- String.format(propertiesConf.getString(configName), guid);
- int limit = AtlasConfiguration.SEARCH_MAX_LIMIT.getInt();
- return discoveryService.searchByDSL(schemaQuery, new QueryParams(limit, 0));
- }
- throw new SchemaNotFoundException("Schema is not configured for type " + typeName + ". Configure " + configName);
- }
-
- @Override
- @GraphTransaction
- public String getSchemaForEntity(String guid) throws AtlasException {
- guid = ParamChecker.notEmpty(guid, "Entity id");
- LOG.info("Fetching schema for entity guid={}", guid);
- String typeName = validateDatasetExists(guid);
- return getSchemaForId(typeName, guid);
- }
-
- /**
- * Validate if indeed this is a table type and exists.
- *
- * @param datasetName table name
- */
- private TypeUtils.Pair<String, String> validateDatasetNameExists(String datasetName) throws AtlasException {
- Iterator<AtlasVertex> results = graph.query().has("Referenceable.qualifiedName", datasetName)
- .has(Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name())
- .has(Constants.SUPER_TYPES_PROPERTY_KEY, AtlasClient.DATA_SET_SUPER_TYPE)
- .vertices().iterator();
- while (results.hasNext()) {
- AtlasVertex vertex = results.next();
- return TypeUtils.Pair.of(GraphHelper.getTypeName(vertex), GraphHelper.getGuid(vertex));
- }
- throw new EntityNotFoundException("Dataset with name = " + datasetName + " does not exist");
- }
-
- /**
- * Validate if indeed this is a table type and exists.
- *
- * @param guid entity id
- */
- private String validateDatasetExists(String guid) throws AtlasException {
- for (AtlasVertex vertex : (Iterable<AtlasVertex>) graph.query().has(Constants.GUID_PROPERTY_KEY, guid)
- .has(Constants.SUPER_TYPES_PROPERTY_KEY, AtlasClient.DATA_SET_SUPER_TYPE)
- .vertices()) {
- return GraphHelper.getTypeName(vertex);
- }
- throw new EntityNotFoundException("Dataset with guid = " + guid + " does not exist");
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
deleted file mode 100644
index e86047e..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.discovery;
-
-import org.apache.atlas.query.QueryParams;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Metadata discovery service.
- */
-public interface DiscoveryService {
-
- /**
- * Searches using Full text query
- * @param query query string
- * @param queryParams Default query parameters like limit, offset
- * @return results json
- * @throws DiscoveryException
- */
- String searchByFullText(String query, QueryParams queryParams) throws DiscoveryException;
-
- /**
- * Searches using DSL query
- * @param dslQuery query string
- * @param queryParams Default query parameters like limit, offset
- * @return results json
- * @throws DiscoveryException
- */
- String searchByDSL(String dslQuery, QueryParams queryParams) throws DiscoveryException;
-
- /**
- * Assumes the User is familiar with the persistence structure of the Repository.
- * The given query is run uninterpreted against the underlying Graph Store.
- * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
- *
- * @param gremlinQuery query in gremlin dsl format
- * @return List of Maps
- * @throws org.apache.atlas.discovery.DiscoveryException
- */
- List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException;
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index 7f24d5a..cfaeda2 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -24,7 +24,6 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.SortOrder;
import org.apache.atlas.annotation.GraphTransaction;
-import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasFullTextResult;
@@ -44,7 +43,6 @@ import org.apache.atlas.query.QueryParser;
import org.apache.atlas.query.QueryProcessor;
import org.apache.atlas.query.SelectExpressionHelper;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
@@ -104,7 +102,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private static final String DEFAULT_SORT_ATTRIBUTE_NAME = "name";
private final AtlasGraph graph;
- private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
private final EntityGraphRetriever entityRetriever;
private final AtlasGremlinQueryProvider gremlinQueryProvider;
private final AtlasTypeRegistry typeRegistry;
@@ -116,11 +113,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private final UserProfileService userProfileService;
@Inject
- EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry,
+ EntityDiscoveryService(AtlasTypeRegistry typeRegistry,
AtlasGraph graph, GraphBackedSearchIndexer indexer, SearchTracker searchTracker,
UserProfileService userProfileService) throws AtlasException {
this.graph = graph;
- this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.indexer = indexer;
this.searchTracker = searchTracker;
@@ -693,7 +689,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
Expression expression = either.right().get();
Expression validExpression = QueryProcessor.validate(expression);
- GremlinQuery gremlinQuery = new GremlinTranslator(validExpression, graphPersistenceStrategy).translate();
+ GremlinQuery gremlinQuery = new GremlinTranslator(validExpression).translate();
if (LOG.isDebugEnabled()) {
LOG.debug("Translated Gremlin Query: {}", gremlinQuery.queryStr());
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java b/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
index 8e67e32..b35346e 100755
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
@@ -18,11 +18,9 @@
package org.apache.atlas.discovery;
-import org.apache.atlas.listener.TypesChangeListener;
-
/**
* Interface for indexing types.
*/
-public interface SearchIndexer extends TypesChangeListener {
+public interface SearchIndexer {
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
deleted file mode 100755
index 9b0aa4c..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.discovery.graph;
-
-import java.util.List;
-
-import javax.inject.Inject;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.RequestContext;
-import org.apache.atlas.groovy.GroovyExpression;
-import org.apache.atlas.query.GraphPersistenceStrategies;
-import org.apache.atlas.query.GraphPersistenceStrategies$class;
-import org.apache.atlas.query.TypeUtils;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.MetadataRepository;
-import org.apache.atlas.repository.RepositoryException;
-import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
-import org.apache.atlas.repository.graph.GraphHelper;
-import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.repository.graphdb.GremlinVersion;
-import org.apache.atlas.typesystem.ITypedReferenceableInstance;
-import org.apache.atlas.typesystem.ITypedStruct;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.AttributeInfo;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.ImmutableCollection;
-import com.google.common.collect.ImmutableList;
-
-/**
- * Default implementation of GraphPersistenceStrategy.
- */
-public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategies {
-
- private static final Logger LOG = LoggerFactory.getLogger(DefaultGraphPersistenceStrategy.class);
-
- private final GraphBackedMetadataRepository metadataRepository;
-
- @Inject
- public DefaultGraphPersistenceStrategy(MetadataRepository metadataRepository) {
- this.metadataRepository = (GraphBackedMetadataRepository) metadataRepository;
- }
-
- @Override
- public String typeAttributeName() {
- return metadataRepository.getTypeAttributeName();
- }
-
- @Override
- public String superTypeAttributeName() {
- return metadataRepository.getSuperTypeAttributeName();
- }
-
- @Override
- public String edgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
- try {
- return metadataRepository.getEdgeLabel(dataType, aInfo);
- } catch (AtlasException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public String traitLabel(IDataType<?> dataType, String traitName) {
- return metadataRepository.getTraitLabel(dataType, traitName);
- }
-
- @Override
- public String fieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) {
- try {
- return metadataRepository.getFieldNameInVertex(dataType, aInfo);
- } catch (AtlasException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public List<String> traitNames(AtlasVertex AtlasVertex) {
- return GraphHelper.getTraitNames(AtlasVertex);
- }
-
- @Override
- public Id getIdFromVertex(String dataTypeName, AtlasVertex vertex) {
- return GraphHelper.getIdFromVertex(dataTypeName, vertex);
- }
-
- @Override
- public ITypedReferenceableInstance constructClassInstanceId(ClassType classType, Object value) {
- try {
- AtlasVertex classVertex = (AtlasVertex) value;
- ITypedReferenceableInstance classInstance = classType.createInstance(GraphHelper.getIdFromVertex(classVertex),
- new String[0]);
- return classType.convert(classInstance, Multiplicity.OPTIONAL);
- } catch (AtlasException e) {
- LOG.error("error while constructing an instance", e);
- }
- return null;
- }
-
- @Override
- public <U> U constructInstance(IDataType<U> dataType, Object value) {
- try {
- switch (dataType.getTypeCategory()) {
- case PRIMITIVE:
- case ENUM:
- return dataType.convert(value, Multiplicity.OPTIONAL);
- case ARRAY:
- DataTypes.ArrayType arrType = (DataTypes.ArrayType) dataType;
- IDataType<?> elemType = arrType.getElemType();
- ImmutableCollection.Builder result = ImmutableList.builder();
- List list = (List) value;
- for(Object listElement : list) {
- Object collectionEntry = constructCollectionEntry(elemType, listElement);
- if(collectionEntry != null) {
- result.add(collectionEntry);
- }
- }
- return (U)result.build();
- case MAP:
- // todo
- break;
-
- case STRUCT:
- AtlasVertex structVertex = (AtlasVertex) value;
- StructType structType = (StructType) dataType;
- ITypedStruct structInstance = structType.createInstance();
- TypeSystem.IdType idType = TypeSystem.getInstance().getIdType();
-
- if (dataType.getName().equals(idType.getName())) {
- structInstance.set(idType.typeNameAttrName(), GraphHelper.getSingleValuedProperty(structVertex, typeAttributeName(), String.class));
- structInstance.set(idType.idAttrName(), GraphHelper.getSingleValuedProperty(structVertex, idAttributeName(), String.class));
- String stateValue = GraphHelper.getSingleValuedProperty(structVertex, stateAttributeName(), String.class);
- if (stateValue != null) {
- structInstance.set(idType.stateAttrName(), stateValue);
- }
- structInstance.set(idType.versionAttrName(), structVertex.getProperty(versionAttributeName(), Integer.class));
- } else {
- metadataRepository.getGraphToInstanceMapper()
- .mapVertexToInstance(structVertex, structInstance, structType.fieldMapping().fields);
- }
- return dataType.convert(structInstance, Multiplicity.OPTIONAL);
-
- case TRAIT:
- AtlasVertex traitVertex = (AtlasVertex) value;
- TraitType traitType = (TraitType) dataType;
- ITypedStruct traitInstance = traitType.createInstance();
- // todo - this is not right, we should load the Instance associated with this
- // trait. for now just loading the trait struct.
- // metadataRepository.getGraphToInstanceMapper().mapVertexToTraitInstance(
- // traitVertex, dataType.getName(), , traitType, traitInstance);
- metadataRepository.getGraphToInstanceMapper()
- .mapVertexToInstance(traitVertex, traitInstance, traitType.fieldMapping().fields);
- break;
-
- case CLASS:
- AtlasVertex classVertex = (AtlasVertex) value;
- String guid = classVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class);
- // Check if the instance we need was previously loaded.
- ITypedReferenceableInstance classInstance = RequestContext.get().getInstanceV1(guid);
- if (classInstance == null) {
- classInstance = metadataRepository.getGraphToInstanceMapper().mapGraphToTypedInstance(guid, classVertex);
- }
- return dataType.convert(classInstance, Multiplicity.OPTIONAL);
-
- default:
- throw new UnsupportedOperationException("Load for type " + dataType + "is not supported");
- }
- } catch (AtlasException e) {
- LOG.error("error while constructing an instance", e);
- }
-
- return null;
- }
-
- public <U> U constructCollectionEntry(IDataType<U> elementType, Object value) throws AtlasException {
- switch (elementType.getTypeCategory()) {
- case PRIMITIVE:
- case ENUM:
- return constructInstance(elementType, value);
- //The array values in case of STRUCT, CLASS contain the edgeId if the outgoing edge which links to the STRUCT, CLASS vertex referenced
- case STRUCT:
- case CLASS:
- String edgeId = (String) value;
- return (U) metadataRepository.getGraphToInstanceMapper().getReferredEntity(edgeId, elementType);
- case ARRAY:
- case MAP:
- case TRAIT:
- return null;
- default:
- throw new UnsupportedOperationException("Load for type " + elementType + " in collections is not supported");
- }
- }
-
- @Override
- public String edgeLabel(TypeUtils.FieldInfo fInfo) {
- return fInfo.reverseDataType() == null ? edgeLabel(fInfo.dataType(), fInfo.attrInfo()) :
- edgeLabel(fInfo.reverseDataType(), fInfo.attrInfo());
- }
-
- @Override
- public AtlasEdgeDirection instanceToTraitEdgeDirection() {
- return AtlasEdgeDirection.OUT;
- }
-
- @Override
- public AtlasEdgeDirection traitToInstanceEdgeDirection() {
- return AtlasEdgeDirection.IN;
- }
-
- @Override
- public String idAttributeName() {
- return metadataRepository.getIdAttributeName();
- }
-
- @Override
- public String stateAttributeName() {
- return metadataRepository.getStateAttributeName();
- }
-
- @Override
- public String versionAttributeName() {
- return metadataRepository.getVersionAttributeName();
- }
-
- @Override
- public boolean collectTypeInstancesIntoVar() {
- return GraphPersistenceStrategies$class.collectTypeInstancesIntoVar(this);
- }
-
- @Override
- public boolean filterBySubTypes() {
- return GraphPersistenceStrategies$class.filterBySubTypes(this);
- }
-
- @Override
- public boolean addGraphVertexPrefix(scala.collection.Traversable<GroovyExpression> preStatements) {
- return GraphPersistenceStrategies$class.addGraphVertexPrefix(this, preStatements);
- }
-
- @Override
- public GremlinVersion getSupportedGremlinVersion() {
- return GraphPersistenceStrategies$class.getSupportedGremlinVersion(this);
- }
-
- @Override
- public GroovyExpression generatePersisentToLogicalConversionExpression(GroovyExpression expr, IDataType<?> t) {
- return GraphPersistenceStrategies$class.generatePersisentToLogicalConversionExpression(this,expr, t);
- }
-
- @Override
- public GroovyExpression addInitialQueryCondition(GroovyExpression expr) {
- return GraphPersistenceStrategies$class.addInitialQueryCondition(this, expr);
- }
-
- @Override
- public boolean isPropertyValueConversionNeeded(IDataType<?> t) {
- return GraphPersistenceStrategies$class.isPropertyValueConversionNeeded(this, t);
- }
-
- @Override
- public AtlasGraph getGraph() throws RepositoryException {
- return metadataRepository.getGraph();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java
deleted file mode 100755
index aed8659..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/graph/GraphBackedDiscoveryService.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.discovery.graph;
-
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.annotation.GraphTransaction;
-import org.apache.atlas.discovery.DiscoveryException;
-import org.apache.atlas.discovery.DiscoveryService;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.query.Expressions;
-import org.apache.atlas.query.GremlinEvaluator;
-import org.apache.atlas.query.GremlinQuery;
-import org.apache.atlas.query.GremlinQueryResult;
-import org.apache.atlas.query.GremlinTranslator;
-import org.apache.atlas.query.QueryParams;
-import org.apache.atlas.query.QueryParser;
-import org.apache.atlas.query.QueryProcessor;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.MetadataRepository;
-import org.apache.atlas.repository.graph.GraphHelper;
-import org.apache.atlas.repository.graphdb.AtlasEdge;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.util.CompiledQueryCacheKey;
-import org.apache.atlas.util.NoopGremlinQuery;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-import scala.util.Either;
-import scala.util.parsing.combinator.Parsers;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Graph backed implementation of Search.
- */
-@Singleton
-@Component
-public class GraphBackedDiscoveryService implements DiscoveryService {
-
- private static final Logger LOG = LoggerFactory.getLogger(GraphBackedDiscoveryService.class);
-
- private final AtlasGraph graph;
- private final DefaultGraphPersistenceStrategy graphPersistenceStrategy;
-
- public final static String SCORE = "score";
- /**
- * Where the vertex' internal gremlin id is stored in the Map produced by extractResult()
- */
- public final static String GREMLIN_ID_KEY = "id";
- /**
- * Where the id of an edge's incoming vertex is stored in the Map produced by extractResult()
- */
- public final static String GREMLIN_INVERTEX_KEY = "inVertex";
- /**
- * Where the id of an edge's outgoing vertex is stored in the Map produced by extractResult()
- */
- public final static String GREMLIN_OUTVERTEX_KEY = "outVertex";
- /**
- * Where an edge's label is stored in the Map produced by extractResult()
- */
- public final static String GREMLIN_LABEL_KEY = "label";
-
- @Inject
- GraphBackedDiscoveryService(MetadataRepository metadataRepository, AtlasGraph atlasGraph)
- throws DiscoveryException {
- this.graph = atlasGraph;
- this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
- }
-
- //For titan 0.5.4, refer to http://s3.thinkaurelius.com/docs/titan/0.5.4/index-backends.html for indexed query
- //http://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query
- // .html#query-string-syntax for query syntax
- @Override
- @GraphTransaction
- public String searchByFullText(String query, QueryParams queryParams) throws DiscoveryException {
- String graphQuery = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, query);
- LOG.debug("Full text query: {}", graphQuery);
- Iterator<AtlasIndexQuery.Result<?, ?>> results =graph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery).vertices();
- JSONArray response = new JSONArray();
-
- int index = 0;
- while (results.hasNext() && index < queryParams.offset()) {
- results.next();
- index++;
- }
-
- while (results.hasNext() && response.length() < queryParams.limit()) {
-
- AtlasIndexQuery.Result<?,?> result = results.next();
- AtlasVertex<?,?> vertex = result.getVertex();
-
- JSONObject row = new JSONObject();
- String guid = GraphHelper.getGuid(vertex);
- if (guid != null) { //Filter non-class entities
- try {
- row.put("guid", guid);
- row.put(AtlasClient.TYPENAME, GraphHelper.getTypeName(vertex));
- row.put(SCORE, result.getScore());
- } catch (JSONException e) {
- LOG.error("Unable to create response", e);
- throw new DiscoveryException("Unable to create response");
- }
-
- response.put(row);
- }
- }
- return response.toString();
- }
-
- @Override
- @GraphTransaction
- public String searchByDSL(String dslQuery, QueryParams queryParams) throws DiscoveryException {
- GremlinQueryResult queryResult = evaluate(dslQuery, queryParams);
- return queryResult.toJson();
- }
-
- public GremlinQueryResult evaluate(String dslQuery, QueryParams queryParams) throws DiscoveryException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Executing dsl query={}", dslQuery);
- }
- try {
- GremlinQuery gremlinQuery = parseAndTranslateDsl(dslQuery, queryParams);
- if(gremlinQuery instanceof NoopGremlinQuery) {
- return new GremlinQueryResult(dslQuery, ((NoopGremlinQuery)gremlinQuery).getDataType(), Collections.emptyList());
- }
-
- return new GremlinEvaluator(gremlinQuery, graphPersistenceStrategy, graph).evaluate();
-
- } catch (Exception e) { // unable to catch ExpressionException
- throw new DiscoveryException("Invalid expression : " + dslQuery, e);
- }
- }
-
- private GremlinQuery parseAndTranslateDsl(String dslQuery, QueryParams queryParams) throws DiscoveryException {
-
- CompiledQueryCacheKey entry = new CompiledQueryCacheKey(dslQuery, queryParams);
- GremlinQuery gremlinQuery = QueryProcessor.compiledQueryCache().get(entry);
- if(gremlinQuery == null) {
- Expressions.Expression validatedExpression = parseQuery(dslQuery, queryParams);
-
- //If the final limit is 0, don't launch the query, return with 0 rows
- if (validatedExpression instanceof Expressions.LimitExpression
- && ((Integer)((Expressions.LimitExpression) validatedExpression).limit().rawValue()) == 0) {
- gremlinQuery = new NoopGremlinQuery(validatedExpression.dataType());
- }
- else {
- gremlinQuery = new GremlinTranslator(validatedExpression, graphPersistenceStrategy).translate();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Query = {}", validatedExpression);
- LOG.debug("Expression Tree = {}", validatedExpression.treeString());
- LOG.debug("Gremlin Query = {}", gremlinQuery.queryStr());
- }
- }
- QueryProcessor.compiledQueryCache().put(entry, gremlinQuery);
- }
- return gremlinQuery;
- }
-
- private Expressions.Expression parseQuery(String dslQuery, QueryParams queryParams) throws DiscoveryException {
- Either<Parsers.NoSuccess, Expressions.Expression> either = QueryParser.apply(dslQuery, queryParams);
- if (either.isRight()) {
- Expressions.Expression expression = either.right().get();
- Expressions.Expression validatedExpression = QueryProcessor.validate(expression);
- return validatedExpression;
- } else {
- throw new DiscoveryException("Invalid expression : " + dslQuery + ". " + either.left());
- }
-
- }
-
- /**
- * Assumes the User is familiar with the persistence structure of the Repository.
- * The given query is run uninterpreted against the underlying Graph Store.
- * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
- *
- * @param gremlinQuery query in gremlin dsl format
- * @return List of Maps
- * @throws org.apache.atlas.discovery.DiscoveryException
- */
- @Override
- @GraphTransaction
- public List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException {
- LOG.debug("Executing gremlin query={}", gremlinQuery);
- try {
- Object o = graph.executeGremlinScript(gremlinQuery, false);
- return extractResult(o);
- } catch (AtlasBaseException e) {
- throw new DiscoveryException(e);
- }
- }
-
- private List<Map<String, String>> extractResult(final Object o) throws DiscoveryException {
- List<Map<String, String>> result = new ArrayList<>();
- if (o instanceof List) {
- List l = (List) o;
-
- for (Object value : l) {
- Map<String, String> oRow = new HashMap<>();
- if (value instanceof Map) {
- @SuppressWarnings("unchecked") Map<Object, Object> iRow = (Map) value;
- for (Map.Entry e : iRow.entrySet()) {
- Object k = e.getKey();
- Object v = e.getValue();
- oRow.put(k.toString(), v.toString());
- }
- } else if (value instanceof AtlasVertex) {
- AtlasVertex<?,?> vertex = (AtlasVertex<?,?>)value;
- for (String key : vertex.getPropertyKeys()) {
- Object propertyValue = GraphHelper.getProperty(vertex, key);
- if (propertyValue != null) {
- oRow.put(key, propertyValue.toString());
- }
- }
- oRow.put(GREMLIN_ID_KEY, vertex.getId().toString());
-
- } else if (value instanceof String) {
- oRow.put("", value.toString());
- } else if(value instanceof AtlasEdge) {
- AtlasEdge edge = (AtlasEdge) value;
- oRow.put(GREMLIN_ID_KEY, edge.getId().toString());
- oRow.put(GREMLIN_LABEL_KEY, edge.getLabel());
- oRow.put(GREMLIN_INVERTEX_KEY, edge.getInVertex().getId().toString());
- oRow.put(GREMLIN_OUTVERTEX_KEY, edge.getOutVertex().getId().toString());
- for (String propertyKey : edge.getPropertyKeys()) {
- oRow.put(propertyKey, GraphHelper.getProperty(edge, propertyKey).toString());
- }
- } else {
- throw new DiscoveryException(String.format("Cannot process result %s", String.valueOf(value)));
- }
-
- result.add(oRow);
- }
- }
- else {
- result.add(new HashMap<String, String>() {{
- put("result", o.toString());
- }});
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0877e47c/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java b/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java
deleted file mode 100644
index 27de0ed..0000000
--- a/repository/src/main/java/org/apache/atlas/gremlin/Gremlin2ExpressionFactory.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.gremlin;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.groovy.AbstractFunctionExpression;
-import org.apache.atlas.groovy.CastExpression;
-import org.apache.atlas.groovy.ClosureExpression;
-import org.apache.atlas.groovy.ComparisonExpression;
-import org.apache.atlas.groovy.ComparisonExpression.ComparisonOperator;
-import org.apache.atlas.groovy.ComparisonOperatorExpression;
-import org.apache.atlas.groovy.FieldExpression;
-import org.apache.atlas.groovy.FunctionCallExpression;
-import org.apache.atlas.groovy.GroovyExpression;
-import org.apache.atlas.groovy.IdentifierExpression;
-import org.apache.atlas.groovy.ListExpression;
-import org.apache.atlas.groovy.LiteralExpression;
-import org.apache.atlas.groovy.LogicalExpression;
-import org.apache.atlas.groovy.LogicalExpression.LogicalOperator;
-import org.apache.atlas.groovy.RangeExpression;
-import org.apache.atlas.groovy.TernaryOperatorExpression;
-import org.apache.atlas.groovy.TraversalStepType;
-import org.apache.atlas.query.GraphPersistenceStrategies;
-import org.apache.atlas.query.TypeUtils.FieldInfo;
-import org.apache.atlas.typesystem.types.IDataType;
-
-
-/**
- * Generates gremlin query expressions using Gremlin 2 syntax.
- *
- */
-public class Gremlin2ExpressionFactory extends GremlinExpressionFactory {
-
- private static final String LOOP_METHOD = "loop";
- private static final String CONTAINS = "contains";
- private static final String LOOP_COUNT_FIELD = "loops";
- private static final String PATH_FIELD = "path";
- private static final String ENABLE_PATH_METHOD = "enablePath";
- private static final String BACK_METHOD = "back";
- private static final String LAST_METHOD = "last";
-
- @Override
- public GroovyExpression generateLogicalExpression(GroovyExpression parent, String operator, List<GroovyExpression> operands) {
- return new FunctionCallExpression(TraversalStepType.FILTER, parent, operator, operands);
- }
-
-
- @Override
- public GroovyExpression generateBackReferenceExpression(GroovyExpression parent, boolean inSelect, String alias) {
- if (inSelect && parent == null) {
- return getFieldInSelect();
- }
- else if (inSelect && parent != null) {
- return parent;
- }
- else {
- return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, BACK_METHOD, new LiteralExpression(alias));
- }
- }
-
- @Override
- public GroovyExpression getLoopExpressionParent(GroovyExpression inputQry) {
- return inputQry;
- }
-
- @Override
- public GroovyExpression generateLoopExpression(GroovyExpression parent,GraphPersistenceStrategies s, IDataType dataType, GroovyExpression loopExpr, String alias, Integer times) {
-
- GroovyExpression emitExpr = generateLoopEmitExpression(s, dataType);
- //note that in Gremlin 2 (unlike Gremlin 3), the parent is not explicitly used. It is incorporated
- //in the loopExpr.
- GroovyExpression whileFunction = null;
- if(times != null) {
- GroovyExpression loopsExpr = new FieldExpression(getItVariable(), LOOP_COUNT_FIELD);
- GroovyExpression timesExpr = new LiteralExpression(times);
- whileFunction = new ClosureExpression(new ComparisonExpression(loopsExpr, ComparisonOperator.LESS_THAN, timesExpr));
- }
- else {
- GroovyExpression pathExpr = new FieldExpression(getItVariable(),PATH_FIELD);
- GroovyExpression itObjectExpr = getCurrentObjectExpression();
- GroovyExpression pathContainsExpr = new FunctionCallExpression(pathExpr, CONTAINS, itObjectExpr);
- whileFunction = new ClosureExpression(new TernaryOperatorExpression(pathContainsExpr, LiteralExpression.FALSE, LiteralExpression.TRUE));
- }
- GroovyExpression emitFunction = new ClosureExpression(emitExpr);
- GroovyExpression loopCall = new FunctionCallExpression(TraversalStepType.BRANCH, loopExpr, LOOP_METHOD, new LiteralExpression(alias), whileFunction, emitFunction);
-
- return new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, loopCall, ENABLE_PATH_METHOD);
- }
-
- @Override
- public GroovyExpression typeTestExpression(GraphPersistenceStrategies s, String typeName, GroovyExpression itRef) {
-
- GroovyExpression superTypeAttrExpr = new FieldExpression(itRef, s.superTypeAttributeName());
- GroovyExpression typeNameExpr = new LiteralExpression(typeName);
- GroovyExpression isSuperTypeExpr = new FunctionCallExpression(superTypeAttrExpr, CONTAINS, typeNameExpr);
- GroovyExpression superTypeMatchesExpr = new TernaryOperatorExpression(superTypeAttrExpr, isSuperTypeExpr, LiteralExpression.FALSE);
-
- GroovyExpression typeAttrExpr = new FieldExpression(itRef, s.typeAttributeName());
- GroovyExpression typeMatchesExpr = new ComparisonExpression(typeAttrExpr, ComparisonOperator.EQUALS, typeNameExpr);
- return new LogicalExpression(typeMatchesExpr, LogicalOperator.OR, superTypeMatchesExpr);
-
- }
-
- @Override
- public GroovyExpression generateSelectExpression(GroovyExpression parent, List<LiteralExpression> sourceNames,
- List<GroovyExpression> srcExprs) {
-
- GroovyExpression srcNamesExpr = new ListExpression(sourceNames);
- List<GroovyExpression> selectArgs = new ArrayList<>();
- selectArgs.add(srcNamesExpr);
- for(GroovyExpression expr : srcExprs) {
- selectArgs.add(new ClosureExpression(expr));
- }
- return new FunctionCallExpression(TraversalStepType.MAP_TO_VALUE, parent, SELECT_METHOD, selectArgs);
- }
-
- @Override
- public GroovyExpression generateFieldExpression(GroovyExpression parent, FieldInfo fInfo, String propertyName, boolean inSelect) {
- return new FieldExpression(parent, propertyName);
- }
-
- @Override
- public GroovyExpression generateHasExpression(GraphPersistenceStrategies s, GroovyExpression parent, String propertyName, String symbol,
- GroovyExpression requiredValue, FieldInfo fInfo) throws AtlasException {
- GroovyExpression op = gremlin2CompOp(symbol);
- GroovyExpression propertyNameExpr = new LiteralExpression(propertyName);
- return new FunctionCallExpression(TraversalStepType.FILTER, parent, HAS_METHOD, propertyNameExpr, op, requiredValue);
- }
-
- @Override
- public GroovyExpression generateLikeExpressionUsingFilter(GroovyExpression parent, String propertyName, GroovyExpression propertyValue) throws AtlasException {
- GroovyExpression itExpr = getItVariable();
- GroovyExpression nameExpr = new FieldExpression(itExpr, propertyName);
- GroovyExpression matchesExpr = new FunctionCallExpression(nameExpr, MATCHES, escapePropertyValue(propertyValue));
- GroovyExpression closureExpr = new ClosureExpression(matchesExpr);
-
- return new FunctionCallExpression(TraversalStepType.FILTER, parent, FILTER_METHOD, closureExpr);
- }
-
- private GroovyExpression escapePropertyValue(GroovyExpression propertyValue) {
- GroovyExpression ret = propertyValue;
-
- if (propertyValue instanceof LiteralExpression) {
- LiteralExpression exp = (LiteralExpression) propertyValue;
-
- if (exp != null && exp.getValue() instanceof String) {
- String stringValue = (String) exp.getValue();
-
- // replace '*' with ".*", replace '?' with '.'
- stringValue = stringValue.replaceAll("\\*", ".*")
- .replaceAll("\\?", ".");
-
- ret = new LiteralExpression(stringValue);
- }
- }
-
- return ret;
- }
-
- private GroovyExpression gremlin2CompOp(String op) throws AtlasException {
-
- GroovyExpression tExpr = new IdentifierExpression("T");
- if(op.equals("=")) {
- return new FieldExpression(tExpr, "eq");
- }
- if(op.equals("!=")) {
- return new FieldExpression(tExpr, "neq");
- }
- if(op.equals(">")) {
- return new FieldExpression(tExpr, "gt");
- }
- if(op.equals(">=")) {
- return new FieldExpression(tExpr, "gte");
- }
- if(op.equals("<")) {
- return new FieldExpression(tExpr, "lt");
- }
- if(op.equals("<=")) {
- return new FieldExpression(tExpr, "lte");
- }
- if(op.equals("in")) {
- return new FieldExpression(tExpr, "in");
- }
- throw new AtlasException("Comparison operator " + op + " not supported in Gremlin");
- }
-
- @Override
- protected GroovyExpression initialExpression(GroovyExpression varExpr, GraphPersistenceStrategies s) {
- return generateSeededTraversalExpresssion(false, varExpr);
- }
-
- @Override
- public GroovyExpression generateSeededTraversalExpresssion(boolean isMap, GroovyExpression varExpr) {
- return new FunctionCallExpression(TraversalStepType.START, varExpr, "_");
- }
-
- @Override
- public GroovyExpression generateRangeExpression(GroovyExpression parent, int startIndex, int endIndex) {
- //treat as barrier step, since limits need to be applied globally (even though it
- //is technically a filter step)
- return new RangeExpression(TraversalStepType.BARRIER, parent, startIndex, endIndex);
- }
-
- @Override
- public boolean isRangeExpression(GroovyExpression expr) {
-
- return (expr instanceof RangeExpression);
- }
-
- @Override
- public int[] getRangeParameters(AbstractFunctionExpression expr) {
-
- if (isRangeExpression(expr)) {
- RangeExpression rangeExpression = (RangeExpression) expr;
- return new int[] {rangeExpression.getStartIndex(), rangeExpression.getEndIndex()};
- }
- else {
- return null;
- }
- }
-
- @Override
- public void setRangeParameters(GroovyExpression expr, int startIndex, int endIndex) {
-
- if (isRangeExpression(expr)) {
- RangeExpression rangeExpression = (RangeExpression) expr;
- rangeExpression.setStartIndex(startIndex);
- rangeExpression.setEndIndex(endIndex);
- }
- else {
- throw new IllegalArgumentException(expr.getClass().getName() + " is not a valid range expression - must be an instance of " + RangeExpression.class.getName());
- }
-
- }
-
- @Override
- public List<GroovyExpression> getOrderFieldParents() {
-
- GroovyExpression itExpr = getItVariable();
- List<GroovyExpression> result = new ArrayList<>(2);
- result.add(new FieldExpression(itExpr, "a"));
- result.add(new FieldExpression(itExpr, "b"));
- return result;
- }
-
-
- @Override
- public GroovyExpression generateOrderByExpression(GroovyExpression parent, List<GroovyExpression> translatedOrderBy, boolean isAscending) {
-
- GroovyExpression aPropertyExpr = translatedOrderBy.get(0);
- GroovyExpression bPropertyExpr = translatedOrderBy.get(1);
-
- GroovyExpression aPropertyNotNull = new ComparisonExpression(aPropertyExpr, ComparisonOperator.NOT_EQUALS, LiteralExpression.NULL);
- GroovyExpression bPropertyNotNull = new ComparisonExpression(bPropertyExpr, ComparisonOperator.NOT_EQUALS, LiteralExpression.NULL);
-
- GroovyExpression aCondition = new TernaryOperatorExpression(aPropertyNotNull, new FunctionCallExpression(aPropertyExpr,TO_LOWER_CASE_METHOD), aPropertyExpr);
- GroovyExpression bCondition = new TernaryOperatorExpression(bPropertyNotNull, new FunctionCallExpression(bPropertyExpr,TO_LOWER_CASE_METHOD), bPropertyExpr);
-
- GroovyExpression comparisonFunction = null;
- if(isAscending) {
- comparisonFunction = new ComparisonOperatorExpression(aCondition, bCondition);
- }
- else {
- comparisonFunction = new ComparisonOperatorExpression(bCondition, aCondition);
- }
- return new FunctionCallExpression(TraversalStepType.BARRIER, parent, ORDER_METHOD, new ClosureExpression(comparisonFunction));
- }
-
-
- @Override
- public GroovyExpression getAnonymousTraversalExpression() {
- return new FunctionCallExpression(TraversalStepType.START, "_");
- }
-
-
-
- @Override
- public GroovyExpression generateGroupByExpression(GroovyExpression parent, GroovyExpression groupByExpression,
- GroovyExpression aggregationFunction) {
- GroovyExpression groupByClosureExpr = new ClosureExpression(groupByExpression);
- GroovyExpression itClosure = new ClosureExpression(getItVariable());
- GroovyExpression result = new FunctionCallExpression(TraversalStepType.BARRIER, parent, "groupBy", groupByClosureExpr, itClosure);
- result = new FunctionCallExpression(TraversalStepType.SIDE_EFFECT, result, "cap");
- result = new FunctionCallExpression(TraversalStepType.END, result, "next");
- result = new FunctionCallExpression(result, "values");
- result = new FunctionCallExpression(result, "toList");
-
- GroovyExpression aggregrationFunctionClosure = new ClosureExpression(aggregationFunction);
- result = new FunctionCallExpression(result, "collect", aggregrationFunctionClosure);
- return result;
- }
-
- @Override
- public GroovyExpression getFieldInSelect() {
- return getItVariable();
- }
- @Override
- public GroovyExpression getGroupBySelectFieldParent() {
- GroovyExpression itExpr = getItVariable();
- return new FunctionCallExpression(itExpr, LAST_METHOD);
- }
-
- //assumes cast already performed
- @Override
- public GroovyExpression generateCountExpression(GroovyExpression itExpr) {
- return new FunctionCallExpression(itExpr, "size");
- }
-
- @Override
- public String getTraversalExpressionClass() {
- return "GremlinPipeline";
- }
-
-
- @Override
- public boolean isSelectGeneratesMap(int aliasCount) {
- //in Gremlin 2 select always generates a map
- return true;
- }
-
- @Override
- public GroovyExpression generateMapExpression(GroovyExpression parent, ClosureExpression closureExpression) {
- return new FunctionCallExpression(TraversalStepType.MAP_TO_ELEMENT, parent, "transform", closureExpression);
- }
-
- @Override
- public GroovyExpression generateGetSelectedValueExpression(LiteralExpression key,
- GroovyExpression rowMap) {
- rowMap = new CastExpression(rowMap, "Row");
- GroovyExpression getExpr = new FunctionCallExpression(rowMap, "getColumn", key);
- return getExpr;
- }
-
- @Override
- public GroovyExpression getCurrentTraverserObject(GroovyExpression traverser) {
- return traverser;
- }
-
- public List<String> getAliasesRequiredByExpression(GroovyExpression expr) {
- if(!(expr instanceof FunctionCallExpression)) {
- return Collections.emptyList();
- }
- FunctionCallExpression fc = (FunctionCallExpression)expr;
- if(! fc.getFunctionName().equals(LOOP_METHOD)) {
- return Collections.emptyList();
- }
- LiteralExpression aliasName = (LiteralExpression)fc.getArguments().get(0);
- return Collections.singletonList(aliasName.getValue().toString());
- }
-
- @Override
- public boolean isRepeatExpression(GroovyExpression expr) {
- if(!(expr instanceof FunctionCallExpression)) {
- return false;
- }
- return ((FunctionCallExpression)expr).getFunctionName().equals(LOOP_METHOD);
- }
-}
-