You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by pi...@apache.org on 2021/07/09 16:33:07 UTC
[atlas] branch master updated: ATLAS-4347 : DSL Search : Caching
Mechanism for translated dsl queries
This is an automated email from the ASF dual-hosted git repository.
pinal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 1868a2d ATLAS-4347 : DSL Search : Caching Mechanism for translated dsl queries
1868a2d is described below
commit 1868a2ddaf044fb1f5d4df5a7a3a765b213ad6ed
Author: Pinal <pinal-shah>
AuthorDate: Tue Jul 6 17:13:40 2021 +0530
ATLAS-4347 : DSL Search : Caching Mechanism for translated dsl queries
Signed-off-by: Pinal <pinal-shah>
---
.../java/org/apache/atlas/AtlasConfiguration.java | 1 +
.../query/executors/TraversalBasedExecutor.java | 68 +++++++++++++++++-----
2 files changed, 56 insertions(+), 13 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
index 9ef8487..2f2c8a5 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
@@ -78,6 +78,7 @@ public enum AtlasConfiguration {
PROCESS_NAME_UPDATE_PATCH("atlas.process.name.update.patch", false),
STORE_DIFFERENTIAL_AUDITS("atlas.entity.audit.differential", false),
DSL_EXECUTOR_TRAVERSAL("atlas.dsl.executor.traversal", true),
+ DSL_CACHED_TRANSLATOR("atlas.dsl.cached.translator", true),
DEBUG_METRICS_ENABLED("atlas.debug.metrics.enabled", false),
TASKS_USE_ENABLED("atlas.tasks.enabled", true);
diff --git a/repository/src/main/java/org/apache/atlas/query/executors/TraversalBasedExecutor.java b/repository/src/main/java/org/apache/atlas/query/executors/TraversalBasedExecutor.java
index 724cb82..ecd82e3 100644
--- a/repository/src/main/java/org/apache/atlas/query/executors/TraversalBasedExecutor.java
+++ b/repository/src/main/java/org/apache/atlas/query/executors/TraversalBasedExecutor.java
@@ -17,6 +17,7 @@
*/
package org.apache.atlas.query.executors;
+import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.query.AtlasDSL;
@@ -28,6 +29,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraphTraversal;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.LruCache;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
@@ -44,10 +46,18 @@ public class TraversalBasedExecutor implements DSLQueryExecutor {
private static final String DEFAULT_LIMIT_OFFSET_TEMPLATE = " limit %d offset %d";
private static final String CLAUSE_OFFSET_ZERO = " offset 0";
+ private static final Translator translator;
+
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph;
private final EntityGraphRetriever entityRetriever;
+ static {
+ translator = AtlasConfiguration.DSL_CACHED_TRANSLATOR.getBoolean()
+ ? new CachedTranslator()
+ : new Translator();
+ }
+
public TraversalBasedExecutor(AtlasTypeRegistry typeRegistry, AtlasGraph graph, EntityGraphRetriever entityRetriever) {
this.typeRegistry = typeRegistry;
this.graph = graph;
@@ -106,28 +116,60 @@ public class TraversalBasedExecutor implements DSLQueryExecutor {
}
private GremlinQuery toTraversal(String query, int limit, int offset) throws AtlasBaseException {
- QueryParams params = QueryParams.getNormalizedParams(limit, offset);
-
- query = getStringWithLimitOffset(query, params);
-
- AtlasDSL.Translator dslTranslator = new AtlasDSL.Translator(query, typeRegistry, params.offset(), params.limit());
- GremlinQuery gremlinQuery = dslTranslator.translate();
- AtlasGraphTraversal result = GremlinClauseToTraversalTranslator.run(this.graph, gremlinQuery.getClauses());
+ GremlinQuery gremlinQuery = translator.translate(typeRegistry, query, limit, offset);
+ AtlasGraphTraversal result = GremlinClauseToTraversalTranslator.run(this.graph, gremlinQuery.getClauses());
gremlinQuery.setResult(result);
return gremlinQuery;
}
- private String getStringWithLimitOffset(String query, QueryParams params) {
- if (!query.contains(DSL_KEYWORD_LIMIT) && !query.contains(DSL_KEYWORD_OFFSET)) {
- query += String.format(DEFAULT_LIMIT_OFFSET_TEMPLATE, params.limit(), params.offset());
+ private static class Translator {
+
+ public Translator() {
+ }
+
+ public GremlinQuery translate(AtlasTypeRegistry typeRegistry, String query, int limit, int offset) throws AtlasBaseException {
+ QueryParams params = QueryParams.getNormalizedParams(limit, offset);
+ query = getStringWithLimitOffset(query, params);
+
+ AtlasDSL.Translator dslTranslator = new AtlasDSL.Translator(query, typeRegistry, params.offset(), params.limit());
+ GremlinQuery gremlinQuery = dslTranslator.translate();
+
+ return gremlinQuery;
+ }
+
+ private String getStringWithLimitOffset(String query, QueryParams params) {
+ if (!query.contains(DSL_KEYWORD_LIMIT) && !query.contains(DSL_KEYWORD_OFFSET)) {
+ query += String.format(DEFAULT_LIMIT_OFFSET_TEMPLATE, params.limit(), params.offset());
+ }
+
+ if (query.contains(DSL_KEYWORD_LIMIT) && !query.contains(DSL_KEYWORD_OFFSET)) {
+ query += CLAUSE_OFFSET_ZERO;
+ }
+
+ return query;
}
+ }
- if (query.contains(DSL_KEYWORD_LIMIT) && !query.contains(DSL_KEYWORD_OFFSET)) {
- query += CLAUSE_OFFSET_ZERO;
+ private static class CachedTranslator extends Translator {
+ private final static int DSLQUERY_CACHE_SIZE = 100;
+ private final Map<String, GremlinQuery> cachedQuery;
+
+ public CachedTranslator() {
+ this.cachedQuery = new LruCache<>(DSLQUERY_CACHE_SIZE, 100000);
}
- return query;
+ @Override
+ public GremlinQuery translate(AtlasTypeRegistry typeRegistry, String query, int limit, int offset) throws AtlasBaseException {
+ String key = String.format("%s-%s-%s", query, limit, offset);
+
+ if (!cachedQuery.containsKey(key)) {
+ GremlinQuery gremlinQuery = super.translate(typeRegistry, query, limit, offset);
+ cachedQuery.put(key, gremlinQuery);
+ }
+
+ return cachedQuery.get(key);
+ }
}
}