You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/04/03 12:06:32 UTC
[atlas] branch master updated: ATLAS-3113: Use index query to
search for active entities and better logging in java patch framework
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 66f57da ATLAS-3113: Use index query to search for active entities and better logging in java patch framework
66f57da is described below
commit 66f57da82afd1d348eca4e0c9deccc3838ebe114
Author: Sarath Subramanian <ss...@hortonworks.com>
AuthorDate: Wed Apr 3 05:06:18 2019 -0700
ATLAS-3113: Use index query to search for active entities and better logging in java patch framework
---
.../patches/UniqueAttributePatchHandler.java | 21 +++++++++++++--------
.../store/graph/v2/AtlasGraphUtilsV2.java | 18 +++++++++++++-----
2 files changed, 26 insertions(+), 13 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatchHandler.java b/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatchHandler.java
index 0e707a5..f2238f1 100644
--- a/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatchHandler.java
+++ b/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatchHandler.java
@@ -22,6 +22,7 @@ import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer.UniqueKind;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
@@ -56,7 +57,7 @@ public class UniqueAttributePatchHandler extends AtlasJavaPatchHandler {
for (AtlasEntityType entityType : allEntityTypes) {
String typeName = entityType.getTypeName();
Map<String, AtlasAttribute> uniqAttributes = entityType.getUniqAttributes();
- int entitiesProcessed = 0;
+ int patchAppliedCount = 0;
LOG.info("Applying java patch: {} for type: {}", getPatchId(), typeName);
@@ -67,12 +68,16 @@ public class UniqueAttributePatchHandler extends AtlasJavaPatchHandler {
// register unique attribute property keys in graph
registerUniqueAttrPropertyKeys(attributes);
- Iterator<AtlasVertex> iterator = findActiveEntityVerticesByType(typeName);
+ Iterator<Result<Object, Object>> iterator = findActiveEntityVerticesByType(typeName);
- while (iterator.hasNext()) {
- AtlasVertex entityVertex = iterator.next();
+ int entityCount = 0;
+
+ while (iterator != null && iterator.hasNext()) {
+ AtlasVertex entityVertex = iterator.next().getVertex();
boolean patchApplied = false;
+ entityCount++;
+
for (AtlasAttribute attribute : attributes) {
String uniquePropertyKey = attribute.getVertexUniquePropertyName();
Collection<? extends String> propertyKeys = entityVertex.getPropertyKeys();
@@ -104,11 +109,11 @@ public class UniqueAttributePatchHandler extends AtlasJavaPatchHandler {
}
if (patchApplied) {
- entitiesProcessed++;
+ patchAppliedCount++;
}
- if (entitiesProcessed % 1000 == 0) {
- LOG.info("Java patch: {} : processed {} {} entities.", getPatchId(), entitiesProcessed, typeName);
+ if (entityCount % 1000 == 0) {
+ LOG.info("Java patch: {} : applied {}; processed {} {} entities.", getPatchId(), patchAppliedCount, entityCount, typeName);
}
}
} catch (IndexException e) {
@@ -120,7 +125,7 @@ public class UniqueAttributePatchHandler extends AtlasJavaPatchHandler {
}
}
- LOG.info("Applied java patch ({}) for type: {}; Total processed: {}", getPatchId(), typeName, entitiesProcessed);
+ LOG.info("Applied java patch ({}) for type: {}; Total processed: {}", getPatchId(), typeName, patchAppliedCount);
}
if (patchFailed) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
index 2882f09..80141b4 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
@@ -566,12 +566,20 @@ public class AtlasGraphUtilsV2 {
return ret;
}
- public static Iterator<AtlasVertex> findActiveEntityVerticesByType(String typename) {
- AtlasGraphQuery query = getGraphInstance().query()
- .has(ENTITY_TYPE_PROPERTY_KEY, typename)
- .has(STATE_PROPERTY_KEY, Status.ACTIVE.name());
+ public static Iterator<Result<Object, Object>> findActiveEntityVerticesByType(String typename) {
+ AtlasIndexQuery indexQuery = getActiveEntityIndexQuery(typename);
+
+ return indexQuery.vertices();
+ }
+
+ private static AtlasIndexQuery getActiveEntityIndexQuery(String typename) {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append(INDEX_SEARCH_PREFIX + "\"").append(TYPE_NAME_PROPERTY_KEY).append("\":").append(typename)
+ .append(" AND ")
+ .append(INDEX_SEARCH_PREFIX + "\"").append(STATE_PROPERTY_KEY).append("\":").append(Status.ACTIVE.name());
- return query.vertices().iterator();
+ return getGraphInstance().indexQuery(VERTEX_INDEX, sb.toString());
}
public static List<String> findEntityGUIDsByType(String typename) {