You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/10/23 23:08:27 UTC
[atlas] branch branch-2.0 updated: ATLAS-3417 Improve Atlas
Initialization Performance for fresh installation.
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 465a88f ATLAS-3417 Improve Atlas Initialization Performance for fresh installation.
465a88f is described below
commit 465a88faba0319f29978789ad845ca9fb157c15d
Author: skoritala <sk...@cloudera.com>
AuthorDate: Mon Oct 21 12:03:11 2019 -0700
ATLAS-3417 Improve Atlas Initialization Performance for fresh installation.
After recent changes in atlas quick search improvements, Atlas is taking long time to start on fresh installations.
This checkin tries to improve initialization time for fresh installations.
Signed-off-by: Sarath Subramanian <sa...@apache.org>
(cherry picked from commit 6ad08536043d6192de3b91b7799efbc60461222f)
---
.../repository/graph/GraphBackedSearchIndexer.java | 27 ++++++++++++++++-
.../repository/graph/IndexChangeListener.java | 2 ++
.../atlas/repository/graph/SolrIndexHelper.java | 35 +++++++++++++++-------
3 files changed, 53 insertions(+), 11 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 861e344..7abf761 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -126,6 +126,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (!HAConfiguration.isHAEnabled(configuration)) {
initialize(provider.get());
}
+ notifyInitializationStart();
}
public void addIndexListener(IndexChangeListener listener) {
@@ -221,7 +222,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
//Commit indexes
commit(management);
- notifyChangeListeners(changedTypeDefs);
+ notifyInitializationCompletion(changedTypeDefs);
} catch (RepositoryException | IndexException e) {
LOG.error("Failed to update indexes for changed typedefs", e);
attemptRollback(changedTypeDefs, management);
@@ -967,4 +968,28 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
+ private void notifyInitializationStart() {
+ for (IndexChangeListener indexChangeListener : indexChangeListeners) {
+ try {
+ indexChangeListener.onInitStart();
+ } catch (Throwable t) {
+ LOG.error("Error encountered in notifying the index change listener {}.", indexChangeListener.getClass().getName(), t);
+ //we need to throw exception if any of the listeners throw execption.
+ throw new RuntimeException("Error encountered in notifying the index change listener " + indexChangeListener.getClass().getName(), t);
+ }
+ }
+ }
+
+ private void notifyInitializationCompletion(ChangedTypeDefs changedTypeDefs) {
+ for (IndexChangeListener indexChangeListener : indexChangeListeners) {
+ try {
+ indexChangeListener.onInitCompletion(changedTypeDefs);
+ } catch (Throwable t) {
+ LOG.error("Error encountered in notifying the index change listener {}.", indexChangeListener.getClass().getName(), t);
+ //we need to throw exception if any of the listeners throw execption.
+ throw new RuntimeException("Error encountered in notifying the index change listener " + indexChangeListener.getClass().getName(), t);
+ }
+ }
+ }
+
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java b/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
index b6e6257..4aa16dd 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
@@ -21,4 +21,6 @@ import org.apache.atlas.listener.ChangedTypeDefs;
public interface IndexChangeListener {
void onChange(ChangedTypeDefs changedTypeDefs);
+ void onInitStart();
+ void onInitCompletion(ChangedTypeDefs changedTypeDefs);
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
index 26121fe..7310861 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
@@ -58,6 +58,7 @@ public class SolrIndexHelper implements IndexChangeListener {
public SolrIndexHelper(AtlasTypeRegistry typeRegistry) {
this.typeRegistry = typeRegistry;
}
+ public boolean initializationCompleted = false;
@Override
public void onChange(ChangedTypeDefs changedTypeDefs) {
@@ -65,18 +66,32 @@ public class SolrIndexHelper implements IndexChangeListener {
changedTypeDefs == null || !changedTypeDefs.hasEntityDef()) { // nothing to do if there are no changes to entity-defs
return;
}
+ if(initializationCompleted) {
+ try {
+ AtlasGraph graph = AtlasGraphProvider.getGraphInstance();
+ AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
+ Map<String, Integer> indexFieldName2SearchWeightMap = geIndexFieldNamesWithSearchWeights();
+
+ graphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, indexFieldName2SearchWeightMap);
+ graphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX, getIndexFieldNamesForSuggestions(indexFieldName2SearchWeightMap));
+ } catch (AtlasException e) {
+ LOG.error("Error encountered in handling type system change notification.", e);
+ throw new RuntimeException("Error encountered in handling type system change notification.", e);
+ }
+ }
+ }
- try {
- AtlasGraph graph = AtlasGraphProvider.getGraphInstance();
- AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
- Map<String, Integer> indexFieldName2SearchWeightMap = geIndexFieldNamesWithSearchWeights();
+ @Override
+ public void onInitStart() {
+ LOG.info("SolrIndexHelper Initialization started.");
+ initializationCompleted = false;
+ }
- graphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, indexFieldName2SearchWeightMap);
- graphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX, getIndexFieldNamesForSuggestions(indexFieldName2SearchWeightMap));
- } catch (AtlasException e) {
- LOG.error("Error encountered in handling type system change notification.", e);
- throw new RuntimeException("Error encountered in handling type system change notification.", e);
- }
+ @Override
+ public void onInitCompletion(ChangedTypeDefs changedTypeDefs) {
+ LOG.info("SolrIndexHelper Initialization completed.");
+ initializationCompleted = true;
+ onChange(changedTypeDefs);
}
private List<String> getIndexFieldNamesForSuggestions(Map<String, Integer> indexFieldName2SearchWeightMap) {