You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2019/06/07 22:29:52 UTC
[atlas] branch master updated: ATLAS-3262: disabled full-text index
population when free-text is enabled i.e. for Solr index store
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new b948141 ATLAS-3262: disabled full-text index population when free-text is enabled i.e. for Solr index store
b948141 is described below
commit b9481410bb667c0297ec9fe84c4693f7aec8c4be
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Wed Jun 5 18:49:11 2019 -0700
ATLAS-3262: disabled full-text index population when free-text is enabled i.e. for Solr index store
---
.../repository/graphdb/janus/AtlasJanusGraph.java | 4 +-
.../graphdb/janus/AtlasJanusGraphDatabase.java | 4 +
...Client.java => AtlasJanusGraphIndexClient.java} | 122 ++++++++++++---------
.../janusgraph/diskstorage/solr/Solr6Index.java | 48 ++++----
.../org/apache/atlas/ApplicationProperties.java | 4 +-
.../org/apache/atlas/listener/ChangedTypeDefs.java | 38 +++++--
.../discovery/ClassificationSearchProcessor.java | 13 +--
.../atlas/discovery/EntitySearchProcessor.java | 26 +----
.../atlas/discovery/FreeTextSearchProcessor.java | 66 ++++++-----
.../org/apache/atlas/discovery/SearchContext.java | 107 ++++++++++++------
.../apache/atlas/discovery/SearchProcessor.java | 16 +--
.../repository/graph/GraphBackedSearchIndexer.java | 13 ++-
.../repository/graph/IndexChangeListener.java | 4 +-
.../atlas/repository/graph/SolrIndexHelper.java | 28 ++---
.../repository/patches/AtlasPatchManager.java | 3 +-
.../patches/FreeTextRequestHandlerPatch.java | 64 +++++++++++
.../store/graph/v2/AtlasEntityChangeNotifier.java | 18 +--
.../atlas/util/AtlasRepositoryConfiguration.java | 39 ++++++-
18 files changed, 381 insertions(+), 236 deletions(-)
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index edab08c..499e8d1 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -196,7 +196,9 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
@Override
public AtlasGraphIndexClient getGraphIndexClient() throws AtlasException {
try {
- return new AtlasJanusGraphSolrIndexClient(this);
+ initApplicationProperties();
+
+ return new AtlasJanusGraphIndexClient(this, APPLICATION_PROPERTIES);
} catch (Exception e) {
LOG.error("Error encountered in creating Graph Index Client.", e);
throw new AtlasException(e);
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
index 18e6976..a8945c0 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
@@ -83,6 +83,10 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
Configuration configProperties = ApplicationProperties.get();
+ if (isEmbeddedSolr()) { // AtlasJanusGraphIndexClient.performRequestHandlerAction() fails for embedded-solr; disable freetext until this issue is resolved
+ configProperties.setProperty(ApplicationProperties.ENABLE_FREETEXT_SEARCH_CONF, false);
+ }
+
configProperties.setProperty(SOLR_ZOOKEEPER_URLS, configProperties.getStringArray(SOLR_ZOOKEEPER_URL));
Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
similarity index 56%
rename from graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java
rename to graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
index a55fc36..16be93f 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
@@ -22,6 +22,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.commons.configuration.Configuration;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
@@ -35,67 +36,98 @@ import java.util.*;
import static org.apache.atlas.repository.Constants.FREETEXT_REQUEST_HANDLER;
-public class AtlasJanusGraphSolrIndexClient implements AtlasGraphIndexClient {
- private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphSolrIndexClient.class);
+public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphIndexClient.class);
- private final SolrClient solrClient;
- private final AtlasGraph graph;
+ private final AtlasGraph graph;
+ private final Configuration configuration;
- public AtlasJanusGraphSolrIndexClient(AtlasGraph graph) {
- // get solr client using same settings as that of Janus Graph
- this.solrClient = Solr6Index.getSolrClient();
- this.graph = graph;
-
- if(solrClient == null) {
- LOG.warn("Non SOLR index stores are not supported yet.");
- }
+ public AtlasJanusGraphIndexClient(AtlasGraph graph, Configuration configuration) {
+ this.graph = graph;
+ this.configuration = configuration;
}
@Override
public void applySearchWeight(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) {
- //1) try updating request handler
- //2) if update fails, try creating request handler
+ SolrClient solrClient = null;
try {
- LOG.info("Attempting to update free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
+ solrClient = Solr6Index.getSolrClient(); // get solr client using same settings as that of Janus Graph
- updateSearchWeights(collectionName, attributeName2SearchWeightMap);
+ if (solrClient == null) {
+ LOG.warn("AtlasJanusGraphIndexClient.applySearchWeight(): Non SOLR index stores are not supported yet.");
- LOG.info("Successfully updated free text request handler {} for collection {}..", FREETEXT_REQUEST_HANDLER, collectionName);
+ return;
+ }
- return;
- } catch (Throwable t) {
- LOG.warn("Error encountered in updating request handler {} for collection {}. Attempting to create one", FREETEXT_REQUEST_HANDLER, collectionName, t);
- }
+ //1) try updating request handler
+ //2) if update fails, try creating request handler
- try {
- LOG.info("Attempting to create free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
+ int maxAttempts = configuration != null ? configuration.getInt("index.client.apply.search.weight.max.attempts", 3) : 3;
+ int retryWaitIntervalMs = configuration != null ? configuration.getInt("index.client.apply.search.weight.retry.interval.ms", 1000) : 1000;
+ Throwable lastExcp = null;
+
+ for (int i = 0; i < maxAttempts; i++) {
+ if (i > 0) {
+ LOG.warn("Attempt #{} failed! Waiting for {}ms before retry", i, retryWaitIntervalMs);
+
+ try {
+ Thread.sleep(retryWaitIntervalMs);
+ } catch (Exception excp) {
+ // ignore
+ }
+ }
+
+ try {
+ LOG.info("Attempting to update free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
+
+ updateFreeTextRequestHandler(solrClient, collectionName, attributeName2SearchWeightMap);
+
+ LOG.info("Successfully updated free text request handler {} for collection {}..", FREETEXT_REQUEST_HANDLER, collectionName);
- createFreeTextRequestHandler(collectionName, attributeName2SearchWeightMap);
+ return;
+ } catch (Throwable t) {
+ lastExcp = t;
- LOG.info("Successfully created free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
- } catch (Throwable t) {
- String msg = String.format("Error encountered in creating the request handler '%s' for collection '%s'.", FREETEXT_REQUEST_HANDLER, collectionName);
+ LOG.warn("Error encountered in updating request handler {} for collection {}. Will attempt to create one", FREETEXT_REQUEST_HANDLER, collectionName);
+ }
- LOG.error(msg, t);
+ try {
+ LOG.info("Attempting to create free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
- throw new RuntimeException(msg, t);
+ createFreeTextRequestHandler(solrClient, collectionName, attributeName2SearchWeightMap);
+
+ LOG.info("Successfully created free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
+
+ return;
+ } catch (Throwable t) {
+ lastExcp = t;
+
+ LOG.warn("Error encountered in creating request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName, t);
+ }
+ }
+
+ String msg = String.format("Error encountered in creating/updating request handler %s for collection %s", FREETEXT_REQUEST_HANDLER, collectionName);
+
+ throw lastExcp != null ? new RuntimeException(msg, lastExcp) : new RuntimeException(msg);
+ } finally {
+ Solr6Index.releaseSolrClient(solrClient);
}
}
- private void updateSearchWeights(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) {
- try {
- updateFreeTextRequestHandler(collectionName, attributeName2SearchWeightMap);
- } catch (Throwable t) {
- String msg = String.format("Error encountered in updating the request handler '%s' for collection '%s'", FREETEXT_REQUEST_HANDLER, collectionName);
+ private void updateFreeTextRequestHandler(SolrClient solrClient, String collectionName, Map<String, Integer> attributeName2SearchWeightMap) throws IOException, SolrServerException {
+ String searchWeightString = generateSearchWeightString(graph.getManagementSystem(), collectionName, attributeName2SearchWeightMap);
+ String payLoadString = generatePayLoadForFreeText("update-requesthandler", FREETEXT_REQUEST_HANDLER, searchWeightString);
- LOG.error(msg, t);
+ performRequestHandlerAction(collectionName, solrClient, payLoadString);
+ }
- throw new RuntimeException(msg, t);
- }
+ private void createFreeTextRequestHandler(SolrClient solrClient, String collectionName, Map<String, Integer> attributeName2SearchWeightMap) throws IOException, SolrServerException {
+ String searchWeightString = generateSearchWeightString(graph.getManagementSystem(), collectionName, attributeName2SearchWeightMap);
+ String payLoadString = generatePayLoadForFreeText("create-requesthandler", FREETEXT_REQUEST_HANDLER, searchWeightString);
- LOG.info("Updated free text request handler for collection {}.", collectionName);
+ performRequestHandlerAction(collectionName, solrClient, payLoadString);
}
private String generateSearchWeightString(AtlasGraphManagement management, String indexName, Map<String, Integer> searchWeightsMap) {
@@ -116,20 +148,6 @@ public class AtlasJanusGraphSolrIndexClient implements AtlasGraphIndexClient {
return searchWeightBuilder.toString();
}
- private void updateFreeTextRequestHandler(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) throws IOException, SolrServerException {
- String searchWeightString = generateSearchWeightString(graph.getManagementSystem(), collectionName, attributeName2SearchWeightMap);
- String payLoadString = generatePayLoadForFreeText("update-requesthandler", FREETEXT_REQUEST_HANDLER, searchWeightString);
-
- performRequestHandlerAction(collectionName, solrClient, payLoadString);
- }
-
- private void createFreeTextRequestHandler(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) throws IOException, SolrServerException {
- String searchWeightString = generateSearchWeightString(graph.getManagementSystem(), collectionName, attributeName2SearchWeightMap);
- String payLoadString = generatePayLoadForFreeText("create-requesthandler", FREETEXT_REQUEST_HANDLER, searchWeightString);
-
- performRequestHandlerAction(collectionName, solrClient, payLoadString);
- }
-
@VisibleForTesting
static String generatePayLoadForFreeText(String action, String handlerName, String qfValue) {
return String.format("{" +
diff --git a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
index 0d5ae4f..23a5739 100644
--- a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
+++ b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
@@ -137,7 +137,8 @@ public class Solr6Index implements IndexProvider {
private static final String DEFAULT_ID_FIELD = "id";
private static final char CHROOT_START_CHAR = '/';
- private static Configuration config;
+
+ private static Solr6Index instance = null;
private enum Mode {
HTTP, CLOUD;
@@ -181,7 +182,7 @@ public class Solr6Index implements IndexProvider {
public Solr6Index(final Configuration config) throws BackendException {
// Add Kerberos-enabled SolrHttpClientBuilder
HttpClientUtil.setHttpClientBuilder(new Krb5HttpClientBuilder().getBuilder());
- initConfiguration(config);
+
Preconditions.checkArgument(config!=null);
configuration = config;
mode = Mode.parse(config.get(SOLR_MODE));
@@ -200,34 +201,39 @@ public class Solr6Index implements IndexProvider {
logger.debug("KERBEROS_ENABLED name is " + KERBEROS_ENABLED.getName() + " and it is" + (KERBEROS_ENABLED.isOption() ? " " : " not") + " an option.");
logger.debug("KERBEROS_ENABLED type is " + KERBEROS_ENABLED.getType().name());
}
- solrClient = getSolrClient();
- }
+ solrClient = createSolrClient();
- private static void initConfiguration(Configuration config) {
- if(Solr6Index.config == null) {
- Solr6Index.config = config;
- }
+ Solr6Index.instance = this;
}
public static SolrClient getSolrClient() {
+ return Solr6Index.instance != null ? Solr6Index.instance.createSolrClient() : null;
+ }
+
+ public static void releaseSolrClient(SolrClient solrClient) {
+ if (solrClient != null) {
+ try {
+ solrClient.close();
+ } catch (IOException excp) {
+ logger.warn("Failed to close SolrClient", excp);
+ }
+ }
+ }
+ private SolrClient createSolrClient() {
final ModifiableSolrParams clientParams = new ModifiableSolrParams();
SolrClient solrClient = null;
- if(Solr6Index.config == null) {
- logger.error("The solr client is not being used for the indexing purposes.");
- return null;
- }
- Configuration config = Solr6Index.config;
- Mode mode = Mode.parse(config.get(SOLR_MODE));
+
+ Mode mode = Mode.parse(configuration.get(SOLR_MODE));
switch (mode) {
case CLOUD:
final CloudSolrClient cloudServer = new CloudSolrClient.Builder()
.withLBHttpSolrClientBuilder(
new LBHttpSolrClient.Builder()
.withHttpSolrClientBuilder(new HttpSolrClient.Builder().withInvariantParams(clientParams))
- .withBaseSolrUrls(config.get(HTTP_URLS))
+ .withBaseSolrUrls(configuration.get(HTTP_URLS))
)
- .withZkHost(getZookeeperURLs(config))
+ .withZkHost(getZookeeperURLs(configuration))
.sendUpdatesOnlyToShardLeaders()
.build();
cloudServer.connect();
@@ -235,14 +241,14 @@ public class Solr6Index implements IndexProvider {
logger.info("Created solr client using Cloud based configuration.");
break;
case HTTP:
- clientParams.add(HttpClientUtil.PROP_ALLOW_COMPRESSION, config.get(HTTP_ALLOW_COMPRESSION).toString());
- clientParams.add(HttpClientUtil.PROP_CONNECTION_TIMEOUT, config.get(HTTP_CONNECTION_TIMEOUT).toString());
- clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, config.get(HTTP_MAX_CONNECTIONS_PER_HOST).toString());
- clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS, config.get(HTTP_GLOBAL_MAX_CONNECTIONS).toString());
+ clientParams.add(HttpClientUtil.PROP_ALLOW_COMPRESSION, configuration.get(HTTP_ALLOW_COMPRESSION).toString());
+ clientParams.add(HttpClientUtil.PROP_CONNECTION_TIMEOUT, configuration.get(HTTP_CONNECTION_TIMEOUT).toString());
+ clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, configuration.get(HTTP_MAX_CONNECTIONS_PER_HOST).toString());
+ clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS, configuration.get(HTTP_GLOBAL_MAX_CONNECTIONS).toString());
final HttpClient client = HttpClientUtil.createClient(clientParams);
solrClient = new LBHttpSolrClient.Builder()
.withHttpClient(client)
- .withBaseSolrUrls(config.get(HTTP_URLS))
+ .withBaseSolrUrls(configuration.get(HTTP_URLS))
.build();
logger.info("Created solr client using HTTP based configuration.");
break;
diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
index 6d0153f..6e0aba2 100644
--- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
+++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
@@ -48,8 +48,9 @@ public final class ApplicationProperties extends PropertiesConfiguration {
public static final String STORAGE_BACKEND_CONF = "atlas.graph.storage.backend";
public static final String INDEX_BACKEND_CONF = "atlas.graph.index.search.backend";
public static final String INDEX_MAP_NAME_CONF = "atlas.graph.index.search.map-name";
- public static final String FREE_TEXT_INDEX_ENABLED = "atlas.freetext.index.enabled";
public static final String SOLR_WAIT_SEARCHER_CONF = "atlas.graph.index.search.solr.wait-searcher";
+ public static final String ENABLE_FULLTEXT_SEARCH_CONF = "atlas.search.fulltext.enable";
+ public static final String ENABLE_FREETEXT_SEARCH_CONF = "atlas.search.freetext.enable";
public static final String GRAPHBD_BACKEND_JANUS = "janus";
public static final String STORAGE_BACKEND_HBASE = "hbase";
public static final String STORAGE_BACKEND_HBASE2 = "hbase2";
@@ -57,7 +58,6 @@ public final class ApplicationProperties extends PropertiesConfiguration {
public static final String DEFAULT_GRAPHDB_BACKEND = GRAPHBD_BACKEND_JANUS;
public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = true;
public static final boolean DEFAULT_INDEX_MAP_NAME = false;
- public static final boolean DEFAULT_FREE_TEXT_INDEX_ENABLED = false;
public static final SimpleEntry<String, String> DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true");
public static final SimpleEntry<String, String> DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20");
diff --git a/intg/src/main/java/org/apache/atlas/listener/ChangedTypeDefs.java b/intg/src/main/java/org/apache/atlas/listener/ChangedTypeDefs.java
index 0636677..af00f6c 100644
--- a/intg/src/main/java/org/apache/atlas/listener/ChangedTypeDefs.java
+++ b/intg/src/main/java/org/apache/atlas/listener/ChangedTypeDefs.java
@@ -18,35 +18,37 @@
package org.apache.atlas.listener;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
public class ChangedTypeDefs {
- private List<? extends AtlasBaseTypeDef> createTypeDefs;
+ private List<? extends AtlasBaseTypeDef> createdTypeDefs;
private List<? extends AtlasBaseTypeDef> updatedTypeDefs;
private List<? extends AtlasBaseTypeDef> deletedTypeDefs;
- public ChangedTypeDefs(List<? extends AtlasBaseTypeDef> createTypeDefs,
+ public ChangedTypeDefs(List<? extends AtlasBaseTypeDef> createdTypeDefs,
List<? extends AtlasBaseTypeDef> updatedTypeDefs,
List<? extends AtlasBaseTypeDef> deletedTypeDefs) {
- this.createTypeDefs = createTypeDefs;
+ this.createdTypeDefs = createdTypeDefs;
this.updatedTypeDefs = updatedTypeDefs;
this.deletedTypeDefs = deletedTypeDefs;
}
public ChangedTypeDefs() {
- createTypeDefs = new ArrayList<>();
+ createdTypeDefs = new ArrayList<>();
updatedTypeDefs = new ArrayList<>();
deletedTypeDefs = new ArrayList<>();
}
- public List<? extends AtlasBaseTypeDef> getCreateTypeDefs() {
- return createTypeDefs;
+ public List<? extends AtlasBaseTypeDef> getCreatedTypeDefs() {
+ return createdTypeDefs;
}
- public ChangedTypeDefs setCreateTypeDefs(List<? extends AtlasBaseTypeDef> createTypeDefs) {
- this.createTypeDefs = createTypeDefs;
+ public ChangedTypeDefs setCreatedTypeDefs(List<? extends AtlasBaseTypeDef> createdTypeDefs) {
+ this.createdTypeDefs = createdTypeDefs;
return this;
}
@@ -67,4 +69,24 @@ public class ChangedTypeDefs {
this.deletedTypeDefs = deletedTypeDefs;
return this;
}
+
+ public boolean hasEntityDef() {
+ return hasEntityDef(createdTypeDefs) || hasEntityDef(updatedTypeDefs) || hasEntityDef(deletedTypeDefs);
+ }
+
+ private boolean hasEntityDef(List<? extends AtlasBaseTypeDef> typeDefs) {
+ boolean ret = false;
+
+ if (CollectionUtils.isNotEmpty(typeDefs)) {
+ for (AtlasBaseTypeDef typeDef : typeDefs) {
+ if (typeDef instanceof AtlasEntityDef) {
+ ret = true;
+
+ break;
+ }
+ }
+ }
+
+ return ret;
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index c929d9a..479ddfd 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -41,7 +41,6 @@ import org.slf4j.LoggerFactory;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -82,16 +81,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
- final Set<String> typeAndSubTypes;
- final String typeAndSubTypesQryStr;
-
- if (context.getSearchParameters().getIncludeSubClassifications()) {
- typeAndSubTypes = classificationType.getTypeAndAllSubTypes();
- typeAndSubTypesQryStr = classificationType.getTypeAndAllSubTypesQryStr();
- } else {
- typeAndSubTypes = Collections.singleton(classificationType.getTypeName());
- typeAndSubTypesQryStr = classificationType.getTypeQryStr();
- }
+ final Set<String> typeAndSubTypes = context.getClassificationTypes();
+ final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr();
processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index e4a315f..f7d8f08 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -33,7 +33,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
@@ -65,32 +64,17 @@ public class EntitySearchProcessor extends SearchProcessor {
final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
- final Set<String> typeAndSubTypes;
- final String typeAndSubTypesQryStr;
+ final Set<String> typeAndSubTypes = context.getEntityTypes();
+ final String typeAndSubTypesQryStr = context.getEntityTypesQryStr();
- if (context.getSearchParameters().getIncludeSubTypes()) {
- typeAndSubTypes = entityType.getTypeAndAllSubTypes();
- typeAndSubTypesQryStr = entityType.getTypeAndAllSubTypesQryStr();
- } else {
- typeAndSubTypes = Collections.singleton(entityType.getTypeName());
- typeAndSubTypesQryStr = entityType.getTypeQryStr();
- }
-
- final AtlasClassificationType classificationType = context.getClassificationType();
+ final AtlasClassificationType classificationType = context.getClassificationType();
+ final Set<String> classificationTypeAndSubTypes = context.getClassificationTypes();
final boolean filterClassification;
- final Set<String> classificationTypeAndSubTypes;
if (classificationType != null) {
filterClassification = !context.needClassificationProcessor();
-
- if (context.getSearchParameters().getIncludeSubClassifications()) {
- classificationTypeAndSubTypes = classificationType.getTypeAndAllSubTypes();
- } else {
- classificationTypeAndSubTypes = Collections.singleton(classificationType.getTypeName());
- }
} else {
- filterClassification = false;
- classificationTypeAndSubTypes = Collections.emptySet();
+ filterClassification = false;
}
final Predicate typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator()
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
index b1a84c7..aa43eec 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
@@ -20,14 +20,17 @@ package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
+
/**
* This class is equivalent to legacy FullTextSearchProcessor--except that it uses a better search techniques using SOLR
* than going through Janus Graph index apis.
@@ -37,7 +40,6 @@ public class FreeTextSearchProcessor extends SearchProcessor {
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("FreeTextSearchProcessor");
public static final String SOLR_QT_PARAMETER = "qt";
public static final String SOLR_REQUEST_HANDLER_NAME = "/freetext";
- private static final int MAX_TYPES_STRING_SIZE = 1000;
private final AtlasIndexQuery indexQuery;
@@ -49,50 +51,32 @@ public class FreeTextSearchProcessor extends SearchProcessor {
queryString.append(searchParameters.getQuery());
- String queryFields = null;
- // if search includes entity-type criteria, adding a filter here can help avoid unnecessary
- // processing (and rejection) by subsequent EntitySearchProcessor
- if (context.getEntityType() != null) {
- String typeString = context.getEntityType().getTypeAndAllSubTypesQryStr();
- if (typeString.length() > MAX_TYPES_STRING_SIZE) {
- LOG.info("Dropping the use of types string optimization as there are too many types {} for select type {}.", typeString, context.getEntityType().getTypeName());
- } else {
- LOG.debug("Using the use of types string optimization as there are too many types {} for select type {}.", typeString, context.getEntityType().getTypeName());
-
- final Set<String> types = context.getEntityType().getTypeAndAllSubTypes();
- final AtlasGraphManagement managementSystem = context.getGraph().getManagementSystem();
- AtlasPropertyKey entityTypeNamePropertyKey = managementSystem.getPropertyKey(AtlasGraphUtilsV2.encodePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY));
- String encodedPropertyName = managementSystem.getIndexFieldName(Constants.VERTEX_INDEX, entityTypeNamePropertyKey);
-
-
+ if (StringUtils.isNotEmpty(context.getEntityTypesQryStr()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
+ queryString.append(AND_STR).append(context.getEntityTypesQryStr());
+ }
- StringBuilder typesStringBuilder = new StringBuilder();
- for(String typeName: types) {
- typesStringBuilder.append(" ").append(typeName);
- }
- //append the query with type and substypes listed in it
- String typesString = typesStringBuilder.toString();
- queryString.append(" AND +").append(encodedPropertyName).append(":[");
- queryString.append(typesStringBuilder.toString());
- queryString.append("]");
- }
+ if (StringUtils.isNotEmpty(context.getClassificationTypesQryStr()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
+ queryString.append(AND_STR).append(context.getClassificationTypesQryStr());
}
- //just use the query string as is
+ // just use the query string as is
LOG.debug("Using query string '{}'.", queryString);
+
indexQuery = context.getGraph().indexQuery(prepareGraphIndexQueryParameters(context, queryString));
}
private GraphIndexQueryParameters prepareGraphIndexQueryParameters(SearchContext context, StringBuilder queryString) {
- List<AtlasIndexQueryParameter> parameters = new ArrayList<AtlasIndexQueryParameter>();
+ List<AtlasIndexQueryParameter> parameters = new ArrayList<>();
+
parameters.add(context.getGraph().indexQueryParameter(SOLR_QT_PARAMETER, SOLR_REQUEST_HANDLER_NAME));
+
return new GraphIndexQueryParameters(Constants.VERTEX_INDEX, queryString.toString(), 0, parameters);
}
@Override
public List<AtlasVertex> execute() {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> FullTextSearchProcessorUsingFreeText.execute({})", context);
+ LOG.debug("==> FreeTextSearchProcessor.execute({})", context);
}
List<AtlasVertex> ret = new ArrayList<>();
@@ -100,7 +84,7 @@ public class FreeTextSearchProcessor extends SearchProcessor {
AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
- perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FullTextSearchProcessorUsingFreeText.execute(" + context + ")");
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FreeTextSearchProcessor.execute(" + context + ")");
}
try {
@@ -136,19 +120,33 @@ public class FreeTextSearchProcessor extends SearchProcessor {
resultCount++;
+ String entityTypeName = AtlasGraphUtilsV2.getTypeName(vertex);
+
// skip non-entity vertices
- if (!AtlasGraphUtilsV2.isEntityVertex(vertex)) {
+ if (StringUtils.isEmpty(entityTypeName) || StringUtils.isEmpty(AtlasGraphUtilsV2.getIdFromVertex(vertex))) {
if (LOG.isDebugEnabled()) {
- LOG.debug("FullTextSearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId());
+ LOG.debug("FreeTextSearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId());
}
continue;
}
+ if (!context.includeEntityType(entityTypeName)) {
+ continue;
+ }
+
if (activeOnly && AtlasGraphUtilsV2.getState(vertex) != AtlasEntity.Status.ACTIVE) {
continue;
}
+ if (context.getClassificationType() != null) {
+ List<String> entityClassifications = GraphHelper.getAllTraitNames(vertex);
+
+ if (!context.includeClassificationTypes(entityClassifications)) {
+ continue;
+ }
+ }
+
entityVertices.add(vertex);
}
@@ -170,7 +168,7 @@ public class FreeTextSearchProcessor extends SearchProcessor {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== FullTextSearchProcessor.execute({}): ret.size()={}", context, ret.size());
+ LOG.debug("<== FreeTextSearchProcessor.execute({}): ret.size()={}", context, ret.size());
}
return ret;
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 50edb9f..9bd0382 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -18,9 +18,7 @@
package org.apache.atlas.discovery;
-import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasErrorCode;
-import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
@@ -39,16 +37,13 @@ import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
@@ -70,21 +65,16 @@ public class SearchContext {
private final AtlasClassificationType classificationType;
private SearchProcessor searchProcessor;
private boolean terminateSearch = false;
- private static boolean isIndexSolrBased = false;
+ private final Set<String> typeAndSubTypes;
+ private final Set<String> classificationTypeAndSubTypes;
+ private final String typeAndSubTypesQryStr;
+ private final String classificationTypeAndSubTypesQryStr;
public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
- static {
- try {
- isIndexSolrBased = ApplicationProperties.INDEX_BACKEND_SOLR.equalsIgnoreCase(ApplicationProperties.get().getString(ApplicationProperties.INDEX_BACKEND_CONF));
- } catch (AtlasException e) {
- String msg = String.format("Error encountered in verifying the backend index mode.");
- LOG.error(msg, e);
- throw new RuntimeException(msg, e);
- };
- }
+
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
String classificationName = searchParameters.getClassification();
@@ -119,17 +109,48 @@ public class SearchContext {
// Invalid attributes will raise an exception with 400 error code
validateAttributes(classificationType, searchParameters.getTagFilters());
+ if (entityType != null) {
+ if (searchParameters.getIncludeSubTypes()) {
+ typeAndSubTypes = entityType.getTypeAndAllSubTypes();
+ typeAndSubTypesQryStr = entityType.getTypeAndAllSubTypesQryStr();
+ } else {
+ typeAndSubTypes = Collections.singleton(entityType.getTypeName());
+ typeAndSubTypesQryStr = entityType.getTypeQryStr();
+ }
+ } else {
+ typeAndSubTypes = Collections.emptySet();
+ typeAndSubTypesQryStr = "";
+ }
+
+ if (classificationType != null) {
+ if (classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_NOT_CLASSIFIED || classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION) {
+ classificationTypeAndSubTypes = Collections.emptySet();
+ classificationTypeAndSubTypesQryStr = "";
+ } else if (searchParameters.getIncludeSubClassifications()) {
+ classificationTypeAndSubTypes = classificationType.getTypeAndAllSubTypes();
+ classificationTypeAndSubTypesQryStr = classificationType.getTypeAndAllSubTypesQryStr();
+ } else {
+ classificationTypeAndSubTypes = Collections.singleton(classificationType.getTypeName());
+ classificationTypeAndSubTypesQryStr = classificationType.getTypeQryStr();
+ }
+ } else {
+ classificationTypeAndSubTypes = Collections.emptySet();
+ classificationTypeAndSubTypesQryStr = "";
+ }
+
if (glossaryTermVertex != null) {
addProcessor(new TermSearchProcessor(this, getAssignedEntities(glossaryTermVertex)));
}
if (needFullTextProcessor()) {
- if(!isFreeTextIndexEnabled()) {
- LOG.info("Using Full Text index based search.");
- addProcessor(new FullTextSearchProcessor(this));
- }else {
- LOG.info("Using Free Text index based search.");
+ if (AtlasRepositoryConfiguration.isFreeTextSearchEnabled()) {
+ LOG.debug("Using Free Text index based search.");
+
addProcessor(new FreeTextSearchProcessor(this));
+ } else {
+ LOG.debug("Using Full Text index based search.");
+
+ addProcessor(new FullTextSearchProcessor(this));
}
}
@@ -157,8 +178,36 @@ public class SearchContext {
public AtlasClassificationType getClassificationType() { return classificationType; }
+ public Set<String> getEntityTypes() { return typeAndSubTypes; }
+
+ public Set<String> getClassificationTypes() { return classificationTypeAndSubTypes; }
+
+ public String getEntityTypesQryStr() { return typeAndSubTypesQryStr; }
+
+ public String getClassificationTypesQryStr() { return classificationTypeAndSubTypesQryStr; }
+
public SearchProcessor getSearchProcessor() { return searchProcessor; }
+ public boolean includeEntityType(String entityType) {
+ return typeAndSubTypes.isEmpty() || typeAndSubTypes.contains(entityType);
+ }
+
+ public boolean includeClassificationTypes(Collection<String> classificationTypes) {
+ final boolean ret;
+
+ if (classificationType == null) {
+ ret = true;
+ } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
+ ret = CollectionUtils.isEmpty(classificationTypes);
+ } else if (classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION) {
+ ret = CollectionUtils.isNotEmpty(classificationTypes);
+ } else {
+ ret = CollectionUtils.containsAny(classificationTypeAndSubTypes, classificationTypes);
+ }
+
+ return ret;
+ }
+
public boolean terminateSearch() { return terminateSearch; }
public void terminateSearch(boolean terminateSearch) { this.terminateSearch = terminateSearch; }
@@ -284,18 +333,4 @@ public class SearchContext {
private AtlasEntityType getTermEntityType() {
return typeRegistry.getEntityTypeByName(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ENTITY_TYPE);
}
-
- public static boolean isFreeTextIndexEnabled() {
- try {
- return isIndexSolrBased() && ApplicationProperties.get().getBoolean(ApplicationProperties.FREE_TEXT_INDEX_ENABLED, ApplicationProperties.DEFAULT_FREE_TEXT_INDEX_ENABLED);
- } catch (AtlasException e) {
- String msg = String.format("Error encountered in fetching the configuration %s.", ApplicationProperties.FREE_TEXT_INDEX_ENABLED);
- LOG.error(msg, e);
- throw new RuntimeException(msg, e);
- }
- }
-
- public static boolean isIndexSolrBased() {
- return isIndexSolrBased;
- }
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 2e09bed..a7ccaeb 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -29,12 +29,8 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
-import org.apache.atlas.type.AtlasArrayType;
-import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasEnumType;
-import org.apache.atlas.type.AtlasStructType;
+import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
-import org.apache.atlas.type.AtlasType;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.SearchPredicateUtil.*;
import org.apache.commons.collections.CollectionUtils;
@@ -46,15 +42,7 @@ import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.regex.Pattern;
import static org.apache.atlas.util.SearchPredicateUtil.*;
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 56655a8..408dac4 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -165,13 +165,15 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (LOG.isDebugEnabled()) {
LOG.debug("Processing changed typedefs {}", changedTypeDefs);
}
+
AtlasGraphManagement management = null;
+
try {
management = provider.get().getManagementSystem();
// Update index for newly created types
- if (CollectionUtils.isNotEmpty(changedTypeDefs.getCreateTypeDefs())) {
- for (AtlasBaseTypeDef typeDef : changedTypeDefs.getCreateTypeDefs()) {
+ if (CollectionUtils.isNotEmpty(changedTypeDefs.getCreatedTypeDefs())) {
+ for (AtlasBaseTypeDef typeDef : changedTypeDefs.getCreatedTypeDefs()) {
updateIndexForTypeDef(management, typeDef);
}
}
@@ -196,7 +198,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.error("Failed to update indexes for changed typedefs", e);
attemptRollback(changedTypeDefs, management);
}
- notifyChangeListeners();
+
+ notifyChangeListeners(changedTypeDefs);
}
public Set<String> getVertexIndexKeys() {
@@ -805,10 +808,10 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.info("Index creation for type {} complete", typeDef.getName());
}
- private void notifyChangeListeners() {
+ private void notifyChangeListeners(ChangedTypeDefs changedTypeDefs) {
for (IndexChangeListener indexChangeListener : indexChangeListeners) {
try {
- indexChangeListener.onChange();
+ indexChangeListener.onChange(changedTypeDefs);
} catch (Throwable t) {
LOG.error("Error encountered in notifying the index change listener {}.", indexChangeListener.getClass().getName(), t);
//we need to throw exception if any of the listeners throw execption.
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java b/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
index 9cde947..b6e6257 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
@@ -17,6 +17,8 @@
*/
package org.apache.atlas.repository.graph;
+import org.apache.atlas.listener.ChangedTypeDefs;
+
public interface IndexChangeListener {
- void onChange();
+ void onChange(ChangedTypeDefs changedTypeDefs);
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
index 20a517f..205662c 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
@@ -18,13 +18,14 @@
package org.apache.atlas.repository.graph;
import org.apache.atlas.AtlasException;
-import org.apache.atlas.discovery.SearchContext;
+import org.apache.atlas.listener.ChangedTypeDefs;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,6 +36,7 @@ import java.util.Map;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_TEXT_KEY;
+import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
/**
This is a component that will go through all entity type definitions and create free text index
@@ -45,6 +47,8 @@ public class SolrIndexHelper implements IndexChangeListener {
private static final Logger LOG = LoggerFactory.getLogger(SolrIndexHelper.class);
public static final int DEFAULT_SEARCHWEIGHT_FOR_STRINGS = 3;
+ public static final int SEARCHWEIGHT_FOR_CLASSIFICATIONS = 10;
+ public static final int SEARCHWEIGHT_FOR_TYPENAME = 1;
private final AtlasTypeRegistry typeRegistry;
@@ -54,12 +58,9 @@ public class SolrIndexHelper implements IndexChangeListener {
}
@Override
- public void onChange() {
- LOG.info("SolrIndexHelper.onChange()");
-
- if(!SearchContext.isIndexSolrBased()) {
- LOG.warn("Not a Solr based index store. Free text search is not supported");
-
+ public void onChange(ChangedTypeDefs changedTypeDefs) {
+ if (!AtlasRepositoryConfiguration.isFreeTextSearchEnabled() ||
+ changedTypeDefs == null || !changedTypeDefs.hasEntityDef()) { // nothing to do if there are no changes to entity-defs
return;
}
@@ -79,7 +80,8 @@ public class SolrIndexHelper implements IndexChangeListener {
Map<String, Integer> attributesWithSearchWeights = new HashMap<>();
Collection<AtlasEntityDef> allEntityDefs = typeRegistry.getAllEntityDefs();
- attributesWithSearchWeights.put(CLASSIFICATION_TEXT_KEY,10);
+ attributesWithSearchWeights.put(CLASSIFICATION_TEXT_KEY, SEARCHWEIGHT_FOR_CLASSIFICATIONS);
+ attributesWithSearchWeights.put(TYPE_NAME_PROPERTY_KEY, SEARCHWEIGHT_FOR_TYPENAME);
if (CollectionUtils.isNotEmpty(allEntityDefs)) {
for (AtlasEntityDef entityDef : allEntityDefs) {
@@ -106,14 +108,14 @@ public class SolrIndexHelper implements IndexChangeListener {
//this will make the string data searchable like in FullTextIndex Searcher using Free Text searcher.
searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
} else if (!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) { //validate the value provided in the model.
- String msg = String.format("Invalid search weight '%d' for attribute %s.%s", searchWeight, entityDef.getName(), attributeName);
-
- LOG.error(msg);
+ LOG.warn("Invalid search weight {} for attribute {}.{}. Will use default {}", searchWeight, entityDef.getName(), attributeName, DEFAULT_SEARCHWEIGHT_FOR_STRINGS);
- throw new RuntimeException(msg);
+ searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
}
- LOG.info("Applying search weight {} for attribute {}.{}", searchWeight, entityDef.getName(), attributeName);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Applying search weight {} for attribute {}.{}", searchWeight, entityDef.getName(), attributeName);
+ }
attributesWithSearchWeights.put(attributeName, searchWeight);
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
index 259f246..567fcb2 100644
--- a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
+++ b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
@@ -51,7 +51,8 @@ public class AtlasPatchManager {
public void applyAll() {
final AtlasPatchHandler handlers[] = {
new UniqueAttributePatch(context),
- new ClassificationTextPatch(context)
+ new ClassificationTextPatch(context),
+ new FreeTextRequestHandlerPatch(context)
};
try {
diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/FreeTextRequestHandlerPatch.java b/repository/src/main/java/org/apache/atlas/repository/patches/FreeTextRequestHandlerPatch.java
new file mode 100644
index 0000000..7a6149f
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/patches/FreeTextRequestHandlerPatch.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.patches;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.listener.ChangedTypeDefs;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.repository.graph.SolrIndexHelper;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED;
+
+public class FreeTextRequestHandlerPatch extends AtlasPatchHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(FreeTextRequestHandlerPatch.class);
+
+ private static final String PATCH_ID = "JAVA_PATCH_0000_003";
+ private static final String PATCH_DESCRIPTION = "Creates Solr request handler for use in free-text searches";
+
+ private final PatchContext context;
+
+ public FreeTextRequestHandlerPatch(PatchContext context) {
+ super(context.getPatchRegistry(), PATCH_ID, PATCH_DESCRIPTION);
+
+ this.context = context;
+ }
+
+ @Override
+ public void apply() throws AtlasBaseException {
+ AtlasTypeRegistry typeRegistry = context.getTypeRegistry();
+ Collection<AtlasEntityDef> entityDefs = typeRegistry.getAllEntityDefs();
+
+ if (CollectionUtils.isNotEmpty(entityDefs)) {
+ SolrIndexHelper indexHelper = new SolrIndexHelper(typeRegistry);
+ ChangedTypeDefs changedTypeDefs = new ChangedTypeDefs(null, new ArrayList<>(entityDefs), null);
+
+ indexHelper.onChange(changedTypeDefs);
+ }
+
+ setStatus(APPLIED);
+
+ LOG.info("FreeTextRequestHandlerPatch.apply(): patchId={}, status={}", getPatchId(), getStatus());
+ }
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
index 035b02c..2e47a50 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
@@ -481,16 +481,12 @@ public class AtlasEntityChangeNotifier {
}
private void doFullTextMapping(List<AtlasEntityHeader> entityHeaders) {
- if (CollectionUtils.isEmpty(entityHeaders)) {
+ if(AtlasRepositoryConfiguration.isFreeTextSearchEnabled() || !AtlasRepositoryConfiguration.isFullTextSearchEnabled()) {
return;
}
- try {
- if(!AtlasRepositoryConfiguration.isFullTextSearchEnabled()) {
- return;
- }
- } catch (AtlasException e) {
- LOG.warn("Unable to determine if FullText is disabled. Proceeding with FullText mapping");
+ if (CollectionUtils.isEmpty(entityHeaders)) {
+ return;
}
MetricRecorder metric = RequestContext.get().startMetricRecord("fullTextMapping");
@@ -520,12 +516,8 @@ public class AtlasEntityChangeNotifier {
}
private void updateFullTextMapping(String entityId, List<AtlasClassification> classifications) {
- try {
- if(!AtlasRepositoryConfiguration.isFullTextSearchEnabled()) {
- return;
- }
- } catch (AtlasException e) {
- LOG.warn("Unable to determine if FullText is disabled. Proceeding with FullText mapping");
+ if(AtlasRepositoryConfiguration.isFreeTextSearchEnabled() || !AtlasRepositoryConfiguration.isFullTextSearchEnabled()) {
+ return;
}
if (StringUtils.isEmpty(entityId) || CollectionUtils.isEmpty(classifications)) {
diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java b/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java
index 1d29605..c37d757 100644
--- a/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java
+++ b/repository/src/main/java/org/apache/atlas/util/AtlasRepositoryConfiguration.java
@@ -48,7 +48,6 @@ public class AtlasRepositoryConfiguration {
private static final Integer DEFAULT_TYPE_UPDATE_LOCK_MAX_WAIT_TIME_IN_SECONDS = Integer.valueOf(15);
private static final String CONFIG_TYPE_UPDATE_LOCK_MAX_WAIT_TIME_IN_SECONDS = "atlas.server.type.update.lock.max.wait.time.seconds";
- private static final String ENABLE_FULLTEXT_SEARCH_PROPERTY = "atlas.search.fulltext.enable";
private static final String JANUS_GRAPH_DATABASE_IMPLEMENTATION_CLASS = "org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase";
private static final String DEFAULT_GRAPH_DATABASE_IMPLEMENTATION_CLASS = JANUS_GRAPH_DATABASE_IMPLEMENTATION_CLASS;
@@ -56,13 +55,25 @@ public class AtlasRepositoryConfiguration {
private static List<String> skippedOperations = null;
private static final String ENTITY_NOTIFICATION_VERSION_PROPERTY = "atlas.notification.entity.version";
+ private static boolean isInitialized = false;
+ private static boolean isFullTextSearchEnabled = true;
+ private static boolean isFreeTextSearchEnabled = true;
+
/**
* Configures whether the full text vertex property is populated. Turning this off
* effectively disables full text searches, since all no entities created or updated after
* turning this off will match full text searches.
*/
- public static boolean isFullTextSearchEnabled() throws AtlasException {
- return ApplicationProperties.get().getBoolean(ENABLE_FULLTEXT_SEARCH_PROPERTY, true);
+ public static boolean isFullTextSearchEnabled() {
+ initialize();
+
+ return isFullTextSearchEnabled;
+ }
+
+ public static boolean isFreeTextSearchEnabled() {
+ initialize();
+
+ return isFreeTextSearchEnabled;
}
public static boolean isV2EntityNotificationEnabled() {
@@ -236,4 +247,26 @@ public class AtlasRepositoryConfiguration {
return ret == null ? DEFAULT_TYPE_UPDATE_LOCK_MAX_WAIT_TIME_IN_SECONDS : ret;
}
+
+ private static void initialize() {
+ if (!isInitialized) {
+ try {
+ isFreeTextSearchEnabled = ApplicationProperties.get().getBoolean(ApplicationProperties.ENABLE_FREETEXT_SEARCH_CONF, true);
+
+ if (isFreeTextSearchEnabled) { // currently free-text is supported only for Solr
+ isFreeTextSearchEnabled = ApplicationProperties.INDEX_BACKEND_SOLR.equalsIgnoreCase(ApplicationProperties.get().getString(ApplicationProperties.INDEX_BACKEND_CONF));
+ }
+
+ if (isFreeTextSearchEnabled) { // if free-text is enabled, disable full-text - to avoid performance penalty
+ isFullTextSearchEnabled = false;
+ } else {
+ isFullTextSearchEnabled = ApplicationProperties.get().getBoolean(ApplicationProperties.ENABLE_FULLTEXT_SEARCH_CONF, true);
+ }
+
+ isInitialized = true;
+ } catch (AtlasException excp) {
+ LOG.error("Failed to initialize. isFullTextSearchEnabled={}, isFreeTextSearchEnabled={}", isFullTextSearchEnabled, isFreeTextSearchEnabled, excp);
+ }
+ }
+ }
}