You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2019/06/06 00:58:16 UTC
[atlas] branch master updated: ATLAS-3246: enhancements in
free-text search functionality
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 17803ee ATLAS-3246: enhancements in free-text search functionality
17803ee is described below
commit 17803eed8adfaa3c68b7de806085797b751eeec9
Author: skoritala <sk...@cloudera.com>
AuthorDate: Tue May 28 16:11:30 2019 -0700
ATLAS-3246: enhancements in free-text search functionality
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
---
.../org/apache/atlas/repository/Constants.java | 2 +-
distro/src/bin/atlas_config.py | 4 +-
distro/src/conf/solr/schema.xml | 26 ----
distro/src/conf/solr/solrconfig.xml | 15 ---
.../repository/graphdb/AtlasGraphIndexClient.java | 13 +-
.../repository/graphdb/janus/AtlasJanusGraph.java | 2 +-
.../janus/AtlasJanusGraphSolrIndexClient.java | 135 ++++++++++++++++++---
.../apache/atlas/model/typedef/AtlasStructDef.java | 2 +-
.../repository/graph/GraphBackedSearchIndexer.java | 108 +++++++----------
.../repository/graph/IndexChangeListener.java | 21 +---
.../atlas/repository/graph/SolrIndexHelper.java | 121 ++++++++++++++++++
11 files changed, 303 insertions(+), 146 deletions(-)
diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java
index c0851fe..46c8b01 100644
--- a/common/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/common/src/main/java/org/apache/atlas/repository/Constants.java
@@ -39,7 +39,7 @@ public final class Constants {
public static final String GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "guid");
public static final String RELATIONSHIP_GUID_PROPERTY_KEY = encodePropertyKey(RELATIONSHIP_PROPERTY_KEY_PREFIX + GUID_PROPERTY_KEY);
public static final String HISTORICAL_GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "historicalGuids");
-
+ public static final String FREETEXT_REQUEST_HANDLER = "/freetext";
/**
* Entity type name property key.
*/
diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py
index 5079d4b..f09026f 100755
--- a/distro/src/bin/atlas_config.py
+++ b/distro/src/bin/atlas_config.py
@@ -32,7 +32,7 @@ LIB = "lib"
CONF = "conf"
LOG = "logs"
WEBAPP = "server" + os.sep + "webapp"
-SOLR_CONF_DIR = "conf" + os.sep + "solr"
+CONFIG_SETS_CONF = "server" + os.sep + "solr" + os.sep + "configsets" + os.sep + "_default" + os.sep + "conf"
DATA = "data"
ATLAS_CONF = "ATLAS_CONF"
ATLAS_LOG = "ATLAS_LOG_DIR"
@@ -112,7 +112,7 @@ def elasticsearchBinDir(dir):
return os.environ.get(SOLR_BIN, os.path.join(dir, "elasticsearch", BIN))
def solrConfDir(dir):
- return os.environ.get(SOLR_CONF, os.path.join(dir, SOLR_CONF_DIR))
+ return os.environ.get(SOLR_CONF, os.path.join(dir, "solr", CONFIG_SETS_CONF))
def solrPort():
return os.environ.get(SOLR_PORT, DEFAULT_SOLR_PORT)
diff --git a/distro/src/conf/solr/schema.xml b/distro/src/conf/solr/schema.xml
index 3f82ed7..7c2ad3c 100644
--- a/distro/src/conf/solr/schema.xml
+++ b/distro/src/conf/solr/schema.xml
@@ -524,20 +524,6 @@
class="solr.UUIDField"
indexed="true" />
- <fieldType name="freetext" class="solr.TextField" omitNorms="true"
- omitTermFreqAndPositions="true">
- <analyzer type="index">
- <tokenizer class="solr.StandardTokenizerFactory"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- <filter class="solr.EdgeNGramFilterFactory" minGramSize="2" maxGramSize="16" />
- <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
- </analyzer>
- <analyzer type="query">
- <tokenizer class="solr.KeywordTokenizerFactory"/>
- <filter class="solr.LowerCaseFilterFactory"/>
- </analyzer>
- </fieldType>
-
<dynamicField name="*_uuid" type="uuid" indexed="true" stored="true"/>
@@ -545,16 +531,4 @@
<field name="ttl" type="string" indexed="true" stored="true" />
<field name="expire_at" type="date" indexed="true" stored="true" />
<field name="timestamp" type="date" indexed="true" stored="true" />
- <field name="allt1s" type="freetext" multiValued="true" indexed="true" stored="false"/>
- <field name="allt2s" type="freetext" multiValued="true" indexed="true" stored="false"/>
- <field name="allt3s" type="freetext" multiValued="true" indexed="true" stored="false"/>
- <field name="allt4s" type="freetext" multiValued="true" indexed="true" stored="false"/>
- <field name="allt5s" type="freetext" multiValued="true" indexed="true" stored="false"/>
- <field name="allt6s" type="freetext" multiValued="true" indexed="true" stored="false"/>
- <field name="allt7s" type="freetext" multiValued="true" indexed="true" stored="false"/>
- <field name="allt8s" type="freetext" multiValued="true" indexed="true" stored="false"/>
- <field name="allt9s" type="freetext" multiValued="true" indexed="true" stored="false"/>
- <field name="allt10s" type="freetext" multiValued="true" indexed="true" stored="false"/>
-
-
</schema>
diff --git a/distro/src/conf/solr/solrconfig.xml b/distro/src/conf/solr/solrconfig.xml
index a83e03b..21d19ef 100644
--- a/distro/src/conf/solr/solrconfig.xml
+++ b/distro/src/conf/solr/solrconfig.xml
@@ -478,21 +478,6 @@
</lst>
</requestHandler>
- <requestHandler name="/freetext" class="solr.SearchHandler">
- <lst name="defaults">
- <str name="defType">edismax</str>
- <int name="rows">100</int>
- <str name="lowercaseOperators">true</str>
- <str name="qf">
- allt10s^10 allt9s^9 allt8s^8 allt7s^7 allt6s^6 allt5s^5 allt4s^4 allt3s^3 allt2s^2 allt1s^1
- </str>
- <str name="hl.fl">*</str>
- <str name="hl.requireFieldMatch">true</str>
- <str name="lowercaseOperators">true</str>
- <str name="facet.limit">5</str>
- </lst>
- </requestHandler>
-
<!--
The export request handler is used to export full sorted result sets.
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
index 482e6f6..2ced6fb 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
@@ -17,15 +17,18 @@
*/
package org.apache.atlas.repository.graphdb;
+import java.util.Map;
+
/**
* Represents a graph client work with indices used by Jansgraph.
*/
public interface AtlasGraphIndexClient {
+
/**
- * The implementers should create a mapping from source propertyName to mapping field name.
- * @param indexName the name of index that needs to be worked with.
- * @param sourcePropertyName the name of the source attribute.
- * @param targetPropertyName the name of the target attribute to which the mapping is getting done.
+ * The implementers should apply the search weights for the passed in attributes.
+ * @param collectionName the name of the collection for which the search weight needs to be applied
+ * @param attributeName2SearchWeightMap the map containing search weights from attribute name to search weights.
*/
- void createCopyField(String indexName, String sourcePropertyName, String targetPropertyName);
+ void applySearchWeight(String collectionName, Map<String, Integer> attributeName2SearchWeightMap);
+
}
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index e973f72..edab08c 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -196,7 +196,7 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
@Override
public AtlasGraphIndexClient getGraphIndexClient() throws AtlasException {
try {
- return new AtlasJanusGraphSolrIndexClient();
+ return new AtlasJanusGraphSolrIndexClient(this);
} catch (Exception e) {
LOG.error("Error encountered in creating Graph Index Client.", e);
throw new AtlasException(e);
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java
index f8f8061..a55fc36 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphSolrIndexClient.java
@@ -17,45 +17,144 @@
*/
package org.apache.atlas.repository.graphdb.janus;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.request.schema.SchemaRequest;
-import org.apache.solr.client.solrj.response.schema.SchemaResponse;
+import org.apache.solr.client.solrj.request.V2Request;
import org.janusgraph.diskstorage.solr.Solr6Index;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.Arrays;
+import java.util.*;
+
+import static org.apache.atlas.repository.Constants.FREETEXT_REQUEST_HANDLER;
public class AtlasJanusGraphSolrIndexClient implements AtlasGraphIndexClient {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphSolrIndexClient.class);
+
private final SolrClient solrClient;
+ private final AtlasGraph graph;
+
+
+ public AtlasJanusGraphSolrIndexClient(AtlasGraph graph) {
+ // get solr client using same settings as that of Janus Graph
+ this.solrClient = Solr6Index.getSolrClient();
+ this.graph = graph;
- public AtlasJanusGraphSolrIndexClient() throws Exception {
- //well, this is temp hack to get solr client using same settings as that of Janus Graph
- solrClient = Solr6Index.getSolrClient();
if(solrClient == null) {
- LOG.warn("The indexing system is not solr based. Non SOLR based indexing systems are not supported yet.");
+ LOG.warn("Non SOLR index stores are not supported yet.");
}
}
@Override
- public void createCopyField(String collectionName, String srcFieldName, String mappedCopyFieldName) {
- if(solrClient == null) {
- LOG.error("The indexing system is not solr based. Copy fields can not be created in non SOLR based indexing systems. This request will be treated as no op.");
+ public void applySearchWeight(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) {
+ //1) try updating request handler
+ //2) if update fails, try creating request handler
+
+ try {
+ LOG.info("Attempting to update free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
+
+ updateSearchWeights(collectionName, attributeName2SearchWeightMap);
+
+ LOG.info("Successfully updated free text request handler {} for collection {}..", FREETEXT_REQUEST_HANDLER, collectionName);
+
return;
+ } catch (Throwable t) {
+ LOG.warn("Error encountered in updating request handler {} for collection {}. Attempting to create one", FREETEXT_REQUEST_HANDLER, collectionName, t);
}
- SchemaRequest.AddCopyField addCopyFieldRequest =
- new SchemaRequest.AddCopyField(srcFieldName, Arrays.asList(mappedCopyFieldName));
- SchemaResponse.UpdateResponse addCopyFieldResponse = null;
+
try {
- addCopyFieldResponse = addCopyFieldRequest.process(solrClient, collectionName);
- } catch (SolrServerException | IOException e) {
- String msg = String.format("Error encountered in creating the copy field from %s to %s for collection %s.", srcFieldName, mappedCopyFieldName, collectionName);
- LOG.error(msg);
- throw new RuntimeException(msg, e);
+ LOG.info("Attempting to create free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
+
+ createFreeTextRequestHandler(collectionName, attributeName2SearchWeightMap);
+
+ LOG.info("Successfully created free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
+ } catch (Throwable t) {
+ String msg = String.format("Error encountered in creating the request handler '%s' for collection '%s'.", FREETEXT_REQUEST_HANDLER, collectionName);
+
+ LOG.error(msg, t);
+
+ throw new RuntimeException(msg, t);
}
}
+
+ private void updateSearchWeights(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) {
+ try {
+ updateFreeTextRequestHandler(collectionName, attributeName2SearchWeightMap);
+ } catch (Throwable t) {
+ String msg = String.format("Error encountered in updating the request handler '%s' for collection '%s'", FREETEXT_REQUEST_HANDLER, collectionName);
+
+ LOG.error(msg, t);
+
+ throw new RuntimeException(msg, t);
+ }
+
+ LOG.info("Updated free text request handler for collection {}.", collectionName);
+ }
+
+ private String generateSearchWeightString(AtlasGraphManagement management, String indexName, Map<String, Integer> searchWeightsMap) {
+ StringBuilder searchWeightBuilder = new StringBuilder();
+ Set<Map.Entry<String, Integer>> searchWeightFields = searchWeightsMap.entrySet();
+
+ for (Map.Entry<String, Integer> entry : searchWeightFields) {
+ AtlasPropertyKey propertyKey = management.getPropertyKey(entry.getKey());
+ String indexFieldName = management.getIndexFieldName(indexName, propertyKey);
+
+ searchWeightBuilder.append(" ")
+ .append(indexFieldName)
+ .append("^")
+ .append(entry.getValue().intValue());
+
+ }
+
+ return searchWeightBuilder.toString();
+ }
+
+ private void updateFreeTextRequestHandler(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) throws IOException, SolrServerException {
+ String searchWeightString = generateSearchWeightString(graph.getManagementSystem(), collectionName, attributeName2SearchWeightMap);
+ String payLoadString = generatePayLoadForFreeText("update-requesthandler", FREETEXT_REQUEST_HANDLER, searchWeightString);
+
+ performRequestHandlerAction(collectionName, solrClient, payLoadString);
+ }
+
+ private void createFreeTextRequestHandler(String collectionName, Map<String, Integer> attributeName2SearchWeightMap) throws IOException, SolrServerException {
+ String searchWeightString = generateSearchWeightString(graph.getManagementSystem(), collectionName, attributeName2SearchWeightMap);
+ String payLoadString = generatePayLoadForFreeText("create-requesthandler", FREETEXT_REQUEST_HANDLER, searchWeightString);
+
+ performRequestHandlerAction(collectionName, solrClient, payLoadString);
+ }
+
+ @VisibleForTesting
+ static String generatePayLoadForFreeText(String action, String handlerName, String qfValue) {
+ return String.format("{" +
+ " %s : { " +
+ " 'name' : '%s', " +
+ " 'class': 'solr.SearchHandler' , " +
+ " 'defaults': " + "{" +
+ " 'defType': 'edismax' , " +
+ " 'rows': 100 , " +
+ " 'lowercaseOperators': true , " +
+ " 'qf': '%s' , " +
+ " 'hl.fl': '*' , " +
+ " 'hl.requireFieldMatch': true , " +
+ " 'lowercaseOperators': true , " +
+ " }" +
+ " }" +
+ "}", action, handlerName, qfValue);
+ }
+
+ private void performRequestHandlerAction(String collectionName, SolrClient solrClient,
+ String actionPayLoad) throws IOException, SolrServerException {
+ V2Request v2Request = new V2Request.Builder(String.format("/collections/%s/config", collectionName))
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(actionPayLoad)
+ .build();
+ v2Request.process(solrClient);
+ }
}
diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
index 7b5b675..8c1dbd7 100644
--- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
+++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
@@ -261,7 +261,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
public static class AtlasAttributeDef implements Serializable {
private static final long serialVersionUID = 1L;
public static final int DEFAULT_SEARCHWEIGHT = -1;
- public static final int DEFAULT_SEARCHWEIGHT_FOR_STRINGS = 3;
+
public static final String SEARCH_WEIGHT_ATTR_NAME = "searchWeight";
public static final String ATTRDEF_OPTION_SOFT_REFERENCE = "isSoftReference";
private final String STRING_TRUE = "true";
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index dadc3c5..56655a8 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -23,7 +23,6 @@ import com.google.common.base.Preconditions;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
-import org.apache.atlas.discovery.SearchContext;
import org.apache.atlas.discovery.SearchIndexer;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration;
@@ -59,8 +58,6 @@ import java.util.List;
import java.util.Set;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
-import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
-import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET;
@@ -92,15 +89,27 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
// Added for type lookup when indexing the new typedefs
private final AtlasTypeRegistry typeRegistry;
-
- private final AtlasGraphIndexClient atlasGraphIndexClient;
+ private final List<IndexChangeListener> indexChangeListeners = new ArrayList<>();
//allows injection of a dummy graph for testing
private IAtlasGraphProvider provider;
private boolean recomputeIndexedKeys = true;
private Set<String> vertexIndexKeys = new HashSet<>();
- private final String[] mappedCopyFieldNames;
+
+ public static boolean isValidSearchWeight(int searchWeight) {
+ if (searchWeight != -1 ) {
+ if (searchWeight < 1 || searchWeight > 10) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static boolean isStringAttribute(AtlasStructDef.AtlasAttributeDef attributeDef) {
+ return AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeDef.getTypeName());
+ }
+
public enum UniqueKind { NONE, GLOBAL_UNIQUE, PER_TYPE_UNIQUE }
@Inject
@@ -113,23 +122,19 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
throws IndexException, RepositoryException {
this.provider = provider;
this.typeRegistry = typeRegistry;
-
- mappedCopyFieldNames = new String[11];
- for(int i=0; i < mappedCopyFieldNames.length; i++) {
- mappedCopyFieldNames[i] = String.format("allt%ds", i);
- }
-
- try {
- atlasGraphIndexClient = provider.get().getGraphIndexClient();
- } catch (Exception e) {
- LOG.error("Error encountered in creating solr client.", e);
- throw new RepositoryException("Error encountered in creating solr client.", e);
- }
+ //make sure solr index follows graph backed index listener
+ addIndexListener(new SolrIndexHelper(typeRegistry));
if (!HAConfiguration.isHAEnabled(configuration)) {
initialize(provider.get());
}
+
+ }
+
+ public void addIndexListener(IndexChangeListener listener) {
+ indexChangeListeners.add(listener);
}
+
/**
* Initialize global indices for JanusGraph on server activation.
*
@@ -191,7 +196,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.error("Failed to update indexes for changed typedefs", e);
attemptRollback(changedTypeDefs, management);
}
-
+ notifyChangeListeners();
}
public Set<String> getVertexIndexKeys() {
@@ -283,7 +288,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex(management, MODIFICATION_TIMESTAMP_PROPERTY_KEY, UniqueKind.NONE, Long.class, SINGLE, false, false);
createVertexIndex(management, STATE_PROPERTY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, CREATED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
- createVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false, 10);
+ createVertexIndex(management, CLASSIFICATION_TEXT_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, MODIFIED_BY_KEY, UniqueKind.NONE, String.class, SINGLE, false, false);
createVertexIndex(management, TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, SET, true, true);
createVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, UniqueKind.NONE, String.class, LIST, true, true);
@@ -356,12 +361,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.info("Completed deleting indexes for type {}", typeDef.getName());
}
- private static boolean isStringAttribute(AtlasStructDef.AtlasAttributeDef attributeDef) {
- return AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeDef.getTypeName());
- }
-
private void createIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) {
- final String propertyName = AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + attributeDef.getName());
+ final String propertyName = getEncodedPropertyName(typeName, attributeDef);
AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality());
boolean isUnique = attributeDef.getIsUnique();
boolean isIndexable = attributeDef.getIsIndexable();
@@ -370,22 +371,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
boolean isArrayType = isArrayType(attribTypeName);
boolean isMapType = isMapType(attribTypeName);
final String uniqPropName = isUnique ? AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + attributeDef.getName()) : null;
- int searchWeight = attributeDef.getSearchWeight();
-
- if(attributeDef.getSearchWeight() == DEFAULT_SEARCHWEIGHT) {
- if (isStringAttribute(attributeDef)) {
- //We will use default search weight of 3 for string attributes.
- //this will make the string data searchable like in FullTextIndex Searcher using Free Text searcher.
- LOG.info("Applying default search weight of {} for attribute{}.", DEFAULT_SEARCHWEIGHT_FOR_STRINGS, propertyName);
- searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
- }
- }
-
- if(!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) {
- String msg = String.format("Invalid search weight '%d' was provided for property %s.", searchWeight, propertyName);
- LOG.error(msg);
- throw new RuntimeException(msg);
- }
try {
AtlasType atlasType = typeRegistry.getType(typeName);
@@ -424,7 +409,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (isRelationshipType(atlasType)) {
createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false);
} else {
- createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, false, searchWeight);
+ createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, false);
if (uniqPropName != null) {
createVertexIndex(management, uniqPropName, UniqueKind.PER_TYPE_UNIQUE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, true);
@@ -463,6 +448,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
+ /**
+ * gets the encoded property name for the attribute passed in.
+ * @param typeName the type system of the attribute
+ * @param attributeDef the attribute definition
+ * @return the encoded property name for the attribute passed in.
+ */
+ public static String getEncodedPropertyName(String typeName, AtlasAttributeDef attributeDef) {
+ return AtlasGraphUtilsV2.encodePropertyKey(typeName + "." + attributeDef.getName());
+ }
+
private void createLabelIfNeeded(final AtlasGraphManagement management, final String propertyName, final String attribTypeName) {
// If any of the referenced typename is of type Entity or Struct then the edge label needs to be created
for (String typeName : AtlasTypeUtil.getReferencedTypeNames(attribTypeName)) {
@@ -564,10 +559,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
public void createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class propertyClass,
AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes) {
- createVertexIndex(management, propertyName, uniqueKind, propertyClass, cardinality, createCompositeIndex, createCompositeIndexWithTypeAndSuperTypes, -1);
- }
- private void createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class propertyClass,
- AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes, int searchWeight) {
if (propertyName != null) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
@@ -585,16 +576,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
- //for now, we will support free text based enhancements for SOLR backed index systems only.
- //WARNING--it is suggested that we don't change the search weight once is assigned with a number in the range 1 to 10.
- //otherwise, we might end up mapping the index to multiple copy fields...unwanted to space usage.
- if( searchWeight != -1 && SearchContext.isIndexSolrBased()) {
- String encodedPropertyName = management.getIndexFieldName(VERTEX_INDEX, propertyKey);
- String mappedCopyFieldName = mappedCopyFieldNames[searchWeight];
- atlasGraphIndexClient.createCopyField(VERTEX_INDEX, encodedPropertyName, mappedCopyFieldName);
- LOG.info("Created copy field from {} to {} for collection {} for property {}.", encodedPropertyName, mappedCopyFieldName, VERTEX_INDEX, propertyName);
- }
-
if (propertyKey != null) {
if (createCompositeIndex || uniqueKind == UniqueKind.GLOBAL_UNIQUE || uniqueKind == UniqueKind.PER_TYPE_UNIQUE) {
createVertexCompositeIndex(management, propertyClass, propertyKey, uniqueKind == UniqueKind.GLOBAL_UNIQUE);
@@ -620,7 +601,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (LOG.isDebugEnabled()) {
LOG.debug("Creating vertex-centric index for edge label: {} direction: {} for property: {} of type: {} ",
- edgeLabel, edgeDirection.name(), propertyName, propertyClass.getName());
+ edgeLabel, edgeDirection.name(), propertyName, propertyClass.getName());
}
final String indexName = edgeLabel + propertyKey.getName();
@@ -824,12 +805,15 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
LOG.info("Index creation for type {} complete", typeDef.getName());
}
- public static boolean isValidSearchWeight(int searchWeight) {
- if(searchWeight != -1 ) {
- if(searchWeight < 1 || searchWeight > 10) {
- return false;
+ private void notifyChangeListeners() {
+ for (IndexChangeListener indexChangeListener : indexChangeListeners) {
+ try {
+ indexChangeListener.onChange();
+ } catch (Throwable t) {
+ LOG.error("Error encountered in notifying the index change listener {}.", indexChangeListener.getClass().getName(), t);
+ //we need to throw exception if any of the listeners throw execption.
+ throw new RuntimeException("Error encountered in notifying the index change listener " + indexChangeListener.getClass().getName(), t);
}
}
- return true;
}
}
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java b/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
similarity index 52%
copy from graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
copy to repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
index 482e6f6..9cde947 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/IndexChangeListener.java
@@ -6,26 +6,17 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.atlas.repository.graphdb;
+package org.apache.atlas.repository.graph;
-/**
- * Represents a graph client work with indices used by Jansgraph.
- */
-public interface AtlasGraphIndexClient {
- /**
- * The implementers should create a mapping from source propertyName to mapping field name.
- * @param indexName the name of index that needs to be worked with.
- * @param sourcePropertyName the name of the source attribute.
- * @param targetPropertyName the name of the target attribute to which the mapping is getting done.
- */
- void createCopyField(String indexName, String sourcePropertyName, String targetPropertyName);
+public interface IndexChangeListener {
+ void onChange();
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
new file mode 100644
index 0000000..20a517f
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.discovery.SearchContext;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_TEXT_KEY;
+
+/**
+ This is a component that will go through all entity type definitions and create free text index
+ request handler with SOLR. This is a no op class in non-solr index based deployments.
+ This component needs to be initialized after type definitions are completely fixed with the needed patches (Ordder 3 initialization).
+ */
+public class SolrIndexHelper implements IndexChangeListener {
+ private static final Logger LOG = LoggerFactory.getLogger(SolrIndexHelper.class);
+
+ public static final int DEFAULT_SEARCHWEIGHT_FOR_STRINGS = 3;
+
+ private final AtlasTypeRegistry typeRegistry;
+
+
+ public SolrIndexHelper(AtlasTypeRegistry typeRegistry) {
+ this.typeRegistry = typeRegistry;
+ }
+
+ @Override
+ public void onChange() {
+ LOG.info("SolrIndexHelper.onChange()");
+
+ if(!SearchContext.isIndexSolrBased()) {
+ LOG.warn("Not a Solr based index store. Free text search is not supported");
+
+ return;
+ }
+
+ try {
+ AtlasGraph atlasGraph = AtlasGraphProvider.getGraphInstance();
+ AtlasGraphIndexClient atlasGraphIndexClient = atlasGraph.getGraphIndexClient();
+ Map<String, Integer> attributeName2SearchWeightMap = getAttributesWithSearchWeights();
+
+ atlasGraphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, attributeName2SearchWeightMap);
+ } catch (AtlasException e) {
+ LOG.error("Error encountered in handling type system change notification.", e);
+ throw new RuntimeException("Error encountered in handling type system change notification.", e);
+ }
+ }
+
+ private Map<String, Integer> getAttributesWithSearchWeights() {
+ Map<String, Integer> attributesWithSearchWeights = new HashMap<>();
+ Collection<AtlasEntityDef> allEntityDefs = typeRegistry.getAllEntityDefs();
+
+ attributesWithSearchWeights.put(CLASSIFICATION_TEXT_KEY,10);
+
+ if (CollectionUtils.isNotEmpty(allEntityDefs)) {
+ for (AtlasEntityDef entityDef : allEntityDefs) {
+ processEntity(attributesWithSearchWeights, entityDef);
+ }
+ }
+
+ return attributesWithSearchWeights;
+ }
+
+ private void processEntity(Map<String, Integer> attributesWithSearchWeights, AtlasEntityDef entityDef) {
+ for (AtlasAttributeDef attributeDef : entityDef.getAttributeDefs()) {
+ processAttributeDefinition(attributesWithSearchWeights, entityDef, attributeDef);
+ }
+ }
+
+ private void processAttributeDefinition(Map<String, Integer> attributesWithSearchWeights, AtlasEntityDef entityDef, AtlasAttributeDef attributeDef) {
+ if (GraphBackedSearchIndexer.isStringAttribute(attributeDef)) {
+ final String attributeName = GraphBackedSearchIndexer.getEncodedPropertyName(entityDef.getName(), attributeDef);
+ int searchWeight = attributeDef.getSearchWeight();
+
+ if (searchWeight == DEFAULT_SEARCHWEIGHT) {
+ //We will use default search weight of 3 for string attributes.
+ //this will make the string data searchable like in FullTextIndex Searcher using Free Text searcher.
+ searchWeight = DEFAULT_SEARCHWEIGHT_FOR_STRINGS;
+ } else if (!GraphBackedSearchIndexer.isValidSearchWeight(searchWeight)) { //validate the value provided in the model.
+ String msg = String.format("Invalid search weight '%d' for attribute %s.%s", searchWeight, entityDef.getName(), attributeName);
+
+ LOG.error(msg);
+
+ throw new RuntimeException(msg);
+ }
+
+ LOG.info("Applying search weight {} for attribute {}.{}", searchWeight, entityDef.getName(), attributeName);
+
+ attributesWithSearchWeights.put(attributeName, searchWeight);
+ }
+ }
+}
\ No newline at end of file