You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2020/05/25 22:31:06 UTC
[atlas] branch master updated: ATLAS-3654: enable support for solr
in standalone mode
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 6248e36 ATLAS-3654: enable support for solr in standalone mode
6248e36 is described below
commit 6248e361fc01256ba263a6f4523c50541f6fc86b
Author: Damian Warszawski <da...@ing.com>
AuthorDate: Tue May 19 23:57:56 2020 +0200
ATLAS-3654: enable support for solr in standalone mode
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
---
distro/src/bin/atlas_config.py | 13 +++-
.../src/documents/Setup/InstallationInstruction.md | 17 ++++-
.../graphdb/janus/AtlasJanusGraphIndexClient.java | 74 +++++++++++++++-------
.../janusgraph/diskstorage/solr/Solr6Index.java | 16 ++++-
4 files changed, 91 insertions(+), 29 deletions(-)
diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py
index f09026f..2d61433 100755
--- a/distro/src/bin/atlas_config.py
+++ b/distro/src/bin/atlas_config.py
@@ -66,7 +66,9 @@ CONF_FILE="atlas-application.properties"
STORAGE_BACKEND_CONF="atlas.graph.storage.backend"
HBASE_STORAGE_LOCAL_CONF_ENTRY="atlas.graph.storage.hostname\s*=\s*localhost"
SOLR_INDEX_CONF_ENTRY="atlas.graph.index.search.backend\s*=\s*solr"
-SOLR_INDEX_LOCAL_CONF_ENTRY="atlas.graph.index.search.solr.zookeeper-url\s*=\s*localhost"
+SOLR_INDEX_MODE_CONF_ENTRY="atlas.graph.index.search.solr.mode"
+SOLR_INDEX_LOCAL_STANDALONE_CONF_ENTRY="atlas.graph.index.search.solr.http-urls\s*=(http|https)://localhost"
+SOLR_INDEX_LOCAL_CLOUD_CONF_ENTRY="atlas.graph.index.search.solr.zookeeper-url\s*=\s*localhost"
SOLR_INDEX_ZK_URL="atlas.graph.index.search.solr.zookeeper-url"
TOPICS_TO_CREATE="atlas.notification.topics"
ATLAS_HTTP_PORT="atlas.server.http.port"
@@ -453,7 +455,14 @@ def is_solr_local(confdir):
return False
confdir = os.path.join(confdir, CONF_FILE)
- return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and grep(confdir, SOLR_INDEX_LOCAL_CONF_ENTRY) is not None
+ return is_solr_local_cloud_mode(confdir) or is_solr_local_standalone_mode(confdir)
+
+def is_solr_local_cloud_mode(confdir):
+ return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and getConfig(confdir, SOLR_INDEX_MODE_CONF_ENTRY) == 'cloud' and grep(confdir, SOLR_INDEX_LOCAL_CLOUD_CONF_ENTRY) is not None
+
+
+def is_solr_local_standalone_mode(confdir):
+ return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and getConfig(confdir, SOLR_INDEX_MODE_CONF_ENTRY) == 'http' and grep(confdir, SOLR_INDEX_LOCAL_STANDALONE_CONF_ENTRY) is not None
def is_elasticsearch_local():
if os.environ.get(MANAGE_LOCAL_ELASTICSEARCH, "False").lower() == 'false':
diff --git a/docs/src/documents/Setup/InstallationInstruction.md b/docs/src/documents/Setup/InstallationInstruction.md
index d1b22d6..d22cb0c 100644
--- a/docs/src/documents/Setup/InstallationInstruction.md
+++ b/docs/src/documents/Setup/InstallationInstruction.md
@@ -150,7 +150,7 @@ By default, Apache Atlas uses JanusGraph as the graph repository and is the only
* Start Apache Solr in cloud mode.
SolrCloud mode uses a ZooKeeper Service as a highly available, central location for cluster management. For a small cluster, running with an existing ZooKeeper quorum should be fine. For larger clusters, you would want to run separate multiple ZooKeeper quorum with at least 3 servers.
- Note: Apache Atlas currently supports Apache Solr in "cloud" mode only. "http" mode is not supported. For more information, refer Apache Solr documentation - https://cwiki.apache.org/confluence/display/solr/SolrCloud
+ For more information, refer Apache Solr documentation - https://cwiki.apache.org/confluence/display/solr/SolrCloud
* For e.g., to bring up an Apache Solr node listening on port 8983 on a machine, you can use the command:
@@ -197,6 +197,21 @@ Pre-requisites for running Apache Solr in cloud mode
* SolrCloud has support for replication and sharding. It is highly recommended to use SolrCloud with at least two Apache Solr nodes running on different servers with replication enabled.
If using SolrCloud, then you also need ZooKeeper installed and configured with 3 or 5 ZooKeeper nodes
+ * Start Apache Solr in http mode - alternative setup to Solr in cloud mode.
+
+ Solr Standalone is used for a single instance, and it keeps configuration information on the file system. It does not require zookeeper and provides high performance for medium size index.
+ Can be consider as a good option for fast prototyping as well as valid configuration for development environments. In some cases it demonstrates a better performance than solr cloud mode in production grade setup of Atlas.
+
+ * Change ATLAS configuration to point to Standalone Apache Solr instance setup. Please make sure the following configurations are set to the below values in ATLAS_HOME/conf/atlas-application.properties
+
+<SyntaxHighlighter wrapLines={true} language="powershell" style={theme.dark}>
+{`atlas.graph.index.search.backend=solr
+atlas.graph.index.search.solr.mode=http
+atlas.graph.index.search.solr.http-urls=<a single or list of URLs for the Solr instances must be provided.> eg: localhost:2181,10.1.6.5:2181`}
+</SyntaxHighlighter>
+
+ Note: Solr standalone can be run in embedded mode using `embedded-hbase-solr` profile.
+
*Configuring Elasticsearch as the indexing backend for the Graph Repository (Tech Preview)*
By default, Apache Atlas uses [JanusGraph](https://janusgraph.org/) as the graph repository and is the only graph repository implementation available currently. For configuring [JanusGraph](https://janusgraph.org/) to work with Elasticsearch, please follow the instructions below
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
index ba65f3d..29bd2a4 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
@@ -32,14 +32,17 @@ import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.request.V2Request;
-import org.apache.solr.client.solrj.response.V2Response;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.TermsResponse;
+import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.NamedList;
import org.janusgraph.diskstorage.solr.Solr6Index;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,6 +82,8 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return;
}
+ Solr6Index.Mode solrMode = Solr6Index.getSolrMode();
+
//1) try updating request handler
//2) if update fails, try creating request handler
@@ -100,7 +105,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
try {
LOG.info("Attempting to update free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
- updateFreeTextRequestHandler(solrClient, collectionName, indexFieldName2SearchWeightMap);
+ updateFreeTextRequestHandler(solrClient, collectionName, indexFieldName2SearchWeightMap, solrMode);
LOG.info("Successfully updated free text request handler {} for collection {}..", FREETEXT_REQUEST_HANDLER, collectionName);
@@ -114,7 +119,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
try {
LOG.info("Attempting to create free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
- createFreeTextRequestHandler(solrClient, collectionName, indexFieldName2SearchWeightMap);
+ createFreeTextRequestHandler(solrClient, collectionName, indexFieldName2SearchWeightMap, solrMode);
LOG.info("Successfully created free text request handler {} for collection {}", FREETEXT_REQUEST_HANDLER, collectionName);
return;
@@ -256,10 +261,13 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return;
}
+ Solr6Index.Mode solrMode = Solr6Index.getSolrMode();
+
//update the request handler
performRequestHandlerAction(collectionName,
solrClient,
- generatePayLoadForSuggestions(generateSuggestionsString(suggestionProperties)));
+ generatePayLoadForSuggestions(generateSuggestionsString(suggestionProperties)),
+ solrMode);
} catch (Throwable t) {
String msg = String.format("Error encountered in creating the request handler '%s' for collection '%s'", Constants.TERMS_REQUEST_HANDLER, collectionName);
@@ -458,35 +466,53 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return ret.toString();
}
- private V2Response updateFreeTextRequestHandler(SolrClient solrClient, String collectionName,
- Map<String, Integer> indexFieldName2SearchWeightMap) throws IOException, SolrServerException, AtlasBaseException {
+ private SolrResponse updateFreeTextRequestHandler(SolrClient solrClient, String collectionName,
+ Map<String, Integer> indexFieldName2SearchWeightMap,
+ Solr6Index.Mode mode) throws IOException, SolrServerException, AtlasBaseException {
String searchWeightString = generateSearchWeightString(indexFieldName2SearchWeightMap);
String payLoadString = generatePayLoadForFreeText("update-requesthandler", searchWeightString);
- return performRequestHandlerAction(collectionName, solrClient, payLoadString);
+ return performRequestHandlerAction(collectionName, solrClient, payLoadString, mode);
}
- private V2Response createFreeTextRequestHandler(SolrClient solrClient, String collectionName,
- Map<String, Integer> indexFieldName2SearchWeightMap) throws IOException, SolrServerException, AtlasBaseException {
+ private SolrResponse createFreeTextRequestHandler(SolrClient solrClient, String collectionName,
+ Map<String, Integer> indexFieldName2SearchWeightMap,
+ Solr6Index.Mode mode) throws IOException, SolrServerException, AtlasBaseException {
String searchWeightString = generateSearchWeightString(indexFieldName2SearchWeightMap);
String payLoadString = generatePayLoadForFreeText("create-requesthandler", searchWeightString);
- return performRequestHandlerAction(collectionName, solrClient, payLoadString);
+ return performRequestHandlerAction(collectionName, solrClient, payLoadString, mode);
}
- private V2Response performRequestHandlerAction(String collectionName,
+ private SolrResponse performRequestHandlerAction(String collectionName,
SolrClient solrClient,
- String actionPayLoad) throws IOException, SolrServerException, AtlasBaseException {
- V2Request v2Request = new V2Request.Builder(String.format("/collections/%s/config", collectionName))
- .withMethod(SolrRequest.METHOD.POST)
- .withPayload(actionPayLoad)
- .build();
+ String actionPayLoad,
+ Solr6Index.Mode mode) throws IOException, SolrServerException, AtlasBaseException {
+ switch (mode) {
+ case CLOUD:
+ V2Request v2request = new V2Request.Builder(String.format("/collections/%s/config", collectionName))
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload(actionPayLoad)
+ .build();
+
+ return validateResponseForSuccess(v2request.process(solrClient));
- return validateResponseForSuccess(v2Request.process(solrClient));
+ case HTTP:
+ Collection<ContentStream> contentStreams = ClientUtils.toContentStreams(actionPayLoad, "application/json; charset=UTF-8");
+ GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.POST, String.format("/%s/config", collectionName), null);
+
+ request.setContentStreams(contentStreams);
+ request.setUseV2(false);
+
+ return validateResponseForSuccess(request.process(solrClient));
+
+ default:
+ throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
+ }
}
- private V2Response validateResponseForSuccess(V2Response v2Response) throws AtlasBaseException {
- if(v2Response == null) {
+ private SolrResponse validateResponseForSuccess(SolrResponse solrResponse) throws AtlasBaseException {
+ if(solrResponse == null) {
String msg = "Received null response .";
LOG.error(msg);
@@ -495,10 +521,10 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("V2 Response is {}", v2Response.toString());
+ LOG.debug("V2 Response is {}", solrResponse.toString());
}
- NamedList<Object> response = v2Response.getResponse();
+ NamedList<Object> response = solrResponse.getResponse();
if(response != null) {
Object errorMessages = response.get("errorMessages");
@@ -523,7 +549,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
throw new AtlasBaseException(msg);
} else {
if(LOG.isDebugEnabled()) {
- LOG.debug("Successfully performed response handler action. V2 Response is {}", v2Response.toString());
+ LOG.debug("Successfully performed response handler action. V2 Response is {}", solrResponse.toString());
}
}
@@ -533,7 +559,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
}
}
- return v2Response;
+ return solrResponse;
}
static final class TermFreq {
diff --git a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
index 484c161..6509fef 100644
--- a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
+++ b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
@@ -143,7 +143,7 @@ public class Solr6Index implements IndexProvider {
private static Solr6Index instance = null;
public static final ConfigOption<Boolean> CREATE_SOLR_CLIENT_PER_REQUEST = new ConfigOption(SOLR_NS, "create-client-per-request", "when false, allows the sharing of solr client across other components.", org.janusgraph.diskstorage.configuration.ConfigOption.Type.LOCAL, false);
- private enum Mode {
+ public enum Mode {
HTTP, CLOUD;
public static Mode parse(String mode) {
@@ -183,7 +183,6 @@ public class Solr6Index implements IndexProvider {
private final boolean waitSearcher;
private final boolean kerberosEnabled;
-
public Solr6Index(final Configuration config) throws BackendException {
// Add Kerberos-enabled SolrHttpClientBuilder
HttpClientUtil.setHttpClientBuilder(new Krb5HttpClientBuilder().getBuilder());
@@ -217,6 +216,19 @@ public class Solr6Index implements IndexProvider {
Solr6Index.instance = this;
}
+ public static Mode getSolrMode() {
+ Solr6Index solr6Index = Solr6Index.instance;
+ Mode ret = (solr6Index != null) ? Mode.parse(solr6Index.configuration.get(SOLR_MODE)) : null;
+
+ if (ret == null) {
+ logger.warn("SolrMode is not set. Assuming {}", Mode.CLOUD);
+
+ ret = Mode.CLOUD;
+ }
+
+ return ret;
+ }
+
public static SolrClient getSolrClient() {
if (Solr6Index.instance != null) {
if (createSolrClientPerRequest) {