You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2021/07/27 22:00:06 UTC
[atlas] branch master updated: ATLAS-4340: Set Solr wait-searcher
property to false by default to make Solr commits async
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new a7ac64d ATLAS-4340: Set Solr wait-searcher property to false by default to make Solr commits async
a7ac64d is described below
commit a7ac64d54954064be6b0f56c04f05bbe128772d5
Author: Sarath Subramanian <sa...@apache.org>
AuthorDate: Tue Jul 27 14:59:48 2021 -0700
ATLAS-4340: Set Solr wait-searcher property to false by default to make Solr commits async
---
.../src/test/resources/atlas-application.properties | 1 +
.../src/test/resources/atlas-application.properties | 1 +
.../src/test/resources/atlas-application.properties | 1 +
.../src/test/resources/atlas-application.properties | 3 +--
distro/pom.xml | 8 ++++----
distro/src/bin/atlas_config.py | 6 +++---
distro/src/bin/atlas_start.py | 18 +++++++++++++-----
distro/src/test/python/scripts/TestMetadata.py | 6 +++---
.../src/test/resources/atlas-application.properties | 1 +
.../java/org/apache/atlas/ApplicationProperties.java | 14 ++++++++------
intg/src/test/resources/atlas-application.properties | 3 +--
webapp/src/test/resources/atlas-application.properties | 2 +-
12 files changed, 38 insertions(+), 26 deletions(-)
diff --git a/addons/falcon-bridge/src/test/resources/atlas-application.properties b/addons/falcon-bridge/src/test/resources/atlas-application.properties
index 898b69c..3b12e5f 100644
--- a/addons/falcon-bridge/src/test/resources/atlas-application.properties
+++ b/addons/falcon-bridge/src/test/resources/atlas-application.properties
@@ -24,6 +24,7 @@ atlas.rest.address=http://localhost:31000
# Graph database implementation. Value inserted by maven.
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
+atlas.graph.index.search.solr.wait-searcher=true
# Graph Storage
atlas.graph.storage.backend=berkeleyje
diff --git a/addons/hbase-bridge/src/test/resources/atlas-application.properties b/addons/hbase-bridge/src/test/resources/atlas-application.properties
index 898b69c..3b12e5f 100644
--- a/addons/hbase-bridge/src/test/resources/atlas-application.properties
+++ b/addons/hbase-bridge/src/test/resources/atlas-application.properties
@@ -24,6 +24,7 @@ atlas.rest.address=http://localhost:31000
# Graph database implementation. Value inserted by maven.
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
+atlas.graph.index.search.solr.wait-searcher=true
# Graph Storage
atlas.graph.storage.backend=berkeleyje
diff --git a/addons/kafka-bridge/src/test/resources/atlas-application.properties b/addons/kafka-bridge/src/test/resources/atlas-application.properties
index 91fd8b0..4a12cf6 100644
--- a/addons/kafka-bridge/src/test/resources/atlas-application.properties
+++ b/addons/kafka-bridge/src/test/resources/atlas-application.properties
@@ -24,6 +24,7 @@ atlas.rest.address=http://localhost:31000
# Graph database implementation. Value inserted by maven.
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
+atlas.graph.index.search.solr.wait-searcher=true
# Graph Storage
atlas.graph.storage.backend=berkeleyje
diff --git a/authorization/src/test/resources/atlas-application.properties b/authorization/src/test/resources/atlas-application.properties
index 2e02678..d735900 100644
--- a/authorization/src/test/resources/atlas-application.properties
+++ b/authorization/src/test/resources/atlas-application.properties
@@ -18,8 +18,7 @@
#system property
atlas.data=${sys:user.dir}/target/data
-
-
+atlas.graph.index.search.solr.wait-searcher=true
#re-use existing property
atlas.graph.data=${atlas.data}/graph
diff --git a/distro/pom.xml b/distro/pom.xml
index d84f5e7..078bb2e 100644
--- a/distro/pom.xml
+++ b/distro/pom.xml
@@ -41,7 +41,7 @@ atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=
atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
atlas.graph.index.search.solr.zookeeper-session-timeout=60000
-atlas.graph.index.search.solr.wait-searcher=true
+atlas.graph.index.search.solr.wait-searcher=false
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
@@ -162,7 +162,7 @@ atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=localhost:2181
atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
atlas.graph.index.search.solr.zookeeper-session-timeout=60000
-atlas.graph.index.search.solr.wait-searcher=true
+atlas.graph.index.search.solr.wait-searcher=false
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
@@ -321,7 +321,7 @@ atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=localhost:2181
atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
atlas.graph.index.search.solr.zookeeper-session-timeout=60000
-atlas.graph.index.search.solr.wait-searcher=true
+atlas.graph.index.search.solr.wait-searcher=false
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
@@ -411,7 +411,7 @@ atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=localhost:2181
atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
atlas.graph.index.search.solr.zookeeper-session-timeout=60000
-atlas.graph.index.search.solr.wait-searcher=true
+atlas.graph.index.search.solr.wait-searcher=false
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py
index 493a34a..31e6fd0 100755
--- a/distro/src/bin/atlas_config.py
+++ b/distro/src/bin/atlas_config.py
@@ -125,7 +125,7 @@ def solrHomeDir(dir):
return os.environ.get(SOLR_HOME, os.path.join(dir, "data", "solr"))
def solrConfDir(dir):
- return os.environ.get(SOLR_CONF, os.path.join(dir, "solr", CONFIG_SETS_CONF))
+ return os.environ.get(SOLR_CONF, os.path.join(dir, "conf", "solr"))
def solrPort():
return os.environ.get(SOLR_PORT, DEFAULT_SOLR_PORT)
@@ -516,7 +516,7 @@ def get_atlas_url_port(confdir):
else:
port = getConfigWithDefault(confdir, ATLAS_HTTP_PORT, DEFAULT_ATLAS_HTTP_PORT)
- print("starting atlas on port %s" % port)
+ print("Starting Atlas server on port: %s" % port)
return port
def get_atlas_url_host(confdir):
@@ -524,7 +524,7 @@ def get_atlas_url_host(confdir):
host = getConfigWithDefault(confdir, ATLAS_SERVER_BIND_ADDRESS, DEFAULT_ATLAS_SERVER_HOST)
if (host == '0.0.0.0'):
host = DEFAULT_ATLAS_SERVER_HOST
- print("starting atlas on host %s" % host)
+ print("\nStarting Atlas server on host: %s" % host)
return host
def wait_for_startup(confdir, wait):
diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py
index 7cf35a9..ffbe162 100755
--- a/distro/src/bin/atlas_start.py
+++ b/distro/src/bin/atlas_start.py
@@ -110,14 +110,17 @@ def main():
mc.server_pid_not_running(pid)
if is_hbase and mc.is_hbase_local(confdir):
- print("configured for local hbase.")
+ print("\nConfigured for local HBase.")
mc.configure_hbase(atlas_home)
+
+ print("Starting local HBase...")
mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir)
- print("hbase started.")
+
+ print("Local HBase started!")
#solr setup
if mc.is_solr_local(confdir):
- print("configured for local solr.")
+ print("\nConfigured for local Solr.")
if mc.is_cassandra_local(confdir):
print("Cassandra embedded configured.")
@@ -128,12 +131,17 @@ def main():
mc.run_zookeeper(mc.zookeeperBinDir(atlas_home), "start", logdir)
print("zookeeper started.")
+ print("Starting local Solr...")
mc.run_solr(mc.solrBinDir(atlas_home), "start", mc.get_solr_zk_url(confdir), mc.solrPort(), logdir, True, mc.solrHomeDir(atlas_home))
- print("solr started.")
- print("setting up solr collections...")
+ print("Local Solr started!")
+
+ print("\nCreating Solr collections for Atlas using config: " + mc.solrConfDir(atlas_home))
+
mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "vertex_index", logdir)
+
mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "edge_index", logdir)
+
mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "fulltext_index", logdir)
#elasticsearch setup
diff --git a/distro/src/test/python/scripts/TestMetadata.py b/distro/src/test/python/scripts/TestMetadata.py
index 662fbdd..b813f92 100644
--- a/distro/src/test/python/scripts/TestMetadata.py
+++ b/distro/src/test/python/scripts/TestMetadata.py
@@ -109,13 +109,13 @@ class TestMetadata(unittest.TestCase):
call(['atlas_home/solr/bin/solr', 'start', '-z', 'localhost:9838', '-p', '9838', '-s', 'atlas_home/data/solr'], 'atlas_home/logs',
False, True),
call(['atlas_home/solr/bin/solr', 'create', '-c', 'vertex_index', '-d',
- 'atlas_home/solr/server/solr/configsets/_default/conf', '-shards', '1', '-replicationFactor',
+ 'atlas_home/conf/solr', '-shards', '1', '-replicationFactor',
'1'], 'atlas_home/logs', False, True),
call(['atlas_home/solr/bin/solr', 'create', '-c', 'edge_index', '-d',
- 'atlas_home/solr/server/solr/configsets/_default/conf', '-shards', '1', '-replicationFactor',
+ 'atlas_home/conf/solr', '-shards', '1', '-replicationFactor',
'1'], 'atlas_home/logs', False, True),
call(['atlas_home/solr/bin/solr', 'create', '-c', 'fulltext_index', '-d',
- 'atlas_home/solr/server/solr/configsets/_default/conf', '-shards', '1', '-replicationFactor',
+ 'atlas_home/conf/solr', '-shards', '1', '-replicationFactor',
'1'], 'atlas_home/logs', False, True)]
runProcess_mock.assert_has_calls(calls)
diff --git a/graphdb/janus/src/test/resources/atlas-application.properties b/graphdb/janus/src/test/resources/atlas-application.properties
index a355234..5510a2c 100644
--- a/graphdb/janus/src/test/resources/atlas-application.properties
+++ b/graphdb/janus/src/test/resources/atlas-application.properties
@@ -18,6 +18,7 @@
######### Graph Database to Use #########
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
+atlas.graph.index.search.solr.wait-searcher=true
######### Atlas Server Configs #########
atlas.rest.address=http://localhost:31000
diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
index bf97ab1..682206d 100644
--- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
+++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
@@ -64,7 +64,7 @@ public final class ApplicationProperties extends PropertiesConfiguration {
public static final String LDAP_BIND_PASSWORD = "atlas.authentication.method.ldap.bind.password";
public static final String MASK_LDAP_PASSWORD = "********";
public static final String DEFAULT_GRAPHDB_BACKEND = GRAPHBD_BACKEND_JANUS;
- public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = true;
+ public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = false;
public static final boolean DEFAULT_INDEX_MAP_NAME = false;
public static final AtlasRunMode DEFAULT_ATLAS_RUN_MODE = AtlasRunMode.PROD;
public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size";
@@ -342,11 +342,12 @@ public final class ApplicationProperties extends PropertiesConfiguration {
if (indexBackend.equalsIgnoreCase(INDEX_BACKEND_SOLR)) {
LOG.info("Atlas is running in MODE: {}.", runMode.name());
- if(runMode == AtlasRunMode.PROD) {
- //we do not want these configurations to be over written in Production mode.
- clearPropertyDirect(SOLR_WAIT_SEARCHER_CONF);
- addPropertyDirect(SOLR_WAIT_SEARCHER_CONF, DEFAULT_SOLR_WAIT_SEARCHER);
- LOG.info("Setting solr-wait-searcher property '" + DEFAULT_SOLR_WAIT_SEARCHER + "'");
+ if (runMode == AtlasRunMode.PROD) {
+ if (!containsKey(SOLR_WAIT_SEARCHER_CONF)) {
+ addPropertyDirect(SOLR_WAIT_SEARCHER_CONF, DEFAULT_SOLR_WAIT_SEARCHER);
+ }
+
+ LOG.info("Setting solr.wait-searcher property '" + getBoolean(SOLR_WAIT_SEARCHER_CONF) + "'");
clearPropertyDirect(INDEX_MAP_NAME_CONF);
addPropertyDirect(INDEX_MAP_NAME_CONF, DEFAULT_INDEX_MAP_NAME);
@@ -361,6 +362,7 @@ public final class ApplicationProperties extends PropertiesConfiguration {
addPropertyDirect(INDEX_SEARCH_MAX_RESULT_SET_SIZE, indexMaxResultSetSize);
LOG.info("Setting " + INDEX_SEARCH_MAX_RESULT_SET_SIZE + " = " + indexMaxResultSetSize);
+ LOG.info("Setting " + SOLR_WAIT_SEARCHER_CONF + " = " + getBoolean(SOLR_WAIT_SEARCHER_CONF));
setDbCacheConfDefaults();
}
diff --git a/intg/src/test/resources/atlas-application.properties b/intg/src/test/resources/atlas-application.properties
index 50ce01e..2139cec 100644
--- a/intg/src/test/resources/atlas-application.properties
+++ b/intg/src/test/resources/atlas-application.properties
@@ -18,8 +18,7 @@
#system property
atlas.data=${sys:user.dir}/target/data
-
-
+atlas.graph.index.search.solr.wait-searcher=true
#re-use existing property
atlas.graph.data=${atlas.data}/graph
diff --git a/webapp/src/test/resources/atlas-application.properties b/webapp/src/test/resources/atlas-application.properties
index 1d45e78..2f43329 100644
--- a/webapp/src/test/resources/atlas-application.properties
+++ b/webapp/src/test/resources/atlas-application.properties
@@ -20,7 +20,7 @@
atlas.rest.address=http://localhost:31000
######### Graph Database Configs #########
-
+atlas.graph.index.search.solr.wait-searcher=true
# Graph database implementation. Value inserted by maven.
atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase