You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by su...@apache.org on 2016/05/13 05:56:36 UTC
incubator-atlas git commit: ATLAS-663,
ATLAS-673 Install Setup: SOLR (tbeerbower via sumasai)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 98769871e -> af63bb887
ATLAS-663,ATLAS-673 Install Setup: SOLR (tbeerbower via sumasai)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/af63bb88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/af63bb88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/af63bb88
Branch: refs/heads/master
Commit: af63bb887e817a64d55f75a37e815cecc4fe0c26
Parents: 9876987
Author: Suma Shivaprasad <su...@gmail.com>
Authored: Thu May 12 22:56:30 2016 -0700
Committer: Suma Shivaprasad <su...@gmail.com>
Committed: Thu May 12 22:56:30 2016 -0700
----------------------------------------------------------------------
.gitignore | 4 +
distro/pom.xml | 38 ++++++++-
distro/src/bin/atlas_config.py | 90 +++++++++++++++++++-
distro/src/bin/atlas_start.py | 16 +++-
distro/src/bin/atlas_stop.py | 4 +
distro/src/conf/atlas-application.properties | 16 ++--
distro/src/conf/hbase/hbase-site.xml.template | 4 +-
.../src/main/assemblies/standalone-package.xml | 7 ++
distro/src/test/python/scripts/TestMetadata.py | 23 ++++-
docs/src/site/twiki/Configuration.twiki | 11 ++-
docs/src/site/twiki/InstallationSteps.twiki | 18 +++-
release-log.txt | 1 +
12 files changed, 210 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 20db282..f5899e2 100755
--- a/.gitignore
+++ b/.gitignore
@@ -56,6 +56,10 @@ test-output
#hbase package downloaded
distro/hbase/*.tar.gz
+
+#solr package downloaded
+distro/solr/*.tgz
+
.cache-main
# emacs files
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/pom.xml
----------------------------------------------------------------------
diff --git a/distro/pom.xml b/distro/pom.xml
index ee9add0..1ab092d 100644
--- a/distro/pom.xml
+++ b/distro/pom.xml
@@ -31,8 +31,12 @@
<properties>
<hbase.dir>${project.build.directory}/hbase</hbase.dir>
- <hbase.tar>http://apache.mirrors.pair.com/hbase/stable/hbase-1.1.4-bin.tar.gz</hbase.tar>
- <hbase.folder>hbase-1.1.4</hbase.folder>
+ <hbase.tar>http://archive.apache.org/dist/hbase/${hbase.version}/hbase-${hbase.version}-bin.tar.gz</hbase.tar>
+ <hbase.folder>hbase-${hbase.version}</hbase.folder>
+
+ <solr.dir>${project.build.directory}/solr</solr.dir>
+ <solr.tar>http://archive.apache.org/dist/lucene/solr/${solr.version}/solr-${solr.version}.tgz</solr.tar>
+ <solr.folder>solr-${solr.version}</solr.folder>
</properties>
<profiles>
@@ -77,6 +81,7 @@
<version>1.7</version>
<executions>
<execution>
+ <id>hbase</id>
<phase>generate-resources</phase>
<goals>
<goal>run</goal>
@@ -104,6 +109,35 @@
</target>
</configuration>
</execution>
+ <execution>
+ <id>solr</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target name="Download SOLR">
+ <mkdir dir="${solr.dir}"/>
+ <mkdir dir="${project.basedir}/solr"/>
+ <get
+ src="${solr.tar}"
+ dest="${project.basedir}/solr/${solr.folder}.tgz"
+ usetimestamp="true"
+ verbose="true" skipexisting="true"
+ />
+ <untar
+ src="${project.basedir}/solr/${solr.folder}.tgz"
+ dest="${project.build.directory}/solr.temp"
+ compression="gzip"
+ />
+ <copy todir="${solr.dir}">
+ <fileset dir="${project.build.directory}/solr.temp/${solr.folder}">
+ <include name="**/*"/>
+ </fileset>
+ </copy>
+ </target>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/bin/atlas_config.py
----------------------------------------------------------------------
diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py
index dbe1191..8f3479c 100755
--- a/distro/src/bin/atlas_config.py
+++ b/distro/src/bin/atlas_config.py
@@ -31,6 +31,7 @@ LIB = "lib"
CONF = "conf"
LOG = "logs"
WEBAPP = "server" + os.sep + "webapp"
+CONFIG_SETS_CONF = "server" + os.sep + "solr" + os.sep + "configsets" + os.sep + "basic_configs" + os.sep + "conf"
DATA = "data"
ATLAS_CONF = "ATLAS_CONF"
ATLAS_LOG = "ATLAS_LOG_DIR"
@@ -42,13 +43,28 @@ ATLAS_SERVER_HEAP = "ATLAS_SERVER_HEAP"
ATLAS_DATA = "ATLAS_DATA_DIR"
ATLAS_HOME = "ATLAS_HOME_DIR"
HBASE_CONF_DIR = "HBASE_CONF_DIR"
+MANAGE_LOCAL_HBASE = "MANAGE_LOCAL_HBASE"
+MANAGE_LOCAL_SOLR = "MANAGE_LOCAL_SOLR"
+SOLR_BIN = "SOLR_BIN"
+SOLR_CONF = "SOLR_CONF"
+SOLR_PORT = "SOLR_PORT"
+DEFAULT_SOLR_PORT = "9838"
+SOLR_SHARDS = "SOLR_SHARDS"
+DEFAULT_SOLR_SHARDS = "1"
+SOLR_REPLICATION_FACTOR = "SOLR_REPLICATION_FACTOR"
+DEFAULT_SOLR_REPLICATION_FACTOR = "1"
+
ENV_KEYS = ["JAVA_HOME", ATLAS_OPTS, ATLAS_SERVER_OPTS, ATLAS_SERVER_HEAP, ATLAS_LOG, ATLAS_PID, ATLAS_CONF,
- "ATLASCPPATH", ATLAS_DATA, ATLAS_HOME, ATLAS_WEBAPP, HBASE_CONF_DIR]
+ "ATLASCPPATH", ATLAS_DATA, ATLAS_HOME, ATLAS_WEBAPP, HBASE_CONF_DIR, SOLR_PORT]
IS_WINDOWS = platform.system() == "Windows"
ON_POSIX = 'posix' in sys.builtin_module_names
CONF_FILE="atlas-application.properties"
HBASE_STORAGE_CONF_ENTRY="atlas.graph.storage.backend\s*=\s*hbase"
HBASE_STORAGE_LOCAL_CONF_ENTRY="atlas.graph.storage.hostname\s*=\s*localhost"
+SOLR_INDEX_CONF_ENTRY="atlas.graph.index.search.backend\s*=\s*solr5"
+SOLR_INDEX_LOCAL_CONF_ENTRY="atlas.graph.index.search.solr.zookeeper-url\s*=\s*localhost"
+SOLR_INDEX_ZK_URL="atlas.graph.index.search.solr.zookeeper-url"
+
DEBUG = False
def scriptDir():
@@ -74,6 +90,21 @@ def hbaseBinDir(dir):
def hbaseConfDir(dir):
return os.environ.get(HBASE_CONF_DIR, os.path.join(dir, "hbase", CONF))
+def solrBinDir(dir):
+ return os.environ.get(SOLR_BIN, os.path.join(dir, "solr", BIN))
+
+def solrConfDir(dir):
+ return os.environ.get(SOLR_CONF, os.path.join(dir, "solr", CONFIG_SETS_CONF))
+
+def solrPort():
+ return os.environ.get(SOLR_PORT, DEFAULT_SOLR_PORT)
+
+def solrShards():
+ return os.environ.get(SOLR_SHARDS, DEFAULT_SOLR_SHARDS)
+
+def solrReplicationFactor():
+ return os.environ.get(SOLR_REPLICATION_FACTOR, DEFAULT_SOLR_REPLICATION_FACTOR)
+
def logDir(dir):
localLog = os.path.join(dir, LOG)
return os.environ.get(ATLAS_LOG, localLog)
@@ -357,6 +388,9 @@ def is_hbase(confdir):
return grep(confdir, HBASE_STORAGE_CONF_ENTRY) is not None
def is_hbase_local(confdir):
+ if os.environ.get(MANAGE_LOCAL_HBASE, "True").lower() == 'false':
+ return False
+
confdir = os.path.join(confdir, CONF_FILE)
return grep(confdir, HBASE_STORAGE_CONF_ENTRY) is not None and grep(confdir, HBASE_STORAGE_LOCAL_CONF_ENTRY) is not None
@@ -380,10 +414,56 @@ def run_hbase_action(dir, action, hbase_conf_dir = None, logdir = None, wait=Tru
return runProcess(cmd, logdir, False, wait)
+def is_solr(confdir):
+ confdir = os.path.join(confdir, CONF_FILE)
+ return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None
+
+def is_solr_local(confdir):
+ if os.environ.get(MANAGE_LOCAL_SOLR, "True").lower() == 'false':
+ return False
+
+ confdir = os.path.join(confdir, CONF_FILE)
+ return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None and grep(confdir, SOLR_INDEX_LOCAL_CONF_ENTRY) is not None
+
+def get_solr_zk_url(confdir):
+ confdir = os.path.join(confdir, CONF_FILE)
+ return getConfig(confdir, SOLR_INDEX_ZK_URL)
+
+def run_solr(dir, action, zk_url = None, port = None, logdir = None, wait=True):
+
+ solrScript = "solr"
+
+ if IS_WINDOWS:
+ solrScript = "solr.cmd"
+
+ if zk_url is None:
+ if port is None:
+ cmd = [os.path.join(dir, solrScript), action]
+ else:
+ cmd = [os.path.join(dir, solrScript), action, '-p', str(port)]
+ else:
+ if port is None:
+ cmd = [os.path.join(dir, solrScript), action, '-z', zk_url]
+ else:
+ cmd = [os.path.join(dir, solrScript), action, '-z', zk_url, '-p', port]
+
+ return runProcess(cmd, logdir, False, wait)
+
+def create_solr_collection(dir, confdir, index, logdir = None, wait=True):
+ solrScript = "solr"
+
+ if IS_WINDOWS:
+ solrScript = "solr.cmd"
+
+ cmd = [os.path.join(dir, solrScript), 'create', '-c', index, '-d', confdir, '-shards', solrShards(), '-replicationFactor', solrReplicationFactor()]
+
+ return runProcess(cmd, logdir, False, wait)
+
def configure_hbase(dir):
env_conf_dir = os.environ.get(HBASE_CONF_DIR)
conf_dir = os.path.join(dir, "hbase", CONF)
tmpl_dir = os.path.join(dir, CONF, "hbase")
+ data_dir = dataDir(atlasDir())
if env_conf_dir is None or env_conf_dir == conf_dir:
hbase_conf_file = "hbase-site.xml"
@@ -403,6 +483,7 @@ def configure_hbase(dir):
f.close()
config = template.replace("${hbase_home}", dir)
+ config = config.replace("${atlas_data}", data_dir)
config = config.replace("${url_prefix}", url_prefix)
f = open(conf_file,'w')
@@ -423,6 +504,13 @@ def grep(file, value):
return line
return None
+def getConfig(file, key):
+ key = key + "\s*="
+ for line in open(file).readlines():
+ if re.match(key, line):
+ return line.split('=')[1].strip()
+ return None
+
def isCygwin():
return platform.system().startswith("CYGWIN")
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/bin/atlas_start.py
----------------------------------------------------------------------
diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py
index 73b8920..cb869bb 100755
--- a/distro/src/bin/atlas_start.py
+++ b/distro/src/bin/atlas_start.py
@@ -35,6 +35,7 @@ def main():
confdir = mc.dirMustExist(mc.confDir(atlas_home))
mc.executeEnvSh(confdir)
logdir = mc.dirMustExist(mc.logDir(atlas_home))
+ mc.dirMustExist(mc.dataDir(atlas_home))
if mc.isCygwin():
# Pathnames that are passed to JVM must be converted to Windows format.
jvm_atlas_home = mc.convertCygwinPath(atlas_home)
@@ -86,7 +87,7 @@ def main():
else:
if mc.is_hbase(confdir):
raise Exception("Could not find hbase-site.xml in %s. Please set env var HBASE_CONF_DIR to the hbase client conf dir", hbase_conf_dir)
-
+
if mc.isCygwin():
atlas_classpath = mc.convertCygwinPath(atlas_classpath, True)
@@ -111,6 +112,19 @@ def main():
mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir)
print "hbase started."
+ #solr setup
+ if mc.is_solr_local(confdir):
+ print "configured for local solr."
+ mc.run_solr(mc.solrBinDir(atlas_home), "start", mc.get_solr_zk_url(confdir), mc.solrPort(), logdir)
+ print "solr started."
+
+ #solr indexes
+ if mc.is_solr(confdir):
+ print "setting up solr collections..."
+ mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "vertex_index", logdir)
+ mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "edge_index", logdir)
+ mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "fulltext_index", logdir)
+
web_app_path = os.path.join(web_app_dir, "atlas")
if (mc.isCygwin()):
web_app_path = mc.convertCygwinPath(web_app_path)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/bin/atlas_stop.py
----------------------------------------------------------------------
diff --git a/distro/src/bin/atlas_stop.py b/distro/src/bin/atlas_stop.py
index 265084e..a72c724 100755
--- a/distro/src/bin/atlas_stop.py
+++ b/distro/src/bin/atlas_stop.py
@@ -54,6 +54,10 @@ def main():
if os.path.exists(atlas_pid_file):
os.remove(atlas_pid_file)
+ # stop solr
+ if mc.is_solr_local(confdir):
+ mc.run_solr(mc.solrBinDir(atlas_home), "stop", None, mc.solrPort(), None, True)
+
# stop hbase
if mc.is_hbase_local(confdir):
mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "stop", None, None, True)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/conf/atlas-application.properties
----------------------------------------------------------------------
diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties
index b2b62aa..68a0021 100755
--- a/distro/src/conf/atlas-application.properties
+++ b/distro/src/conf/atlas-application.properties
@@ -30,11 +30,11 @@ atlas.graph.storage.hbase.regions-per-server=1
atlas.graph.storage.lock.wait-time=10000
#Solr
-#atlas.graph.index.search.backend=solr
+atlas.graph.index.search.backend=solr5
# Solr cloud mode properties
-#atlas.graph.index.search.solr.mode=cloud
-#atlas.graph.index.search.solr.zookeeper-url=localhost:2181
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.zookeeper-url=localhost:2181
#Solr http mode properties
#atlas.graph.index.search.solr.mode=http
@@ -42,11 +42,11 @@ atlas.graph.storage.lock.wait-time=10000
# Graph Search Index
#ElasticSearch
-atlas.graph.index.search.backend=elasticsearch
-atlas.graph.index.search.directory=${sys:atlas.home}/data/es
-atlas.graph.index.search.elasticsearch.client-only=false
-atlas.graph.index.search.elasticsearch.local-mode=true
-atlas.graph.index.search.elasticsearch.create.sleep=2000
+#atlas.graph.index.search.backend=elasticsearch
+#atlas.graph.index.search.directory=${sys:atlas.home}/data/es
+#atlas.graph.index.search.elasticsearch.client-only=false
+#atlas.graph.index.search.elasticsearch.local-mode=true
+#atlas.graph.index.search.elasticsearch.create.sleep=2000
######### Notification Configs #########
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/conf/hbase/hbase-site.xml.template
----------------------------------------------------------------------
diff --git a/distro/src/conf/hbase/hbase-site.xml.template b/distro/src/conf/hbase/hbase-site.xml.template
index f5c4aad..d599c20 100644
--- a/distro/src/conf/hbase/hbase-site.xml.template
+++ b/distro/src/conf/hbase/hbase-site.xml.template
@@ -19,11 +19,11 @@
<configuration>
<property>
<name>hbase.rootdir</name>
- <value>${url_prefix}${hbase_home}/root</value>
+ <value>${url_prefix}${atlas_data}/hbase-root</value>
</property>
<property>
<name>hbase.zookeeper.property.dataDir</name>
- <value>${hbase_home}/zookeeper-data</value>
+ <value>${atlas_data}/hbase-zookeeper-data</value>
</property>
<property>
<name>hbase.master.info.port</name>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/main/assemblies/standalone-package.xml
----------------------------------------------------------------------
diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml
index 527f604..09ff47d 100755
--- a/distro/src/main/assemblies/standalone-package.xml
+++ b/distro/src/main/assemblies/standalone-package.xml
@@ -62,6 +62,13 @@
</fileSet>
<fileSet>
+ <directory>target/solr</directory>
+ <outputDirectory>solr</outputDirectory>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ <fileSet>
<directory>../logs</directory>
<outputDirectory>logs</outputDirectory>
<directoryMode>0777</directoryMode>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/distro/src/test/python/scripts/TestMetadata.py
----------------------------------------------------------------------
diff --git a/distro/src/test/python/scripts/TestMetadata.py b/distro/src/test/python/scripts/TestMetadata.py
index 9e55869..00ca3e2 100644
--- a/distro/src/test/python/scripts/TestMetadata.py
+++ b/distro/src/test/python/scripts/TestMetadata.py
@@ -20,6 +20,7 @@ limitations under the License.
import sys
from os import environ
from mock import patch
+from mock import call
import unittest
import logging
import atlas_config as mc
@@ -32,6 +33,7 @@ logger = logging.getLogger()
class TestMetadata(unittest.TestCase):
@patch.object(mc,"runProcess")
@patch.object(mc,"configure_hbase")
+ @patch.object(mc,"getConfig")
@patch.object(mc,"grep")
@patch.object(mc,"exist_pid")
@patch.object(mc,"writePid")
@@ -41,7 +43,7 @@ class TestMetadata(unittest.TestCase):
@patch("os.path.exists")
@patch.object(mc, "java")
- def test_main(self, java_mock, exists_mock, expandWebApp_mock, atlasDir_mock, executeEnvSh_mock, writePid_mock, exist_pid_mock, grep_mock, configure_hbase_mock, runProcess_mock):
+ def test_main(self, java_mock, exists_mock, expandWebApp_mock, atlasDir_mock, executeEnvSh_mock, writePid_mock, exist_pid_mock, grep_mock, getConfig_mock, configure_hbase_mock, runProcess_mock):
sys.argv = []
exists_mock.return_value = True
expandWebApp_mock.return_value = "webapp"
@@ -50,13 +52,28 @@ class TestMetadata(unittest.TestCase):
exist_pid_mock(789)
exist_pid_mock.assert_called_with(789)
grep_mock.return_value = "hbase"
+ getConfig_mock.return_value = "localhost:9838"
atlas.main()
self.assertTrue(configure_hbase_mock.called)
+
if IS_WINDOWS:
- runProcess_mock.assert_called_with(['atlas_home\\hbase\\bin\\start-hbase.cmd', '--config', 'atlas_home\\hbase\\conf'], 'atlas_home\\logs', False, True)
+ calls = [call(['atlas_home\\hbase\\bin\\start-hbase.cmd', '--config', 'atlas_home\\hbase\\conf', 'start', 'master'], 'atlas_home\\logs', False, True),
+ call(['atlas_home\\solr\\bin\\solr.cmd', 'start', '-z', 'localhost:9838', '-p', '9838'], 'atlas_home\\logs', False, True),
+ call(['atlas_home\\solr\\bin\\solr.cmd', 'create', '-c', 'vertex_index', '-d', 'atlas_home\\solr\\server\\solr\\configsets\\basic_configs\\conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home\\logs', False, True),
+ call(['atlas_home\\solr\\bin\\solr.cmd', 'create', '-c', 'edge_index', '-d', 'atlas_home\\solr\\server\\solr\\configsets\\basic_configs\\conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home\\logs', False, True),
+ call(['atlas_home\\solr\\bin\\solr.cmd', 'create', '-c', 'fulltext_index', '-d', 'atlas_home\\solr\\server\\solr\\configsets\\basic_configs\\conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home\\logs', False, True)]
+
+ runProcess_mock.assert_has_calls(calls)
else:
- runProcess_mock.assert_called_with(['atlas_home/hbase/bin/hbase-daemon.sh', '--config', 'atlas_home/hbase/conf', 'start', 'master'], 'atlas_home/logs', False, True)
+ calls = [call(['atlas_home/hbase/bin/hbase-daemon.sh', '--config', 'atlas_home/hbase/conf', 'start', 'master'], 'atlas_home/logs', False, True),
+ call(['atlas_home/solr/bin/solr', 'start', '-z', 'localhost:9838', '-p', '9838'], 'atlas_home/logs', False, True),
+ call(['atlas_home/solr/bin/solr', 'create', '-c', 'vertex_index', '-d', 'atlas_home/solr/server/solr/configsets/basic_configs/conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home/logs', False, True),
+ call(['atlas_home/solr/bin/solr', 'create', '-c', 'edge_index', '-d', 'atlas_home/solr/server/solr/configsets/basic_configs/conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home/logs', False, True),
+ call(['atlas_home/solr/bin/solr', 'create', '-c', 'fulltext_index', '-d', 'atlas_home/solr/server/solr/configsets/basic_configs/conf', '-shards', '1', '-replicationFactor', '1'], 'atlas_home/logs', False, True)]
+
+ runProcess_mock.assert_has_calls(calls)
+
self.assertTrue(java_mock.called)
if IS_WINDOWS:
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/docs/src/site/twiki/Configuration.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki
index 0793c82..879c06c 100644
--- a/docs/src/site/twiki/Configuration.twiki
+++ b/docs/src/site/twiki/Configuration.twiki
@@ -54,7 +54,10 @@ Without Ranger, HBase shell can be used to set the permissions.
</verbatim>
Note that HBase is included in the distribution so that a standalone instance of HBase can be started as the default
-storage backend for the graph repository.
+storage backend for the graph repository. By default, the standalone HBase instance will be started and stopped along
+with the Atlas server when the configuraton 'atlas.graph.storage.backend' is set to 'hbase' and
+'atlas.graph.storage.hostname' is set to 'localhost'. Set the MANAGE_LOCAL_HBASE environment variable to 'False' to
+override this behavior.
---+++ Graph Search Index
This section sets up the graph db - titan - to use an search indexing system. The example
@@ -77,6 +80,12 @@ Please note that Solr installation in Cloud mode is a prerequisite before config
atlas.graph.index.search.solr.zookeeper-url=<the ZK quorum setup for solr as comma separated value> eg: 10.1.6.4:2181,10.1.6.5:2181
</verbatim>
+Also note that Solr is included in the distribution so that a standalone instance of Solr can be started as the default
+search indexing backend. By default, the standalone Solr instance will be started and stopped along
+with the Atlas server when the configuraton 'atlas.graph.index.search.backend' is set to 'solr5' and
+'atlas.graph.index.search.solr.zookeeper-url' is set to 'localhost'. Set the MANAGE_LOCAL_SOLR environment variable to
+'False' to override this behavior.
+
---+++ Choosing between Persistence and Indexing Backends
Refer http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html and http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html for choosing between the persistence backends.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/docs/src/site/twiki/InstallationSteps.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki
index 270d1c4..006e1e8 100644
--- a/docs/src/site/twiki/InstallationSteps.twiki
+++ b/docs/src/site/twiki/InstallationSteps.twiki
@@ -53,6 +53,9 @@ Tar is structured as follows
|- server
|- webapp
|- atlas.war
+|- solr
+ |- bin
+ ...
|- README
|- NOTICE.txt
|- LICENSE.txt
@@ -67,6 +70,10 @@ expanded and moved to hbase/conf/hbase-site.xml for the initial standalone HBase
graph persistence for a different HBase instance, please see "Graph persistence engine - HBase" in the
[[Configuration][Configuration]] section.
+Also note that Solr is included in the distribution so that a standalone instance of Solr can be started as the default
+search indexing backend. To configure ATLAS search indexing for a different Solr instance, please see
+"Graph Search Index - Solr" in the [[Configuration][Configuration]] section.
+
---+++ Installing & Running Atlas
---++++ Installing Atlas
@@ -184,14 +191,14 @@ For configuring Titan to work with Solr, please follow the instructions below
$SOLR_HOME/bin/solr start -c -z <zookeeper_host:port> -p 8983
</verbatim>
- * Run the following commands from SOLR_HOME directory to create collections in Solr corresponding to the indexes that Atlas uses. In the case that the ATLAS and SOLR instance are on 2 different hosts,
+ * Run the following commands from SOLR_BIN (e.g. $SOLR_HOME/bin) directory to create collections in Solr corresponding to the indexes that Atlas uses. In the case that the ATLAS and SOLR instance are on 2 different hosts,
first copy the required configuration files from ATLAS_HOME/conf/solr on the ATLAS instance host to the Solr instance host. SOLR_CONF in the below mentioned commands refer to the directory where the solr configuration files
have been copied to on Solr host:
<verbatim>
- bin/solr create -c vertex_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor
- bin/solr create -c edge_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor
- bin/solr create -c fulltext_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor
+ $SOLR_BIN/solr create -c vertex_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor
+ $SOLR_BIN/solr create -c edge_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor
+ $SOLR_BIN/solr create -c fulltext_index -d SOLR_CONF -shards #numShards -replicationFactor #replicationFactor
</verbatim>
Note: If numShards and replicationFactor are not specified, they default to 1 which suffices if you are trying out solr with ATLAS on a single node instance.
@@ -200,6 +207,9 @@ For configuring Titan to work with Solr, please follow the instructions below
The number of replicas (replicationFactor) can be set according to the redundancy required.
+ Also note that solr will automatically be called to create the indexes when the Atlas server is started if the
+ SOLR_BIN and SOLR_CONF environment variables are set and the search indexing backend is set to 'solr5'.
+
* Change ATLAS configuration to point to the Solr instance setup. Please make sure the following configurations are set to the below values in ATLAS_HOME/conf/atlas-application.properties
<verbatim>
atlas.graph.index.search.backend=solr5
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/af63bb88/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 5d37d07..a067fe9 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -20,6 +20,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
+ATLAS-663,ATLAS-673 Install Setup: SOLR (tbeerbower via sumasai)
ATLAS-629 Kafka messages in ATLAS_HOOK might be lost in HA mode at the instant of failover. (yhemanth)
ATLAS-758 hdfs location of hive table is pointing to old location even after rename ( sumasai )
ATLAS-667 Entity delete should check for required reverse references ( dkantor via sumasai )