You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/05/27 11:37:57 UTC
incubator-atlas git commit: ATLAS-823 Atlas should use external HBase
and SOLR (tbeerbower via shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 2e02ae628 -> 0143486ad
ATLAS-823 Atlas should use external HBase and SOLR (tbeerbower via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/0143486a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/0143486a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/0143486a
Branch: refs/heads/master
Commit: 0143486ad2024f14f9f8fb2f03c4013f1462c2ce
Parents: 2e02ae6
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Fri May 27 17:07:51 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Fri May 27 17:07:51 2016 +0530
----------------------------------------------------------------------
distro/pom.xml | 85 +++++++++++++-------
distro/src/bin/atlas_config.py | 4 +-
distro/src/bin/atlas_start.py | 2 -
distro/src/conf/atlas-application.properties | 19 +++--
distro/src/conf/atlas-env.sh | 4 +
.../src/main/assemblies/standalone-package.xml | 2 +-
docs/src/site/twiki/Configuration.twiki | 20 ++---
docs/src/site/twiki/InstallationSteps.twiki | 28 +++++--
release-log.txt | 1 +
9 files changed, 105 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0143486a/distro/pom.xml
----------------------------------------------------------------------
diff --git a/distro/pom.xml b/distro/pom.xml
index 1ab092d..22f70ae 100644
--- a/distro/pom.xml
+++ b/distro/pom.xml
@@ -28,15 +28,11 @@
<artifactId>atlas-distro</artifactId>
<description>Apache Atlas Distribution</description>
<name>Apache Atlas Distribution</name>
-
<properties>
- <hbase.dir>${project.build.directory}/hbase</hbase.dir>
- <hbase.tar>http://archive.apache.org/dist/hbase/${hbase.version}/hbase-${hbase.version}-bin.tar.gz</hbase.tar>
- <hbase.folder>hbase-${hbase.version}</hbase.folder>
-
- <solr.dir>${project.build.directory}/solr</solr.dir>
- <solr.tar>http://archive.apache.org/dist/lucene/solr/${solr.version}/solr-${solr.version}.tgz</solr.tar>
- <solr.folder>solr-${solr.version}</solr.folder>
+ <titan.storage.backend>berkeleyje</titan.storage.backend>
+ <titan.index.backend>elasticsearch</titan.index.backend>
+ <hbase.embedded>false</hbase.embedded>
+ <solr.embedded>false</solr.embedded>
</properties>
<profiles>
@@ -76,10 +72,55 @@
<build>
<plugins>
<plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assemblies/standalone-package.xml</descriptor>
+ <descriptor>src/main/assemblies/src-package.xml</descriptor>
+ </descriptors>
+ <finalName>apache-atlas-${project.version}</finalName>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <!-- embedded_services profile to package and configure embedded hbase and solr with the distribution -->
+ <profile>
+ <id>embedded_services</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <titan.storage.backend>hbase</titan.storage.backend>
+ <titan.index.backend>solr5</titan.index.backend>
+ <hbase.embedded>true</hbase.embedded>
+ <solr.embedded>true</solr.embedded>
+
+ <hbase.dir>${project.build.directory}/hbase</hbase.dir>
+ <hbase.tar>http://archive.apache.org/dist/hbase/${hbase.version}/hbase-${hbase.version}-bin.tar.gz</hbase.tar>
+ <hbase.folder>hbase-${hbase.version}</hbase.folder>
+
+ <solr.dir>${project.build.directory}/solr</solr.dir>
+ <solr.tar>http://archive.apache.org/dist/lucene/solr/${solr.version}/solr-${solr.version}.tgz</solr.tar>
+ <solr.folder>solr-${solr.version}</solr.folder>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
<executions>
+ <!-- package hbase -->
<execution>
<id>hbase</id>
<phase>generate-resources</phase>
@@ -109,6 +150,7 @@
</target>
</configuration>
</execution>
+ <!-- package solr -->
<execution>
<id>solr</id>
<phase>generate-resources</phase>
@@ -140,32 +182,13 @@
</execution>
</executions>
</plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>single</goal>
- </goals>
- <phase>package</phase>
- <configuration>
- <descriptors>
- <descriptor>src/main/assemblies/standalone-package.xml</descriptor>
- <descriptor>src/main/assemblies/src-package.xml</descriptor>
- </descriptors>
- <finalName>apache-atlas-${project.version}</finalName>
- <tarLongFileMode>gnu</tarLongFileMode>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
</profile>
</profiles>
<build>
- <outputDirectory>target/bin</outputDirectory>
+ <outputDirectory>target</outputDirectory>
<resources>
<resource>
<directory>src/bin</directory>
@@ -173,6 +196,12 @@
<includes>
<include>**/*.py</include>
</includes>
+ <targetPath>bin</targetPath>
+ </resource>
+ <resource>
+ <directory>src/conf</directory>
+ <filtering>true</filtering>
+ <targetPath>conf</targetPath>
</resource>
</resources>
<plugins>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0143486a/distro/src/bin/atlas_config.py
----------------------------------------------------------------------
diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py
index 8f3479c..fab4046 100755
--- a/distro/src/bin/atlas_config.py
+++ b/distro/src/bin/atlas_config.py
@@ -388,7 +388,7 @@ def is_hbase(confdir):
return grep(confdir, HBASE_STORAGE_CONF_ENTRY) is not None
def is_hbase_local(confdir):
- if os.environ.get(MANAGE_LOCAL_HBASE, "True").lower() == 'false':
+ if os.environ.get(MANAGE_LOCAL_HBASE, "False").lower() == 'false':
return False
confdir = os.path.join(confdir, CONF_FILE)
@@ -419,7 +419,7 @@ def is_solr(confdir):
return grep(confdir, SOLR_INDEX_CONF_ENTRY) is not None
def is_solr_local(confdir):
- if os.environ.get(MANAGE_LOCAL_SOLR, "True").lower() == 'false':
+ if os.environ.get(MANAGE_LOCAL_SOLR, "False").lower() == 'false':
return False
confdir = os.path.join(confdir, CONF_FILE)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0143486a/distro/src/bin/atlas_start.py
----------------------------------------------------------------------
diff --git a/distro/src/bin/atlas_start.py b/distro/src/bin/atlas_start.py
index cb869bb..1ba38f8 100755
--- a/distro/src/bin/atlas_start.py
+++ b/distro/src/bin/atlas_start.py
@@ -118,8 +118,6 @@ def main():
mc.run_solr(mc.solrBinDir(atlas_home), "start", mc.get_solr_zk_url(confdir), mc.solrPort(), logdir)
print "solr started."
- #solr indexes
- if mc.is_solr(confdir):
print "setting up solr collections..."
mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "vertex_index", logdir)
mc.create_solr_collection(mc.solrBinDir(atlas_home), mc.solrConfDir(atlas_home), "edge_index", logdir)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0143486a/distro/src/conf/atlas-application.properties
----------------------------------------------------------------------
diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties
index 26f84e9..bb12538 100755
--- a/distro/src/conf/atlas-application.properties
+++ b/distro/src/conf/atlas-application.properties
@@ -18,11 +18,11 @@
######### Graph Database Configs #########
# Graph Storage
-#atlas.graph.storage.backend=berkeleyje
-#atlas.graph.storage.directory=${sys:atlas.home}/data/berkley
+atlas.graph.storage.backend=${titan.storage.backend}
+atlas.graph.storage.directory=${sys:atlas.home}/data/berkley
-#Hbase as stoarge backend
-atlas.graph.storage.backend=hbase
+#Hbase as storage backend
+#atlas.graph.storage.backend=hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=localhost
@@ -30,7 +30,7 @@ atlas.graph.storage.hbase.regions-per-server=1
atlas.graph.storage.lock.wait-time=10000
#Solr
-atlas.graph.index.search.backend=solr5
+atlas.graph.index.search.backend=${titan.index.backend}
# Solr cloud mode properties
atlas.graph.index.search.solr.mode=cloud
@@ -42,11 +42,10 @@ atlas.graph.index.search.solr.zookeeper-url=localhost:2181
# Graph Search Index
#ElasticSearch
-#atlas.graph.index.search.backend=elasticsearch
-#atlas.graph.index.search.directory=${sys:atlas.home}/data/es
-#atlas.graph.index.search.elasticsearch.client-only=false
-#atlas.graph.index.search.elasticsearch.local-mode=true
-#atlas.graph.index.search.elasticsearch.create.sleep=2000
+atlas.graph.index.search.directory=${sys:atlas.home}/data/es
+atlas.graph.index.search.elasticsearch.client-only=false
+atlas.graph.index.search.elasticsearch.local-mode=true
+atlas.graph.index.search.elasticsearch.create.sleep=2000
######### Notification Configs #########
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0143486a/distro/src/conf/atlas-env.sh
----------------------------------------------------------------------
diff --git a/distro/src/conf/atlas-env.sh b/distro/src/conf/atlas-env.sh
index 4f1f46c..c671c45 100644
--- a/distro/src/conf/atlas-env.sh
+++ b/distro/src/conf/atlas-env.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -55,3 +56,6 @@
# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
#export ATLAS_EXPANDED_WEBAPP_DIR=
+
+export MANAGE_LOCAL_HBASE=${hbase.embedded}
+export MANAGE_LOCAL_SOLR=${solr.embedded}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0143486a/distro/src/main/assemblies/standalone-package.xml
----------------------------------------------------------------------
diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml
index 09ff47d..1c7b2c5 100755
--- a/distro/src/main/assemblies/standalone-package.xml
+++ b/distro/src/main/assemblies/standalone-package.xml
@@ -28,7 +28,7 @@
<baseDirectory>apache-atlas-${project.version}</baseDirectory>
<fileSets>
<fileSet>
- <directory>src/conf/</directory>
+ <directory>target/conf/</directory>
<outputDirectory>conf</outputDirectory>
</fileSet>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0143486a/docs/src/site/twiki/Configuration.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki
index 879c06c..f532a73 100644
--- a/docs/src/site/twiki/Configuration.twiki
+++ b/docs/src/site/twiki/Configuration.twiki
@@ -53,11 +53,11 @@ Without Ranger, HBase shell can be used to set the permissions.
echo "grant 'atlas', 'RWXCA', 'titan'" | hbase shell
</verbatim>
-Note that HBase is included in the distribution so that a standalone instance of HBase can be started as the default
-storage backend for the graph repository. By default, the standalone HBase instance will be started and stopped along
-with the Atlas server when the configuraton 'atlas.graph.storage.backend' is set to 'hbase' and
-'atlas.graph.storage.hostname' is set to 'localhost'. Set the MANAGE_LOCAL_HBASE environment variable to 'False' to
-override this behavior.
+Note that if the embedded_services profile is used then HBase is included in the distribution so that a standalone
+instance of HBase can be started as the default storage backend for the graph repository. Using the embedded_services
+profile will configure Atlas so that HBase instance will be started and stopped along with the Atlas server by default.
+To use the embedded_services profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
+section.
---+++ Graph Search Index
This section sets up the graph db - titan - to use an search indexing system. The example
@@ -80,11 +80,11 @@ Please note that Solr installation in Cloud mode is a prerequisite before config
atlas.graph.index.search.solr.zookeeper-url=<the ZK quorum setup for solr as comma separated value> eg: 10.1.6.4:2181,10.1.6.5:2181
</verbatim>
-Also note that Solr is included in the distribution so that a standalone instance of Solr can be started as the default
-search indexing backend. By default, the standalone Solr instance will be started and stopped along
-with the Atlas server when the configuraton 'atlas.graph.index.search.backend' is set to 'solr5' and
-'atlas.graph.index.search.solr.zookeeper-url' is set to 'localhost'. Set the MANAGE_LOCAL_SOLR environment variable to
-'False' to override this behavior.
+Also note that if the embedded_services profile is used then Solr is included in the distribution so that a standalone
+instance of Solr can be started as the default search indexing backend. Using the embedded_services profile will
+configure Atlas so that the standalone Solr instance will be started and stopped along with the Atlas server by default.
+To use the embedded_services profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
+section.
---+++ Choosing between Persistence and Indexing Backends
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0143486a/docs/src/site/twiki/InstallationSteps.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki
index 006e1e8..fe6b4a7 100644
--- a/docs/src/site/twiki/InstallationSteps.twiki
+++ b/docs/src/site/twiki/InstallationSteps.twiki
@@ -18,6 +18,17 @@ mvn clean package -Pdist
</verbatim>
+To build a distribution that packages HBase and Solr, build with the embedded_services profile.
+
+<verbatim>
+
+mvn clean package -Pdist,embedded_services
+
+</verbatim>
+
+Using the embedded_services profile will configure Atlas so that an HBase instance and a Solr instance will be started
+and stopped along with the Atlas server by default.
+
Tar can be found in atlas/distro/target/apache-atlas-${project.version}-bin.tar.gz
Tar is structured as follows
@@ -64,15 +75,18 @@ Tar is structured as follows
</verbatim>
-Note that HBase is included in the distribution so that a standalone instance of HBase can be started as the default
-storage backend for the graph repository. During Atlas installation the conf/hbase/hbase-site.xml.template gets
-expanded and moved to hbase/conf/hbase-site.xml for the initial standalone HBase configuration. To configure ATLAS
+Note that if the embedded_services profile is specified for the build then HBase and Solr are included in the
+distribution.
+
+In this case, a standalone instance of HBase can be started as the default storage backend for the graph repository.
+During Atlas installation the conf/hbase/hbase-site.xml.template gets expanded and moved to hbase/conf/hbase-site.xml
+for the initial standalone HBase configuration. To configure ATLAS
graph persistence for a different HBase instance, please see "Graph persistence engine - HBase" in the
[[Configuration][Configuration]] section.
-Also note that Solr is included in the distribution so that a standalone instance of Solr can be started as the default
-search indexing backend. To configure ATLAS search indexing for a different Solr instance, please see
-"Graph Search Index - Solr" in the [[Configuration][Configuration]] section.
+Also, a standalone instance of Solr can be started as the default search indexing backend. To configure ATLAS search
+indexing for a different Solr instance please see "Graph Search Index - Solr" in the
+[[Configuration][Configuration]] section.
---+++ Installing & Running Atlas
@@ -310,4 +324,4 @@ ensure the setup can run and delete the Zookeeper node at =/apache_atlas/setup_i
run setup again.
If the setup failed due to HBase Titan schema setup errors, it may be necessary to repair the HBase schema. If no
-data has been stored, one can also disable and drop the 'titan' schema in HBase to let setup run again.
\ No newline at end of file
+data has been stored, one can also disable and drop the 'titan' schema in HBase to let setup run again.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/0143486a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index b6d1f9d..5498486 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -22,6 +22,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
+ATLAS-823 Atlas should use external HBase and SOLR (tbeerbower via shwethags)
ATLAS-752 Column renames should retain traits/tags (svimal2106 via shwethags)
ATLAS-821 Atlas UI - Add arrow to navigate to child term (kevalbhatt18 via yhemanth)
ATLAS-812 Atlas UI - Associate Terms with Assets (kevalbhatt18 via yhemanth)