You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by yh...@apache.org on 2016/05/31 17:33:37 UTC
incubator-atlas git commit: ATLAS-833 Make default build profile for
External HBase and Solr (tbeerbower via yhemanth)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 42d5e37e3 -> 208e90b1a
ATLAS-833 Make default build profile for External HBase and Solr (tbeerbower via yhemanth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/208e90b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/208e90b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/208e90b1
Branch: refs/heads/master
Commit: 208e90b1ae33074aeb89c79e48a0a3489d5cc415
Parents: 42d5e37
Author: Hemanth Yamijala <hy...@hortonworks.com>
Authored: Tue May 31 23:03:14 2016 +0530
Committer: Hemanth Yamijala <hy...@hortonworks.com>
Committed: Tue May 31 23:03:14 2016 +0530
----------------------------------------------------------------------
distro/pom.xml | 81 ++++++++++++++++++++---
distro/src/conf/atlas-application.properties | 26 +-------
distro/src/conf/atlas-env.sh | 3 +
docs/src/site/twiki/Configuration.twiki | 12 ++--
docs/src/site/twiki/InstallationSteps.twiki | 22 ++++--
release-log.txt | 1 +
6 files changed, 103 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/distro/pom.xml
----------------------------------------------------------------------
diff --git a/distro/pom.xml b/distro/pom.xml
index 22f70ae..131a488 100644
--- a/distro/pom.xml
+++ b/distro/pom.xml
@@ -28,9 +28,28 @@
<artifactId>atlas-distro</artifactId>
<description>Apache Atlas Distribution</description>
<name>Apache Atlas Distribution</name>
+
+ <!-- by default configure hbase and solr with the distribution -->
<properties>
- <titan.storage.backend>berkeleyje</titan.storage.backend>
- <titan.index.backend>elasticsearch</titan.index.backend>
+
+ <titan.storage.backend>hbase</titan.storage.backend>
+ <titan.storage.properties>#Hbase
+#For standalone mode , specify localhost
+#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
+atlas.graph.storage.hostname=
+atlas.graph.storage.hbase.regions-per-server=1
+atlas.graph.storage.lock.wait-time=10000
+ </titan.storage.properties>
+ <titan.index.backend>solr5</titan.index.backend>
+ <titan.index.properties>#Solr
+#Solr cloud mode properties
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.zookeeper-url=
+
+#Solr http mode properties
+#atlas.graph.index.search.solr.mode=http
+#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
+ </titan.index.properties>
<hbase.embedded>false</hbase.embedded>
<solr.embedded>false</solr.embedded>
</properties>
@@ -93,15 +112,59 @@
</plugins>
</build>
</profile>
- <!-- embedded_services profile to package and configure embedded hbase and solr with the distribution -->
+ <!-- profile to configure berkeley and elasticsearch with the distribution -->
+ <profile>
+ <id>berkeley-elasticsearch</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <titan.storage.backend>berkeleyje</titan.storage.backend>
+ <titan.storage.properties>#Berkeley
+atlas.graph.storage.directory=${sys:atlas.home}/data/berkley
+ </titan.storage.properties>
+ <titan.index.backend>elasticsearch</titan.index.backend>
+ <titan.index.properties>#ElasticSearch
+atlas.graph.index.search.directory=${sys:atlas.home}/data/es
+atlas.graph.index.search.elasticsearch.client-only=false
+atlas.graph.index.search.elasticsearch.local-mode=true
+atlas.graph.index.search.elasticsearch.create.sleep=2000
+ </titan.index.properties>
+ </properties>
+ </profile>
+
+ <!-- profile to configure berkeley and elasticsearch with the distribution -->
<profile>
- <id>embedded_services</id>
+ <id>external-hbase-solr</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ </profile>
+
+ <!-- profile to package and configure embedded hbase and solr with the distribution -->
+ <profile>
+ <id>embedded-hbase-solr</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
- <titan.storage.backend>hbase</titan.storage.backend>
- <titan.index.backend>solr5</titan.index.backend>
+ <titan.storage.properties>#Hbase
+#For standalone mode , specify localhost
+#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
+atlas.graph.storage.hostname=localhost
+atlas.graph.storage.hbase.regions-per-server=1
+atlas.graph.storage.lock.wait-time=10000
+ </titan.storage.properties>
+ <titan.index.properties>#Solr
+#Solr cloud mode properties
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.zookeeper-url=localhost:2181
+
+#Solr http mode properties
+#atlas.graph.index.search.solr.mode=http
+#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
+ </titan.index.properties>
+
<hbase.embedded>true</hbase.embedded>
<solr.embedded>true</solr.embedded>
@@ -193,9 +256,9 @@
<resource>
<directory>src/bin</directory>
<filtering>true</filtering>
- <includes>
- <include>**/*.py</include>
- </includes>
+ <includes>
+ <include>**/*.py</include>
+ </includes>
<targetPath>bin</targetPath>
</resource>
<resource>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/distro/src/conf/atlas-application.properties
----------------------------------------------------------------------
diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties
index bb12538..bfa40e8 100755
--- a/distro/src/conf/atlas-application.properties
+++ b/distro/src/conf/atlas-application.properties
@@ -19,33 +19,13 @@
######### Graph Database Configs #########
# Graph Storage
atlas.graph.storage.backend=${titan.storage.backend}
-atlas.graph.storage.directory=${sys:atlas.home}/data/berkley
-#Hbase as storage backend
-#atlas.graph.storage.backend=hbase
-#For standalone mode , specify localhost
-#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
-atlas.graph.storage.hostname=localhost
-atlas.graph.storage.hbase.regions-per-server=1
-atlas.graph.storage.lock.wait-time=10000
+${titan.storage.properties}
-#Solr
+# Graph Search Index
atlas.graph.index.search.backend=${titan.index.backend}
-# Solr cloud mode properties
-atlas.graph.index.search.solr.mode=cloud
-atlas.graph.index.search.solr.zookeeper-url=localhost:2181
-
-#Solr http mode properties
-#atlas.graph.index.search.solr.mode=http
-#atlas.graph.index.search.solr.http-urls=http://localhost:8983/solr
-
-# Graph Search Index
-#ElasticSearch
-atlas.graph.index.search.directory=${sys:atlas.home}/data/es
-atlas.graph.index.search.elasticsearch.client-only=false
-atlas.graph.index.search.elasticsearch.local-mode=true
-atlas.graph.index.search.elasticsearch.create.sleep=2000
+${titan.index.properties}
######### Notification Configs #########
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/distro/src/conf/atlas-env.sh
----------------------------------------------------------------------
diff --git a/distro/src/conf/atlas-env.sh b/distro/src/conf/atlas-env.sh
index c671c45..6a5d5e4 100644
--- a/distro/src/conf/atlas-env.sh
+++ b/distro/src/conf/atlas-env.sh
@@ -57,5 +57,8 @@
# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
#export ATLAS_EXPANDED_WEBAPP_DIR=
+# indicates whether or not a local instance of HBase should be started for Atlas
export MANAGE_LOCAL_HBASE=${hbase.embedded}
+
+# indicates whether or not a local instance of Solr should be started for Atlas
export MANAGE_LOCAL_SOLR=${solr.embedded}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/docs/src/site/twiki/Configuration.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki
index f532a73..2884f42 100644
--- a/docs/src/site/twiki/Configuration.twiki
+++ b/docs/src/site/twiki/Configuration.twiki
@@ -53,10 +53,10 @@ Without Ranger, HBase shell can be used to set the permissions.
echo "grant 'atlas', 'RWXCA', 'titan'" | hbase shell
</verbatim>
-Note that if the embedded_services profile is used then HBase is included in the distribution so that a standalone
-instance of HBase can be started as the default storage backend for the graph repository. Using the embedded_services
+Note that if the embedded-hbase-solr profile is used then HBase is included in the distribution so that a standalone
+instance of HBase can be started as the default storage backend for the graph repository. Using the embedded-hbase-solr
profile will configure Atlas so that HBase instance will be started and stopped along with the Atlas server by default.
-To use the embedded_services profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
+To use the embedded-hbase-solr profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
section.
---+++ Graph Search Index
@@ -80,10 +80,10 @@ Please note that Solr installation in Cloud mode is a prerequisite before config
atlas.graph.index.search.solr.zookeeper-url=<the ZK quorum setup for solr as comma separated value> eg: 10.1.6.4:2181,10.1.6.5:2181
</verbatim>
-Also note that if the embedded_services profile is used then Solr is included in the distribution so that a standalone
-instance of Solr can be started as the default search indexing backend. Using the embedded_services profile will
+Also note that if the embedded-hbase-solr profile is used then Solr is included in the distribution so that a standalone
+instance of Solr can be started as the default search indexing backend. Using the embedded-hbase-solr profile will
configure Atlas so that the standalone Solr instance will be started and stopped along with the Atlas server by default.
-To use the embedded_services profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
+To use the embedded-hbase-solr profile please see "Building Atlas" in the [[InstallationSteps][Installation Steps]]
section.
---+++ Choosing between Persistence and Indexing Backends
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/docs/src/site/twiki/InstallationSteps.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki
index fe6b4a7..9699a94 100644
--- a/docs/src/site/twiki/InstallationSteps.twiki
+++ b/docs/src/site/twiki/InstallationSteps.twiki
@@ -18,15 +18,29 @@ mvn clean package -Pdist
</verbatim>
-To build a distribution that packages HBase and Solr, build with the embedded_services profile.
+To build a distribution that configures Atlas for external HBase and Solr, build with the external-hbase-solr profile.
<verbatim>
-mvn clean package -Pdist,embedded_services
+mvn clean package -Pdist,external-hbase-solr
</verbatim>
-Using the embedded_services profile will configure Atlas so that an HBase instance and a Solr instance will be started
+Note that when the external-hbase-solr profile is used the following steps need to be completed to make Atlas functional.
+ * Configure atlas.graph.storage.hostname (see "Graph persistence engine - HBase" in the [[Configuration][Configuration]] section).
+ * Configure atlas.graph.index.search.solr.zookeeper-url (see "Graph Search Index - Solr" in the [[Configuration][Configuration]] section).
+ * Set HBASE_CONF_DIR to point to a valid HBase config directory (see "Graph persistence engine - HBase" in the [[Configuration][Configuration]] section).
+ * Create the SOLR indices (see "Graph Search Index - Solr" in the [[Configuration][Configuration]] section).
+
+To build a distribution that packages HBase and Solr, build with the embedded-hbase-solr profile.
+
+<verbatim>
+
+mvn clean package -Pdist,embedded-hbase-solr
+
+</verbatim>
+
+Using the embedded-hbase-solr profile will configure Atlas so that an HBase instance and a Solr instance will be started
and stopped along with the Atlas server by default.
Tar can be found in atlas/distro/target/apache-atlas-${project.version}-bin.tar.gz
@@ -75,7 +89,7 @@ Tar is structured as follows
</verbatim>
-Note that if the embedded_services profile is specified for the build then HBase and Solr are included in the
+Note that if the embedded-hbase-solr profile is specified for the build then HBase and Solr are included in the
distribution.
In this case, a standalone instance of HBase can be started as the default storage backend for the graph repository.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/208e90b1/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 4dbe0d4..7f25604 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -22,6 +22,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
+ATLAS-833 Make default build profile for External HBase and Solr (tbeerbower via yhemanth)
ATLAS-841 mvn clean install of Atlas is failing python unit tests (yhemanth)
ATLAS-834 Handle exceptions from HiveHook executor.submit() (sumasai)
ATLAS-635 Process showing old entity name where as actual entity is renamed ( svimal2106 via sumasai )