You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by su...@apache.org on 2015/12/18 11:44:10 UTC
[10/26] incubator-atlas git commit: ATLAS-354 Kerberized cluster:
quick_start.py fails to add sample data (shwethags)
ATLAS-354 Kerberized cluster: quick_start.py fails to add sample data (shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/b82ff048
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/b82ff048
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/b82ff048
Branch: refs/heads/branch-0.6-incubating
Commit: b82ff048496f416e2ea794a1054e2a9f57667005
Parents: 5165699
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Tue Dec 8 12:39:54 2015 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Tue Dec 8 12:41:26 2015 +0530
----------------------------------------------------------------------
.../src/test/resources/hive-site.xml | 5 --
.../main/java/org/apache/atlas/AtlasClient.java | 2 +-
distro/src/conf/application.properties | 2 +-
docs/src/site/twiki/Configuration.twiki | 51 +++++++++-----------
docs/src/site/twiki/InstallationSteps.twiki | 3 ++
docs/src/site/twiki/security.twiki | 3 +-
release-log.txt | 1 +
.../org/apache/atlas/examples/QuickStart.java | 16 ++++--
8 files changed, 44 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b82ff048/addons/hive-bridge/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/resources/hive-site.xml b/addons/hive-bridge/src/test/resources/hive-site.xml
index 2ec3b7c..b106903 100644
--- a/addons/hive-bridge/src/test/resources/hive-site.xml
+++ b/addons/hive-bridge/src/test/resources/hive-site.xml
@@ -32,11 +32,6 @@
</property>
<property>
- <name>atlas.rest.address</name>
- <value>http://localhost:21000/</value>
- </property>
-
- <property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:derby:${user.dir}/target/metastore_db;create=true</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b82ff048/client/src/main/java/org/apache/atlas/AtlasClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java
index 114548c..becc4db 100755
--- a/client/src/main/java/org/apache/atlas/AtlasClient.java
+++ b/client/src/main/java/org/apache/atlas/AtlasClient.java
@@ -127,7 +127,7 @@ public class AtlasClient {
}
protected Configuration getClientProperties() throws AtlasException {
- return ApplicationProperties.get(ApplicationProperties.CLIENT_PROPERTIES);
+ return ApplicationProperties.get();
}
enum API {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b82ff048/distro/src/conf/application.properties
----------------------------------------------------------------------
diff --git a/distro/src/conf/application.properties b/distro/src/conf/application.properties
index a7fc904..f9888bd 100755
--- a/distro/src/conf/application.properties
+++ b/distro/src/conf/application.properties
@@ -80,4 +80,4 @@ atlas.lineage.hive.table.schema.query.Table=Table where name='%s'\, columns
# SSL config
atlas.enableTLS=false
-######### Security Properties #########
+atlas.rest.address=http://localhost:21000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b82ff048/docs/src/site/twiki/Configuration.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki
index 526b300..8b942ab 100644
--- a/docs/src/site/twiki/Configuration.twiki
+++ b/docs/src/site/twiki/Configuration.twiki
@@ -1,17 +1,11 @@
----+ Configuring Apache Atlas
+---+ Configuring Apache Atlas - Application Properties
----++ Introduction
+All configuration in Atlas uses java properties style configuration. The main configuration file is application.properties which is in the *conf* dir at the deployed location. It consists of the following sections:
-All configuration in Atlas uses java properties style configuration.
----++ Application Properties
+---++ Graph Configs
-The main configuration file is application.properties which is in the *conf* dir at the deployed
-location. It consists of the following sections:
-
----+++ Graph Database Configs
-
----++++ Graph persistence engine
+---+++ Graph persistence engine
This section sets up the graph db - titan - to use a persistence engine. Please refer to
<a href="http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html">link</a> for more
@@ -22,7 +16,7 @@ atlas.graph.storage.backend=berkeleyje
atlas.graph.storage.directory=data/berkley
</verbatim>
----+++++ Graph persistence engine - Hbase
+---++++ Graph persistence engine - Hbase
Basic configuration
@@ -59,8 +53,7 @@ Without Ranger, HBase shell can be used to set the permissions.
echo "grant 'atlas', 'RWXCA', 'titan'" | hbase shell
</verbatim>
----++++ Graph Search Index
-
+---+++ Graph Search Index
This section sets up the graph db - titan - to use an search indexing system. The example
configuration below sets up to use an embedded Elastic search indexing system.
@@ -100,22 +93,22 @@ Solr in cloud mode is the recommended setup.
---+++ Switching Persistence Backend
- For switching the storage backend from BerkeleyDB to HBase and vice versa, refer the documentation for "Graph Persistence Engine" described above and restart ATLAS.
- The data in the indexing backend needs to be cleared else there will be discrepancies between the storage and indexing backend which could result in errors during the search.
- ElasticSearch runs by default in embedded mode and the data could easily be cleared by deleting the ATLAS_HOME/data/es directory.
- For Solr, the collections which were created during ATLAS Installation - vertex_index, edge_index, fulltext_index could be deleted which will cleanup the indexes
+For switching the storage backend from BerkeleyDB to HBase and vice versa, refer the documentation for "Graph Persistence Engine" described above and restart ATLAS.
+The data in the indexing backend needs to be cleared else there will be discrepancies between the storage and indexing backend which could result in errors during the search.
+ElasticSearch runs by default in embedded mode and the data could easily be cleared by deleting the ATLAS_HOME/data/es directory.
+For Solr, the collections which were created during ATLAS Installation - vertex_index, edge_index, fulltext_index could be deleted which will cleanup the indexes
---+++ Switching Index Backend
- Switching the Index backend requires clearing the persistence backend data. Otherwise there will be discrepancies between the persistence and index backends since switching the indexing backend means index data will be lost.
- This leads to "Fulltext" queries not working on the existing data
- For clearing the data for BerkeleyDB, delete the ATLAS_HOME/data/berkeley directory
- For clearing the data for HBase, in Hbase shell, run 'disable titan' and 'drop titan'
+Switching the Index backend requires clearing the persistence backend data. Otherwise there will be discrepancies between the persistence and index backends since switching the indexing backend means index data will be lost.
+This leads to "Fulltext" queries not working on the existing data
+For clearing the data for BerkeleyDB, delete the ATLAS_HOME/data/berkeley directory
+For clearing the data for HBase, in Hbase shell, run 'disable titan' and 'drop titan'
----+++ Hive Lineage Configs
-The higher layer services like hive lineage, schema, etc. are driven by the type system and this
-section encodes the specific types for the hive data model.
+---++ Lineage Configs
+
+The higher layer services like lineage, schema, etc. are driven by the type system and this section encodes the specific types for the hive data model.
# This models reflects the base super types for Data and Process
<verbatim>
@@ -128,7 +121,8 @@ atlas.lineage.hive.process.outputs.name=outputs
atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
</verbatim>
----+++ Notification Configs
+
+---++ Notification Configs
Refer http://kafka.apache.org/documentation.html#configuration for Kafka configuration. All Kafka configs should be prefixed with 'atlas.kafka.'
<verbatim>
@@ -149,16 +143,17 @@ atlas.kafka.entities.group.id=<consumer id>
</verbatim>
----+++ Client Configs
+---++ Client Configs
<verbatim>
atlas.client.readTimeoutMSecs=60000
atlas.client.connectTimeoutMSecs=60000
+atlas.rest.address=<http/https>://<atlas-fqdn>:<atlas port> - default http://localhost:21000
</verbatim>
----+++ Security Properties
+---++ Security Properties
----++++ SSL config
+---+++ SSL config
The following property is used to toggle the SSL feature.
<verbatim>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b82ff048/docs/src/site/twiki/InstallationSteps.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki
index 0eefb20..02a62f5 100644
--- a/docs/src/site/twiki/InstallationSteps.twiki
+++ b/docs/src/site/twiki/InstallationSteps.twiki
@@ -184,6 +184,9 @@ with multiple atlas upgrades), set environment variable METADATA_CONF to the pat
*Using Atlas*
<verbatim>
+* Quick start model - sample model and data
+ bin/quick_start.py [<atlas endpoint>]
+
* Verify if the server is up and running
curl -v http://localhost:21000/api/atlas/admin/version
{"Version":"v0.1"}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b82ff048/docs/src/site/twiki/security.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/security.twiki b/docs/src/site/twiki/security.twiki
index e360004..79c81a5 100755
--- a/docs/src/site/twiki/security.twiki
+++ b/docs/src/site/twiki/security.twiki
@@ -80,7 +80,8 @@ The properties for configuring the SPNEGO support are:
* <code>atlas.http.authentication.type</code> (simple|kerberos) [default: simple] - the authentication type
* <code>atlas.http.authentication.kerberos.principal</code> - the web-application Kerberos principal name. The Kerberos principal name must start with "HTTP/...". For example: "HTTP/localhost@LOCALHOST". There is no default value.
* <code>atlas.http.authentication.kerberos.keytab</code> - the path to the keytab file containing the credentials for the kerberos principal.
-
+ * <code>atlas.rest.address</code> - <http/https>://<atlas-fqdn>:<atlas port>
+
For a more detailed discussion of the HTTP authentication mechanism refer to [[http://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html][Hadoop Auth, Java HTTP SPNEGO 2.6.0 - Server Side Configuration]]. The prefix that document references is "atlas.http.authentication" in the case of the Atlas authentication implementation.
---+++ Client security configuration
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b82ff048/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 1dae6fb..ab6ae56 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -14,6 +14,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
ALL CHANGES:
+ATLAS-354 Kerberized cluster: quick_start.py fails to add sample data (shwethags)
ATLAS-47 Entity mutations for complex types (sumasai via shwethags)
ATLAS-345 UI: Should allow tag addition on any search result that returns a reference-able entity (darshankumar89 via shwethags)
ATLAS-279 UI not displaying results for certain successful "select" search queries (anilsg via shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/b82ff048/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
index 1251062..1232f8f 100755
--- a/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
+++ b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
@@ -20,7 +20,9 @@ package org.apache.atlas.examples;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
+import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
@@ -36,6 +38,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.commons.configuration.Configuration;
import org.codehaus.jettison.json.JSONArray;
import java.util.List;
@@ -46,6 +49,7 @@ import java.util.List;
* todo - move this to examples module.
*/
public class QuickStart {
+ public static final String ATLAS_REST_ADDRESS = "atlas.rest.address";
public static void main(String[] args) throws Exception {
String baseUrl = getServerUrl(args);
@@ -61,10 +65,16 @@ public class QuickStart {
quickStart.search();
}
- static String getServerUrl(String[] args) {
- String baseUrl = "http://localhost:21000";
+ static String getServerUrl(String[] args) throws AtlasException {
if (args.length > 0) {
- baseUrl = args[0];
+ return args[0];
+ }
+
+ Configuration configuration = ApplicationProperties.get();
+ String baseUrl = configuration.getString(ATLAS_REST_ADDRESS);
+ if (baseUrl == null) {
+ System.out.println("Usage: quick_start.py <atlas endpoint of format <http/https>://<atlas-fqdn>:<atlas port> like http://localhost:21000>");
+ System.exit(-1);
}
return baseUrl;