You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2018/01/12 19:18:32 UTC
atlas git commit: ATLAS-2357: Fix IT failures in hive-bridge module
Repository: atlas
Updated Branches:
refs/heads/master f28d0f54d -> 8c9d3da2c
ATLAS-2357: Fix IT failures in hive-bridge module
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8c9d3da2
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8c9d3da2
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8c9d3da2
Branch: refs/heads/master
Commit: 8c9d3da2cecccd043dd8257c4102b8b147e4f6b7
Parents: f28d0f5
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Fri Jan 12 11:18:06 2018 -0800
Committer: Sarath Subramanian <ss...@hortonworks.com>
Committed: Fri Jan 12 11:18:06 2018 -0800
----------------------------------------------------------------------
addons/hive-bridge/pom.xml | 49 ++++++-
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 3 +-
.../java/org/apache/atlas/hive/HiveITBase.java | 9 +-
.../org/apache/atlas/hive/hook/HiveHookIT.java | 9 +-
.../test/resources/atlas-application.properties | 135 +++++++++++++++++
.../src/test/resources/atlas-log4j.xml | 137 +++++++++++++++++
.../src/test/resources/hive-site.xml | 4 +-
.../src/test/resources/policy-store.txt | 9 ++
.../test/resources/users-credentials.properties | 3 +
pom.xml | 23 +--
.../apache/atlas/runner/LocalSolrRunner.java | 2 +-
webapp/pom.xml | 31 +++-
.../test/resources/atlas-application.properties | 146 +++++++++++++++++++
webapp/src/test/resources/policy-store.txt | 9 ++
.../test/resources/users-credentials.properties | 3 +
15 files changed, 528 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml
index 9421e32..8f7024b 100755
--- a/addons/hive-bridge/pom.xml
+++ b/addons/hive-bridge/pom.xml
@@ -346,16 +346,16 @@
</httpConnector>
<war>../../webapp/target/atlas-webapp-${project.version}.war</war>
<daemon>true</daemon>
+ <webAppSourceDirectory>../../webapp/src/test/webapp</webAppSourceDirectory>
<webApp>
<contextPath>/</contextPath>
<descriptor>${project.basedir}/../../webapp/src/test/webapp/WEB-INF/web.xml</descriptor>
- <extraClasspath>${project.basedir}/../../webapp/target/test-classes/</extraClasspath>
</webApp>
<useTestScope>true</useTestScope>
<systemProperties>
<systemProperty>
<name>log4j.configuration</name>
- <value>file://${project.basedir}/../../distro/src/conf/atlas-log4j.xml</value>
+ <value>${project.basedir}/target/test-classes/atlas-log4j.xml</value>
</systemProperty>
<systemProperty>
<name>atlas.log.file</name>
@@ -363,18 +363,35 @@
</systemProperty>
<systemProperty>
<name>atlas.log.dir</name>
- <value>${project.build.directory}/logs</value>
+ <value>${project.basedir}/target/logs</value>
+ </systemProperty>
+ <systemProperty>
+ <name>atlas.graphdb.backend</name>
+ <value>${graphdb.backend.impl}</value>
+ </systemProperty>
+ <systemProperty>
+ <name>keystore.file</name>
+ <value>${project.build.directory}/atlas.keystore
+ </value>
+ </systemProperty>
+ <systemProperty>
+ <name>truststore.file</name>
+ <value>${project.build.directory}/atlas.keystore</value>
+ </systemProperty>
+ <systemProperty>
+ <name>atlas.home</name>
+ <value>${project.basedir}/target</value>
</systemProperty>
<systemProperty>
<name>atlas.data</name>
- <value>${project.build.directory}/data</value>
+ <value>${project.basedir}/target/data</value>
</systemProperty>
<systemProperty>
<key>atlas.conf</key>
- <value>${project.build.directory}/../../../intg/target/test-classes</value>
+ <value>${project.basedir}/target/test-classes</value>
</systemProperty>
<systemProperty>
- <key>atlas.home</key>
+ <key>embedded.solr.directory</key>
<value>${project.basedir}/target</value>
</systemProperty>
</systemProperties>
@@ -458,6 +475,26 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-solr-resources</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/solr</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${basedir}/../../test-tools/src/main/resources/solr</directory>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index 2d2a9bf..7cca48f 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -94,6 +94,7 @@ public class HiveMetaStoreBridge {
public static final String CREATE_TIME = "createTime";
public static final String LAST_ACCESS_TIME = "lastAccessTime";
public static final String HDFS_PATH = "hdfs_path";
+ public static final String SEP = ":".intern();
private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class);
@@ -409,7 +410,7 @@ public class HiveMetaStoreBridge {
public static String getTableProcessQualifiedName(String clusterName, Table table) {
String tableQualifiedName = getTableQualifiedName(clusterName, table);
Date createdTime = getTableCreatedTime(table);
- return tableQualifiedName + HiveHook.SEP + createdTime.getTime();
+ return tableQualifiedName + SEP + createdTime.getTime();
}
/**
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
index b4ea3f9..01418cf 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
@@ -127,6 +127,13 @@ public class HiveITBase {
return "pfile://" + mkdir(path);
}
+ protected String file(String tag) throws Exception {
+ String filename = System.getProperty("user.dir") + "/target/" + tag + "-data-" + random();
+ File file = new File(filename);
+ file.createNewFile();
+ return file.getAbsolutePath();
+ }
+
protected String mkdir(String tag) throws Exception {
String filename = "./target/" + tag + "-data-" + random();
File file = new File(filename);
@@ -155,7 +162,7 @@ public class HiveITBase {
protected String assertEntityIsRegistered(final String typeName, final String property, final String value,
final HiveHookIT.AssertPredicate assertPredicate) throws Exception {
- waitFor(2000, new HiveHookIT.Predicate() {
+ waitFor(80000, new HiveHookIT.Predicate() {
@Override
public void evaluate() throws Exception {
Referenceable entity = atlasClient.getEntity(typeName, property, value);
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 262f7dc..1b1ebec 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -720,13 +720,6 @@ public class HiveHookIT extends HiveITBase {
//TODO -Add update test case
}
- private String file(String tag) throws Exception {
- String filename = "./target/" + tag + "-data-" + random();
- File file = new File(filename);
- file.createNewFile();
- return file.getAbsolutePath();
- }
-
@Test
public void testExportImportUnPartitionedTable() throws Exception {
String tableName = createTable(false);
@@ -1812,7 +1805,7 @@ public class HiveHookIT extends HiveITBase {
}
private void assertEntityIsNotRegistered(final String typeName, final String property, final String value) throws Exception {
- waitFor(1000, new Predicate() {
+ waitFor(80000, new Predicate() {
@Override
public void evaluate() throws Exception {
try {
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/resources/atlas-application.properties b/addons/hive-bridge/src/test/resources/atlas-application.properties
new file mode 100644
index 0000000..eff0aa4
--- /dev/null
+++ b/addons/hive-bridge/src/test/resources/atlas-application.properties
@@ -0,0 +1,135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#system property
+atlas.data=${sys:user.dir}/target/data
+
+
+
+#re-use existing property
+atlas.graph.data=${atlas.data}/graph
+
+#plain property
+atlas.service=atlas
+
+######### Atlas Server Configs #########
+atlas.rest.address=http://localhost:31000
+
+######### Graph Database Configs #########
+
+
+# Graph database implementation. Value inserted by maven.
+atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
+
+# Graph Storage
+atlas.graph.storage.backend=berkeleyje
+
+# Entity repository implementation
+atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.InMemoryEntityAuditRepository
+
+# Graph Search Index Backend
+atlas.graph.index.search.backend=solr
+
+#Berkeley storage directory
+atlas.graph.storage.directory=${sys:atlas.data}/berkley
+
+#hbase
+#For standalone mode , specify localhost
+#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
+
+atlas.graph.storage.hostname=${graph.storage.hostname}
+atlas.graph.storage.hbase.regions-per-server=1
+atlas.graph.storage.lock.wait-time=10000
+
+#ElasticSearch
+atlas.graph.index.search.directory=${sys:atlas.data}/es
+atlas.graph.index.search.elasticsearch.client-only=false
+atlas.graph.index.search.elasticsearch.local-mode=true
+atlas.graph.index.search.elasticsearch.create.sleep=2000
+
+# Solr cloud mode properties
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
+atlas.graph.index.search.solr.embedded=true
+atlas.graph.index.search.max-result-set-size=150
+
+######### Hive Lineage Configs #########
+## Schema
+atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns
+atlas.lineage.schema.query.hive_table_v1=hive_table_v1 where __guid='%s'\, columns
+
+######### Notification Configs #########
+atlas.notification.embedded=true
+
+atlas.kafka.zookeeper.connect=localhost:19026
+atlas.kafka.bootstrap.servers=localhost:19027
+atlas.kafka.data=${sys:atlas.data}/kafka
+atlas.kafka.zookeeper.session.timeout.ms=4000
+atlas.kafka.zookeeper.sync.time.ms=20
+atlas.kafka.consumer.timeout.ms=4000
+atlas.kafka.auto.commit.interval.ms=100
+atlas.kafka.hook.group.id=atlas
+atlas.kafka.entities.group.id=atlas_entities
+#atlas.kafka.auto.commit.enable=false
+
+atlas.kafka.enable.auto.commit=false
+atlas.kafka.auto.offset.reset=earliest
+atlas.kafka.session.timeout.ms=30000
+
+
+
+######### Entity Audit Configs #########
+atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS
+atlas.audit.zookeeper.session.timeout.ms=1000
+atlas.audit.hbase.zookeeper.quorum=localhost
+atlas.audit.hbase.zookeeper.property.clientPort=19026
+
+######### Security Properties #########
+
+# SSL config
+atlas.enableTLS=false
+atlas.server.https.port=31443
+
+######### Security Properties #########
+
+hbase.security.authentication=simple
+
+atlas.hook.falcon.synchronous=true
+
+######### JAAS Configuration ########
+
+atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+atlas.jaas.KafkaClient.loginModuleControlFlag = required
+atlas.jaas.KafkaClient.option.useKeyTab = true
+atlas.jaas.KafkaClient.option.storeKey = true
+atlas.jaas.KafkaClient.option.serviceName = kafka
+atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab
+atlas.jaas.KafkaClient.option.principal = atlas/_HOST@EXAMPLE.COM
+
+######### High Availability Configuration ########
+atlas.server.ha.enabled=false
+#atlas.server.ids=id1
+#atlas.server.address.id1=localhost:21000
+
+#########POLICY FILE PATH #########
+# atlas.auth.policy.file=policy-store.txt
+
+atlas.authentication.method.file=true
+atlas.authentication.method.ldap.type=none
+# atlas.authentication.method.file.filename=users-credentials.properties
+atlas.authentication.method.kerberos=false
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/resources/atlas-log4j.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/resources/atlas-log4j.xml b/addons/hive-bridge/src/test/resources/atlas-log4j.xml
new file mode 100755
index 0000000..a0f9629
--- /dev/null
+++ b/addons/hive-bridge/src/test/resources/atlas-log4j.xml
@@ -0,0 +1,137 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
+ </layout>
+ </appender>
+
+ <appender name="FILE" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${atlas.log.dir}/${atlas.log.file}"/>
+ <param name="Append" value="true"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
+ <param name="maxFileSize" value="100MB" />
+ <param name="maxBackupIndex" value="20" />
+ </layout>
+ </appender>
+
+ <appender name="AUDIT" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${atlas.log.dir}/audit.log"/>
+ <param name="Append" value="true"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %x %m%n"/>
+ <param name="maxFileSize" value="100MB" />
+ <param name="maxBackupIndex" value="20" />
+ </layout>
+ </appender>
+
+ <appender name="METRICS" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${atlas.log.dir}/metric.log"/>
+ <param name="Append" value="true"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %x %m%n"/>
+ <param name="maxFileSize" value="100MB" />
+ </layout>
+ </appender>
+
+ <appender name="FAILED" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${atlas.log.dir}/failed.log"/>
+ <param name="Append" value="true"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %m"/>
+ <param name="maxFileSize" value="100MB" />
+ <param name="maxBackupIndex" value="20" />
+ </layout>
+ </appender>
+
+ <!-- Uncomment the following for perf logs -->
+ <!--
+ <appender name="perf_appender" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="file" value="${atlas.log.dir}/atlas_perf.log" />
+ <param name="datePattern" value="'.'yyyy-MM-dd" />
+ <param name="append" value="true" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d|%t|%m%n" />
+ </layout>
+ </appender>
+
+ <logger name="org.apache.atlas.perf" additivity="false">
+ <level value="debug" />
+ <appender-ref ref="perf_appender" />
+ </logger>
+ -->
+
+ <logger name="org.apache.atlas" additivity="false">
+ <level value="info"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="com.thinkaurelius.titan" additivity="false">
+ <level value="warn"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="org.springframework" additivity="false">
+ <level value="warn"/>
+ <appender-ref ref="console"/>
+ </logger>
+
+ <logger name="org.eclipse" additivity="false">
+ <level value="warn"/>
+ <appender-ref ref="console"/>
+ </logger>
+
+ <logger name="com.sun.jersey" additivity="false">
+ <level value="warn"/>
+ <appender-ref ref="console"/>
+ </logger>
+
+ <!-- to avoid logs - The configuration log.flush.interval.messages = 1 was supplied but isn't a known config -->
+ <logger name="org.apache.kafka.common.config.AbstractConfig" additivity="false">
+ <level value="error"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="AUDIT" additivity="false">
+ <level value="info"/>
+ <appender-ref ref="AUDIT"/>
+ </logger>
+
+ <logger name="METRICS" additivity="false">
+ <level value="debug"/>
+ <appender-ref ref="METRICS"/>
+ </logger>
+
+ <logger name="FAILED" additivity="false">
+ <level value="info"/>
+ <appender-ref ref="AUDIT"/>
+ </logger>
+
+ <root>
+ <priority value="warn"/>
+ <appender-ref ref="FILE"/>
+ </root>
+
+</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/resources/hive-site.xml b/addons/hive-bridge/src/test/resources/hive-site.xml
index 876f41d..ff98668 100644
--- a/addons/hive-bridge/src/test/resources/hive-site.xml
+++ b/addons/hive-bridge/src/test/resources/hive-site.xml
@@ -43,12 +43,12 @@
<property>
<name>hive.metastore.warehouse.dir</name>
- <value>${user.dir}/target/metastore</value>
+ <value>${project.basedir}/target/metastore</value>
</property>
<property>
<name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:derby:${user.dir}/target/metastore_db;create=true</value>
+ <value>jdbc:derby:${project.basedir}/target/metastore_db;create=true</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/resources/policy-store.txt
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/resources/policy-store.txt b/addons/hive-bridge/src/test/resources/policy-store.txt
new file mode 100644
index 0000000..048affe
--- /dev/null
+++ b/addons/hive-bridge/src/test/resources/policy-store.txt
@@ -0,0 +1,9 @@
+##Policy Format
+##r-READ, w-WRITE, u-UPDATE, d-DELETE
+##Policy_Name;;User_Name1:Operations_Allowed,User_Name2:Operations_Allowed;;Group_Name1:Operations_Allowed,Group_Name2:Operations_Allowed;;Resource_Type1:Resource_Name,Resource_Type2:Resource_Name
+##
+adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:*
+dataScientistPolicy;;;;DATA_SCIENTIST:r;;type:*,entity:*
+dataStewardPolicy;;;;DATA_STEWARD:rwu;;type:*,entity:*
+hadoopPolicy;;;;hadoop:rwud;;type:*,entity:*,operation:*
+rangerTagSyncPolicy;;;;RANGER_TAG_SYNC:r;;type:*,entity:*
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/addons/hive-bridge/src/test/resources/users-credentials.properties
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/resources/users-credentials.properties b/addons/hive-bridge/src/test/resources/users-credentials.properties
new file mode 100644
index 0000000..3fc3bb1
--- /dev/null
+++ b/addons/hive-bridge/src/test/resources/users-credentials.properties
@@ -0,0 +1,3 @@
+#username=group::sha256-password
+admin=ADMIN::8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
+rangertagsync=RANGER_TAG_SYNC::e3f67240f5117d1753c940dae9eea772d36ed5fe9bd9c94a300e40413f1afb9d
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c64b26e..3f5c042 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1839,6 +1839,7 @@
<user.dir>${project.basedir}</user.dir>
<atlas.data>${project.build.directory}/data</atlas.data>
<log4j.configuration>atlas-log4j.xml</log4j.configuration>
+ <embedded.solr.directory>${project.basedir}/target</embedded.solr.directory>
</systemProperties>
<skipTests>${skipTests}</skipTests>
<forkCount>${surefire.forkCount}</forkCount>
@@ -1868,6 +1869,7 @@
<projectBaseDir>${projectBaseDir}</projectBaseDir>
<atlas.data>${project.build.directory}/data</atlas.data>
<log4j.configuration>atlas-log4j.xml</log4j.configuration>
+ <embedded.solr.directory>${project.basedir}/target</embedded.solr.directory>
</systemPropertyVariables>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<argLine>-Djava.awt.headless=true -Dproject.version=${project.version}
@@ -2125,27 +2127,6 @@
<skipAssembly>true</skipAssembly>
</configuration>
</plugin>
-
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>validate</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/solr</outputDirectory>
- <resources>
- <resource>
- <directory>${basedir}/test-tools/src/main/resources/solr</directory>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/test-tools/src/main/java/org/apache/atlas/runner/LocalSolrRunner.java
----------------------------------------------------------------------
diff --git a/test-tools/src/main/java/org/apache/atlas/runner/LocalSolrRunner.java b/test-tools/src/main/java/org/apache/atlas/runner/LocalSolrRunner.java
index cdcac64..ed8e9e1 100644
--- a/test-tools/src/main/java/org/apache/atlas/runner/LocalSolrRunner.java
+++ b/test-tools/src/main/java/org/apache/atlas/runner/LocalSolrRunner.java
@@ -37,7 +37,7 @@ import java.util.stream.Collectors;
public class LocalSolrRunner {
protected static final String[] COLLECTIONS = readCollections();
- private static final String TARGET_DIRECTORY = System.getProperty("user.dir") + File.separator + "target";
+ private static final String TARGET_DIRECTORY = System.getProperty("embedded.solr.directory");
private static final String COLLECTIONS_FILE = "collections.txt";
private static final String SOLR_XML = "solr.xml";
private static final String TEMPLATE_DIRECTORY = "core-template";
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 2cf6e10..30711db 100755
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -602,7 +602,7 @@
</systemProperty>
<systemProperty>
<name>atlas.log.dir</name>
- <value>${project.build.directory}/logs</value>
+ <value>${project.basedir}/target/logs</value>
</systemProperty>
<systemProperty>
<name>atlas.graphdb.backend</name>
@@ -619,15 +619,19 @@
</systemProperty>
<systemProperty>
<name>atlas.home</name>
- <value>${project.build.directory}</value>
+ <value>${project.basedir}/target</value>
</systemProperty>
<systemProperty>
<name>atlas.data</name>
- <value>${project.build.directory}/data</value>
+ <value>${project.basedir}/target/data</value>
</systemProperty>
<systemProperty>
<key>atlas.conf</key>
- <value>${project.build.directory}/../../intg/target/test-classes</value>
+ <value>${project.basedir}/target/test-classes</value>
+ </systemProperty>
+ <systemProperty>
+ <key>embedded.solr.directory</key>
+ <value>${project.basedir}/target</value>
</systemProperty>
</systemProperties>
<stopKey>atlas-stop</stopKey>
@@ -706,6 +710,25 @@
<skipEnunciate>${skipEnunciate}</skipEnunciate>
</configuration>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>post-integration-test</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <tasks>
+ <delete dir="${project.build.directory}/data"/>
+ <delete dir="${project.build.directory}/logs"/>
+ </tasks>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/webapp/src/test/resources/atlas-application.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/atlas-application.properties b/webapp/src/test/resources/atlas-application.properties
new file mode 100644
index 0000000..077a90f
--- /dev/null
+++ b/webapp/src/test/resources/atlas-application.properties
@@ -0,0 +1,146 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#system property
+atlas.data=${sys:user.dir}/target/data
+
+
+
+#re-use existing property
+atlas.graph.data=${atlas.data}/graph
+
+#plain property
+atlas.service=atlas
+
+#invalid system property
+atlas.db=${atlasdb}
+
+atlas.TypeSystem.impl=org.apache.atlas.typesystem.types.TypeSystem
+
+
+
+######### Atlas Server Configs #########
+atlas.rest.address=http://localhost:31000
+
+######### Graph Database Configs #########
+
+
+# Graph database implementation. Value inserted by maven.
+atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
+
+# Graph Storage
+atlas.graph.storage.backend=berkeleyje
+
+# Entity repository implementation
+atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.InMemoryEntityAuditRepository
+
+# Graph Search Index Backend
+atlas.graph.index.search.backend=solr
+
+#Berkeley storage directory
+atlas.graph.storage.directory=${sys:atlas.data}/berkley
+
+#hbase
+#For standalone mode , specify localhost
+#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
+
+atlas.graph.storage.hostname=${graph.storage.hostname}
+atlas.graph.storage.hbase.regions-per-server=1
+atlas.graph.storage.lock.wait-time=10000
+
+#ElasticSearch
+atlas.graph.index.search.directory=${sys:atlas.data}/es
+atlas.graph.index.search.elasticsearch.client-only=false
+atlas.graph.index.search.elasticsearch.local-mode=true
+atlas.graph.index.search.elasticsearch.create.sleep=2000
+
+# Solr cloud mode properties
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
+atlas.graph.index.search.solr.embedded=true
+atlas.graph.index.search.max-result-set-size=150
+
+######### Hive Lineage Configs #########
+## Schema
+atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns
+atlas.lineage.schema.query.hive_table_v1=hive_table_v1 where __guid='%s'\, columns
+
+######### Notification Configs #########
+atlas.notification.embedded=true
+
+atlas.kafka.zookeeper.connect=localhost:19026
+atlas.kafka.bootstrap.servers=localhost:19027
+atlas.kafka.data=${sys:atlas.data}/kafka
+atlas.kafka.zookeeper.session.timeout.ms=4000
+atlas.kafka.zookeeper.sync.time.ms=20
+atlas.kafka.consumer.timeout.ms=4000
+atlas.kafka.auto.commit.interval.ms=100
+atlas.kafka.hook.group.id=atlas
+atlas.kafka.entities.group.id=atlas_entities
+#atlas.kafka.auto.commit.enable=false
+
+atlas.kafka.enable.auto.commit=false
+atlas.kafka.auto.offset.reset=earliest
+atlas.kafka.session.timeout.ms=30000
+
+
+
+######### Entity Audit Configs #########
+atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS
+atlas.audit.zookeeper.session.timeout.ms=1000
+atlas.audit.hbase.zookeeper.quorum=localhost
+atlas.audit.hbase.zookeeper.property.clientPort=19026
+
+######### Security Properties #########
+
+# SSL config
+atlas.enableTLS=false
+atlas.server.https.port=31443
+
+######### Security Properties #########
+
+hbase.security.authentication=simple
+
+atlas.hook.falcon.synchronous=true
+
+######### JAAS Configuration ########
+
+atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+atlas.jaas.KafkaClient.loginModuleControlFlag = required
+atlas.jaas.KafkaClient.option.useKeyTab = true
+atlas.jaas.KafkaClient.option.storeKey = true
+atlas.jaas.KafkaClient.option.serviceName = kafka
+atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab
+atlas.jaas.KafkaClient.option.principal = atlas/_HOST@EXAMPLE.COM
+
+######### High Availability Configuration ########
+atlas.server.ha.enabled=false
+#atlas.server.ids=id1
+#atlas.server.address.id1=localhost:21000
+
+#########POLICY FILE PATH #########
+# atlas.auth.policy.file=policy-store.txt
+
+atlas.authentication.method.file=true
+atlas.authentication.method.ldap.type=none
+# atlas.authentication.method.file.filename=users-credentials.properties
+atlas.authentication.method.kerberos=false
+
+######### Gremlin Search Configuration #########
+# Set to false to disable gremlin search.
+atlas.search.gremlin.enable=true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/webapp/src/test/resources/policy-store.txt
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/policy-store.txt b/webapp/src/test/resources/policy-store.txt
new file mode 100644
index 0000000..048affe
--- /dev/null
+++ b/webapp/src/test/resources/policy-store.txt
@@ -0,0 +1,9 @@
+##Policy Format
+##r-READ, w-WRITE, u-UPDATE, d-DELETE
+##Policy_Name;;User_Name1:Operations_Allowed,User_Name2:Operations_Allowed;;Group_Name1:Operations_Allowed,Group_Name2:Operations_Allowed;;Resource_Type1:Resource_Name,Resource_Type2:Resource_Name
+##
+adminPolicy;;admin:rwud;;ROLE_ADMIN:rwud;;type:*,entity:*,operation:*
+dataScientistPolicy;;;;DATA_SCIENTIST:r;;type:*,entity:*
+dataStewardPolicy;;;;DATA_STEWARD:rwu;;type:*,entity:*
+hadoopPolicy;;;;hadoop:rwud;;type:*,entity:*,operation:*
+rangerTagSyncPolicy;;;;RANGER_TAG_SYNC:r;;type:*,entity:*
http://git-wip-us.apache.org/repos/asf/atlas/blob/8c9d3da2/webapp/src/test/resources/users-credentials.properties
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/users-credentials.properties b/webapp/src/test/resources/users-credentials.properties
new file mode 100644
index 0000000..3fc3bb1
--- /dev/null
+++ b/webapp/src/test/resources/users-credentials.properties
@@ -0,0 +1,3 @@
+#username=group::sha256-password
+admin=ADMIN::8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918
+rangertagsync=RANGER_TAG_SYNC::e3f67240f5117d1753c940dae9eea772d36ed5fe9bd9c94a300e40413f1afb9d