You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/07/08 06:03:12 UTC
[1/3] incubator-atlas git commit: ATLAS-987 Atlas hooks should avoid
adding dependent libraries to component CLASSPATH (madhan.neethiraj via
shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master b9ce7a111 -> 1ee2c1bc4
ATLAS-987 Atlas hooks should avoid adding dependent libraries to component CLASSPATH (madhan.neethiraj via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/043d2aa6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/043d2aa6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/043d2aa6
Branch: refs/heads/master
Commit: 043d2aa642e695c7c61c0b771d978fc352caf203
Parents: b9ce7a1
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Fri Jul 8 11:31:10 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Fri Jul 8 11:31:10 2016 +0530
----------------------------------------------------------------------
addons/falcon-bridge-shim/pom.xml | 52 +++
.../atlas/falcon/service/AtlasService.java | 215 ++++++++++++
addons/falcon-bridge/pom.xml | 44 ++-
addons/hive-bridge-shim/pom.xml | 53 +++
.../org/apache/atlas/hive/hook/HiveHook.java | 99 ++++++
addons/hive-bridge/pom.xml | 60 ++--
addons/sqoop-bridge-shim/pom.xml | 52 +++
.../org/apache/atlas/sqoop/hook/SqoopHook.java | 98 ++++++
addons/sqoop-bridge/pom.xml | 61 +++-
addons/storm-bridge-shim/pom.xml | 58 ++++
.../apache/atlas/storm/hook/StormAtlasHook.java | 104 ++++++
addons/storm-bridge/pom.xml | 44 ++-
plugin-classloader/pom.xml | 42 +++
.../classloader/AtlasPluginClassLoader.java | 327 +++++++++++++++++++
.../classloader/AtlasPluginClassLoaderUtil.java | 115 +++++++
.../classloader/AtlasPluginClassLoaderTest.java | 66 ++++
pom.xml | 40 ++-
release-log.txt | 1 +
18 files changed, 1489 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/falcon-bridge-shim/pom.xml
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge-shim/pom.xml b/addons/falcon-bridge-shim/pom.xml
new file mode 100755
index 0000000..ddb62ce
--- /dev/null
+++ b/addons/falcon-bridge-shim/pom.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>apache-atlas</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <version>0.8-incubating-SNAPSHOT</version>
+ <relativePath>../../</relativePath>
+ </parent>
+ <artifactId>falcon-bridge-shim</artifactId>
+ <description>Apache Atlas Falcon Bridge Shim Module</description>
+ <name>Apache Atlas Falcon Bridge Shim</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <falcon.version>0.8</falcon.version>
+ <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
+ </properties>
+
+ <dependencies>
+ <!-- Logging -->
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-plugin-classloader</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.falcon</groupId>
+ <artifactId>falcon-common</artifactId>
+ <version>${falcon.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/falcon-bridge-shim/src/main/java/org/apache/atlas/falcon/service/AtlasService.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge-shim/src/main/java/org/apache/atlas/falcon/service/AtlasService.java b/addons/falcon-bridge-shim/src/main/java/org/apache/atlas/falcon/service/AtlasService.java
new file mode 100755
index 0000000..28607fc
--- /dev/null
+++ b/addons/falcon-bridge-shim/src/main/java/org/apache/atlas/falcon/service/AtlasService.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.falcon.service;
+
+
+import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader;
+import org.apache.falcon.FalconException;
+import org.apache.falcon.entity.v0.Entity;
+import org.apache.falcon.service.ConfigurationChangeListener;
+import org.apache.falcon.service.FalconService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Falcon hook used for atlas entity registration.
+ */
+public class AtlasService implements FalconService, ConfigurationChangeListener {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasService.class);
+
+ private static final String ATLAS_PLUGIN_TYPE = "falcon";
+ private static final String ATLAS_FALCON_HOOK_IMPL_CLASSNAME = "org.apache.atlas.falcon.hook.AtlasService";
+
+ private AtlasPluginClassLoader atlasPluginClassLoader = null;
+ private FalconService falconServiceImpl = null;
+ private ConfigurationChangeListener configChangeListenerImpl = null;
+
+ public AtlasService() {
+ this.initialize();
+ }
+
+ @Override
+ public String getName() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasService.getName()");
+ }
+
+ String ret = null;
+
+ try {
+ activatePluginClassLoader();
+ ret = falconServiceImpl.getName();
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasService.getName()");
+ }
+
+ return ret;
+ }
+
+ @Override
+ public void init() throws FalconException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasService.init()");
+ }
+
+ try {
+ activatePluginClassLoader();
+ falconServiceImpl.init();
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasService.init()");
+ }
+ }
+
+ @Override
+ public void destroy() throws FalconException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasService.destroy()");
+ }
+
+ try {
+ activatePluginClassLoader();
+ falconServiceImpl.destroy();
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasService.destroy()");
+ }
+ }
+
+ @Override
+ public void onAdd(Entity entity) throws FalconException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasService.onAdd(" + entity + ")");
+ }
+
+ try {
+ activatePluginClassLoader();
+ configChangeListenerImpl.onAdd(entity);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasService.onAdd(" + entity + ")");
+ }
+ }
+
+ @Override
+ public void onRemove(Entity entity) throws FalconException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasService.onRemove(" + entity + ")");
+ }
+
+ try {
+ activatePluginClassLoader();
+ configChangeListenerImpl.onRemove(entity);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasService.onRemove(" + entity + ")");
+ }
+ }
+
+ @Override
+ public void onChange(Entity entity, Entity entity1) throws FalconException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasService.onChange(" + entity + ", " + entity1 + ")");
+ }
+
+ try {
+ activatePluginClassLoader();
+ configChangeListenerImpl.onChange(entity, entity1);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasService.onChange(" + entity + ", " + entity1 + ")");
+ }
+ }
+
+ @Override
+ public void onReload(Entity entity) throws FalconException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasService.onReload(" + entity + ")");
+ }
+
+ try {
+ activatePluginClassLoader();
+ configChangeListenerImpl.onReload(entity);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasService.onReload(" + entity + ")");
+ }
+ }
+
+ private void initialize() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasService.initialize()");
+ }
+
+ try {
+ atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass());
+
+ Class<?> cls = Class.forName(ATLAS_FALCON_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader);
+
+ activatePluginClassLoader();
+
+ Object atlasService = cls.newInstance();
+
+ falconServiceImpl = (FalconService) atlasService;
+ configChangeListenerImpl = (ConfigurationChangeListener) atlasService;
+ } catch (Exception excp) {
+ LOG.error("Error instantiating Atlas hook implementation", excp);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasService.initialize()");
+ }
+ }
+
+ private void activatePluginClassLoader() {
+ if (atlasPluginClassLoader != null) {
+ atlasPluginClassLoader.activate();
+ }
+ }
+
+ private void deactivatePluginClassLoader() {
+ if (atlasPluginClassLoader != null) {
+ atlasPluginClassLoader.deactivate();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/falcon-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml
index d79dda9..064fbf6 100644
--- a/addons/falcon-bridge/pom.xml
+++ b/addons/falcon-bridge/pom.xml
@@ -99,13 +99,13 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
- <id>copy-hook-dependencies</id>
+ <id>copy-hook</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
- <outputDirectory>${project.build.directory}/dependency/hook/falcon</outputDirectory>
+ <outputDirectory>${project.build.directory}/dependency/hook/falcon/atlas-falcon-plugin-impl</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
@@ -195,6 +195,46 @@
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>${log4j.version}</version>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-hook-shim</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/dependency/hook/falcon</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ <artifactItems>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>falcon-bridge-shim</artifactId>
+ <version>${project.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>atlas-plugin-classloader</artifactId>
+ <version>${project.version}</version>
+ </artifactItem>
</artifactItems>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/hive-bridge-shim/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge-shim/pom.xml b/addons/hive-bridge-shim/pom.xml
new file mode 100755
index 0000000..c699b29
--- /dev/null
+++ b/addons/hive-bridge-shim/pom.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>apache-atlas</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <version>0.8-incubating-SNAPSHOT</version>
+ <relativePath>../../</relativePath>
+ </parent>
+ <artifactId>hive-bridge-shim</artifactId>
+ <description>Apache Atlas Hive Bridge Shim Module</description>
+ <name>Apache Atlas Hive Bridge Shim</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <hive.version>1.2.1</hive.version>
+ <calcite.version>0.9.2-incubating</calcite.version>
+ <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
+ </properties>
+
+ <dependencies>
+ <!-- Logging -->
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-plugin-classloader</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/hive-bridge-shim/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge-shim/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge-shim/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
new file mode 100755
index 0000000..f4ca94f
--- /dev/null
+++ b/addons/hive-bridge-shim/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.hive.hook;
+
+
+import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader;
+import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
+import org.apache.hadoop.hive.ql.hooks.HookContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hive hook used for atlas entity registration.
+ */
+public class HiveHook implements ExecuteWithHookContext {
+ private static final Logger LOG = LoggerFactory.getLogger(HiveHook.class);
+
+ private static final String ATLAS_PLUGIN_TYPE = "hive";
+ private static final String ATLAS_HIVE_HOOK_IMPL_CLASSNAME = "org.apache.atlas.hive.hook.HiveHook";
+
+ private AtlasPluginClassLoader atlasPluginClassLoader = null;
+ private ExecuteWithHookContext hiveHookImpl = null;
+
+ public HiveHook() {
+ this.initialize();
+ }
+
+ @Override
+ public void run(final HookContext hookContext) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> HiveHook.run(" + hookContext + ")");
+ }
+
+ try {
+ activatePluginClassLoader();
+ hiveHookImpl.run(hookContext);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveHook.run(" + hookContext + ")");
+ }
+ }
+
+ private void initialize() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> HiveHook.initialize()");
+ }
+
+ try {
+ atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass());
+
+ @SuppressWarnings("unchecked")
+ Class<ExecuteWithHookContext> cls = (Class<ExecuteWithHookContext>) Class
+ .forName(ATLAS_HIVE_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader);
+
+ activatePluginClassLoader();
+
+ hiveHookImpl = cls.newInstance();
+ } catch (Exception excp) {
+ LOG.error("Error instantiating Atlas hook implementation", excp);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveHook.initialize()");
+ }
+ }
+
+ private void activatePluginClassLoader() {
+ if (atlasPluginClassLoader != null) {
+ atlasPluginClassLoader.activate();
+ }
+ }
+
+ private void deactivatePluginClassLoader() {
+ if (atlasPluginClassLoader != null) {
+ atlasPluginClassLoader.deactivate();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/hive-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml
index ddefdc2..2d0fb14 100755
--- a/addons/hive-bridge/pom.xml
+++ b/addons/hive-bridge/pom.xml
@@ -168,25 +168,7 @@
<goal>copy</goal>
</goals>
<configuration>
- <artifactItems>
- <artifactItem>
- <groupId>${project.groupId}</groupId>
- <artifactId>${project.artifactId}</artifactId>
- <version>${project.version}</version>
- <overWrite>true</overWrite>
- <outputDirectory>${project.build.directory}/dependency/hook/hive</outputDirectory>
- </artifactItem>
- </artifactItems>
- </configuration>
- </execution>
- <execution>
- <id>copy-hook-dependencies</id>
- <phase>package</phase>
- <goals>
- <goal>copy</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/dependency/hook/hive</outputDirectory>
+ <outputDirectory>${project.build.directory}/dependency/hook/hive/atlas-hive-plugin-impl</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
@@ -271,6 +253,46 @@
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>${log4j.version}</version>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-hook-shim</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/dependency/hook/hive</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ <artifactItems>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>hive-bridge-shim</artifactId>
+ <version>${project.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>atlas-plugin-classloader</artifactId>
+ <version>${project.version}</version>
+ </artifactItem>
</artifactItems>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/sqoop-bridge-shim/pom.xml
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge-shim/pom.xml b/addons/sqoop-bridge-shim/pom.xml
new file mode 100755
index 0000000..efb5a11
--- /dev/null
+++ b/addons/sqoop-bridge-shim/pom.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>apache-atlas</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <version>0.8-incubating-SNAPSHOT</version>
+ <relativePath>../../</relativePath>
+ </parent>
+ <artifactId>sqoop-bridge-shim</artifactId>
+ <description>Apache Atlas Sqoop Bridge Shim Module</description>
+ <name>Apache Atlas Sqoop Bridge Shim</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <sqoop.version>1.4.6.2.3.99.0-195</sqoop.version>
+ <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
+ </properties>
+
+ <dependencies>
+ <!-- Logging -->
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-plugin-classloader</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.sqoop</groupId>
+ <artifactId>sqoop</artifactId>
+ <version>${sqoop.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/sqoop-bridge-shim/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge-shim/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java b/addons/sqoop-bridge-shim/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
new file mode 100644
index 0000000..1170cb6
--- /dev/null
+++ b/addons/sqoop-bridge-shim/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.sqoop.hook;
+
+
+import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader;
+import org.apache.sqoop.SqoopJobDataPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Sqoop hook used for atlas entity registration.
+ */
+public class SqoopHook extends SqoopJobDataPublisher {
+ private static final Logger LOG = LoggerFactory.getLogger(SqoopHook.class);
+
+ private static final String ATLAS_PLUGIN_TYPE = "sqoop";
+ private static final String ATLAS_SQOOP_HOOK_IMPL_CLASSNAME = "org.apache.atlas.sqoop.hook.SqoopHook";
+
+ private AtlasPluginClassLoader atlasPluginClassLoader = null;
+ private SqoopJobDataPublisher sqoopHookImpl = null;
+
+ public SqoopHook() {
+ this.initialize();
+ }
+
+ @Override
+ public void publish(SqoopJobDataPublisher.Data data) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> SqoopHook.run(" + data + ")");
+ }
+
+ try {
+ activatePluginClassLoader();
+ sqoopHookImpl.publish(data);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== SqoopHook.run(" + data + ")");
+ }
+ }
+
+ private void initialize() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> HiveHook.initialize()");
+ }
+
+ try {
+ atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass());
+
+ @SuppressWarnings("unchecked")
+ Class<SqoopJobDataPublisher> cls = (Class<SqoopJobDataPublisher>) Class
+ .forName(ATLAS_SQOOP_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader);
+
+ activatePluginClassLoader();
+
+ sqoopHookImpl = cls.newInstance();
+ } catch (Exception excp) {
+ LOG.error("Error instantiating Atlas hook implementation", excp);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HiveHook.initialize()");
+ }
+ }
+
+ private void activatePluginClassLoader() {
+ if (atlasPluginClassLoader != null) {
+ atlasPluginClassLoader.activate();
+ }
+ }
+
+ private void deactivatePluginClassLoader() {
+ if (atlasPluginClassLoader != null) {
+ atlasPluginClassLoader.deactivate();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/sqoop-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/pom.xml b/addons/sqoop-bridge/pom.xml
index c792945..ceccf36 100644
--- a/addons/sqoop-bridge/pom.xml
+++ b/addons/sqoop-bridge/pom.xml
@@ -170,29 +170,16 @@
<goal>copy</goal>
</goals>
<configuration>
+ <outputDirectory>${project.build.directory}/dependency/hook/sqoop/atlas-sqoop-plugin-impl</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
- <overWrite>true</overWrite>
- <outputDirectory>${project.build.directory}/dependency/hook/sqoop</outputDirectory>
</artifactItem>
- </artifactItems>
- </configuration>
- </execution>
- <execution>
- <id>copy-hook-dependencies</id>
- <phase>package</phase>
- <goals>
- <goal>copy</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/dependency/hook/sqoop</outputDirectory>
- <overWriteReleases>false</overWriteReleases>
- <overWriteSnapshots>false</overWriteSnapshots>
- <overWriteIfNewer>true</overWriteIfNewer>
- <artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
@@ -273,6 +260,46 @@
<artifactId>kafka-clients</artifactId>
<version>${kafka.version}</version>
</artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>${log4j.version}</version>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-hook-shim</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/dependency/hook/sqoop</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ <artifactItems>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>sqoop-bridge-shim</artifactId>
+ <version>${project.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>atlas-plugin-classloader</artifactId>
+ <version>${project.version}</version>
+ </artifactItem>
</artifactItems>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/storm-bridge-shim/pom.xml
----------------------------------------------------------------------
diff --git a/addons/storm-bridge-shim/pom.xml b/addons/storm-bridge-shim/pom.xml
new file mode 100755
index 0000000..49c56e0
--- /dev/null
+++ b/addons/storm-bridge-shim/pom.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>apache-atlas</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <version>0.8-incubating-SNAPSHOT</version>
+ <relativePath>../../</relativePath>
+ </parent>
+ <artifactId>storm-bridge-shim</artifactId>
+ <description>Apache Atlas Storm Bridge Shim Module</description>
+ <name>Apache Atlas Storm Bridge Shim</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <storm.version>1.0.0</storm.version>
+ <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
+ </properties>
+
+ <dependencies>
+ <!-- Logging -->
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-plugin-classloader</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${storm.version}</version>
+ <type>jar</type>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/storm-bridge-shim/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
----------------------------------------------------------------------
diff --git a/addons/storm-bridge-shim/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java b/addons/storm-bridge-shim/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
new file mode 100644
index 0000000..2d304c5
--- /dev/null
+++ b/addons/storm-bridge-shim/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.storm.hook;
+
+
+import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader;
+import org.apache.storm.ISubmitterHook;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.TopologyInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * Storm hook used for atlas entity registration.
+ */
+public class StormAtlasHook implements ISubmitterHook {
+ private static final Logger LOG = LoggerFactory.getLogger(StormAtlasHook.class);
+
+ private static final String ATLAS_PLUGIN_TYPE = "storm";
+ private static final String ATLAS_STORM_HOOK_IMPL_CLASSNAME = "org.apache.atlas.storm.hook.StormAtlasHook";
+
+ private AtlasPluginClassLoader atlasPluginClassLoader = null;
+ private ISubmitterHook stormHook = null;
+
+
+ public StormAtlasHook() {
+ this.initialize();
+ }
+
+ @Override
+ public void notify(TopologyInfo topologyInfo, Map stormConf, StormTopology stormTopology)
+ throws IllegalAccessException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> StormAtlasHook.notify(" + topologyInfo + ", " + stormConf + ", " + stormTopology + ")");
+ }
+
+ try {
+ activatePluginClassLoader();
+ stormHook.notify(topologyInfo, stormConf, stormTopology);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== StormAtlasHook.notify(" + topologyInfo + ", " + stormConf + ", " + stormTopology + ")");
+ }
+ }
+
+ private void initialize() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> StormAtlasHook.initialize()");
+ }
+
+ try {
+ atlasPluginClassLoader = AtlasPluginClassLoader.getInstance(ATLAS_PLUGIN_TYPE, this.getClass());
+
+ @SuppressWarnings("unchecked")
+ Class<ISubmitterHook> cls = (Class<ISubmitterHook>) Class
+ .forName(ATLAS_STORM_HOOK_IMPL_CLASSNAME, true, atlasPluginClassLoader);
+
+ activatePluginClassLoader();
+
+ stormHook = cls.newInstance();
+ } catch (Exception excp) {
+ LOG.error("Error instantiating Atlas hook implementation", excp);
+ } finally {
+ deactivatePluginClassLoader();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== StormAtlasHook.initialize()");
+ }
+ }
+
+ private void activatePluginClassLoader() {
+ if (atlasPluginClassLoader != null) {
+ atlasPluginClassLoader.activate();
+ }
+ }
+
+ private void deactivatePluginClassLoader() {
+ if (atlasPluginClassLoader != null) {
+ atlasPluginClassLoader.deactivate();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/addons/storm-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml
index 9e8bf2f..12c1208 100644
--- a/addons/storm-bridge/pom.xml
+++ b/addons/storm-bridge/pom.xml
@@ -139,13 +139,13 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
- <id>copy-hook-dependencies</id>
+ <id>copy-hook</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
- <outputDirectory>${project.build.directory}/dependency/hook/storm</outputDirectory>
+ <outputDirectory>${project.build.directory}/dependency/hook/storm/atlas-storm-plugin-impl</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
@@ -315,6 +315,46 @@
<artifactId>hbase-common</artifactId>
<version>${hbase.version}</version>
</artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>${gson.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>${log4j.version}</version>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-hook-shim</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/dependency/hook/storm</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ <artifactItems>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>storm-bridge-shim</artifactId>
+ <version>${project.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>atlas-plugin-classloader</artifactId>
+ <version>${project.version}</version>
+ </artifactItem>
</artifactItems>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/plugin-classloader/pom.xml
----------------------------------------------------------------------
diff --git a/plugin-classloader/pom.xml b/plugin-classloader/pom.xml
new file mode 100644
index 0000000..c41b309
--- /dev/null
+++ b/plugin-classloader/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>apache-atlas</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <version>0.8-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>atlas-plugin-classloader</artifactId>
+ <description>Apache Atlas Plugin Classloader</description>
+ <name>Apache Atlas Plugin Classloader</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoader.java
----------------------------------------------------------------------
diff --git a/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoader.java b/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoader.java
new file mode 100644
index 0000000..c5214bd
--- /dev/null
+++ b/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoader.java
@@ -0,0 +1,327 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.plugin.classloader;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
+import java.util.Enumeration;
+
+/**
+ * AtlasPluginClassLoader to use plugin classpath first, before component classpath.
+ */
+public final class AtlasPluginClassLoader extends URLClassLoader {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasPluginClassLoader.class);
+
+ private final MyClassLoader componentClassLoader;
+
+ private AtlasPluginClassLoader(String pluginType, Class<?> pluginClass) throws Exception {
+ this(AtlasPluginClassLoaderUtil.getPluginImplLibPath(pluginType, pluginClass));
+ }
+
+ //visible for testing
+ AtlasPluginClassLoader(String libraryPath) throws Exception {
+ super(AtlasPluginClassLoaderUtil.getFilesInDirectories(new String[]{libraryPath}), null);
+
+ componentClassLoader = AccessController.doPrivileged(new PrivilegedAction<MyClassLoader>() {
+ public MyClassLoader run() {
+ return new MyClassLoader(Thread.currentThread().getContextClassLoader());
+ }
+ });
+ }
+
+ public static AtlasPluginClassLoader getInstance(final String pluginType, final Class<?> pluginClass)
+ throws Exception {
+ AtlasPluginClassLoader ret =
+ AccessController.doPrivileged(new PrivilegedExceptionAction<AtlasPluginClassLoader>() {
+ public AtlasPluginClassLoader run() throws Exception {
+ return new AtlasPluginClassLoader(pluginType, pluginClass);
+ }
+ });
+
+ return ret;
+ }
+
+ @Override
+ public Class<?> findClass(String name) throws ClassNotFoundException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoader.findClass(" + name + ")");
+ }
+
+ Class<?> ret = null;
+
+ try {
+ // first try to find the class in pluginClassloader
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasPluginClassLoader.findClass(" + name + "): calling pluginClassLoader.findClass()");
+ }
+
+ ret = super.findClass(name);
+ } catch (Throwable e) {
+ // on failure to find in pluginClassLoader, try to find in componentClassLoader
+ MyClassLoader savedClassLoader = getComponentClassLoader();
+
+ if (savedClassLoader != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "AtlasPluginClassLoader.findClass(" + name + "): calling componentClassLoader.findClass()");
+ }
+
+ ret = savedClassLoader.findClass(name);
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoader.findClass(" + name + "): " + ret);
+ }
+
+ return ret;
+ }
+
+ @Override
+ public Class<?> loadClass(String name) throws ClassNotFoundException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoader.loadClass(" + name + ")");
+ }
+
+ Class<?> ret = null;
+
+ try {
+ // first try to load the class from pluginClassloader
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasPluginClassLoader.loadClass(" + name + "): calling pluginClassLoader.loadClass()");
+ }
+
+ ret = super.loadClass(name);
+ } catch (Throwable e) {
+ // on failure to load from pluginClassLoader, try to load from componentClassLoader
+ MyClassLoader savedClassLoader = getComponentClassLoader();
+
+ if (savedClassLoader != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "AtlasPluginClassLoader.loadClass(" + name + "): calling componentClassLoader.loadClass()");
+ }
+
+ ret = savedClassLoader.loadClass(name);
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoader.loadClass(" + name + "): " + ret);
+ }
+
+ return ret;
+ }
+
+ @Override
+ public URL findResource(String name) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoader.findResource(" + name + ") ");
+ }
+
+ // first try to find the resource from pluginClassloader
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasPluginClassLoader.findResource(" + name + "): calling pluginClassLoader.findResource()");
+ }
+
+ URL ret = super.findResource(name);
+
+ if (ret == null) {
+ MyClassLoader savedClassLoader = getComponentClassLoader();
+
+ if (savedClassLoader != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasPluginClassLoader.findResource(" + name
+ + "): calling componentClassLoader.getResource()");
+ }
+
+ ret = savedClassLoader.getResource(name);
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoader.findResource(" + name + "): " + ret);
+ }
+
+ return ret;
+ }
+
+ @Override
+ public Enumeration<URL> findResources(String name) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoader.findResources(" + name + ")");
+ }
+
+ Enumeration<URL> ret = null;
+
+ Enumeration<URL> resourcesInPluginClsLoader = findResourcesUsingPluginClassLoader(name);
+ Enumeration<URL> resourcesInComponentClsLoader = findResourcesUsingComponentClassLoader(name);
+
+ if (resourcesInPluginClsLoader != null && resourcesInComponentClsLoader != null) {
+ ret = new MergeEnumeration(resourcesInPluginClsLoader, resourcesInComponentClsLoader);
+ } else if (resourcesInPluginClsLoader != null) {
+ ret = resourcesInPluginClsLoader;
+ } else {
+ ret = resourcesInComponentClsLoader;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoader.findResources(" + name + "): " + ret);
+ }
+
+ return ret;
+ }
+
+ public void activate() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoader.activate()");
+ }
+
+ Thread.currentThread().setContextClassLoader(this);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoader.activate()");
+ }
+ }
+
+ public void deactivate() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoader.deactivate()");
+ }
+
+ MyClassLoader savedClassLoader = getComponentClassLoader();
+
+ if (savedClassLoader != null && savedClassLoader.getParent() != null) {
+ Thread.currentThread().setContextClassLoader(savedClassLoader.getParent());
+ } else {
+ LOG.warn("AtlasPluginClassLoader.deactivate() was not successful.Couldn't not get the saved "
+ + "componentClassLoader...");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoader.deactivate()");
+ }
+ }
+
+ private MyClassLoader getComponentClassLoader() {
+ return componentClassLoader;
+ }
+
+ private Enumeration<URL> findResourcesUsingPluginClassLoader(String name) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoader.findResourcesUsingPluginClassLoader(" + name + ")");
+ }
+
+ Enumeration<URL> ret = null;
+
+ try {
+ ret = super.findResources(name);
+ } catch (Throwable excp) {
+ // Ignore exceptions
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasPluginClassLoader.findResourcesUsingPluginClassLoader(" + name
+ + "): resource not found in plugin", excp);
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoader.findResourcesUsingPluginClassLoader(" + name + "): " + ret);
+ }
+
+ return ret;
+ }
+
+ private Enumeration<URL> findResourcesUsingComponentClassLoader(String name) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoader.findResourcesUsingComponentClassLoader(" + name + ")");
+ }
+
+ Enumeration<URL> ret = null;
+
+ try {
+ MyClassLoader savedClassLoader = getComponentClassLoader();
+
+ if (savedClassLoader != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasPluginClassLoader.findResourcesUsingComponentClassLoader(" + name
+ + "): calling componentClassLoader.getResources()");
+ }
+
+ ret = savedClassLoader.getResources(name);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoader.findResourcesUsingComponentClassLoader(" + name + "): " + ret);
+ }
+ } catch (Throwable t) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasPluginClassLoader.findResourcesUsingComponentClassLoader(" + name
+ + "): class not found in componentClassLoader.", t);
+ }
+ }
+
+ return ret;
+ }
+
+ static class MergeEnumeration implements Enumeration<URL> { //NOPMD
+ private Enumeration<URL> e1 = null;
+ private Enumeration<URL> e2 = null;
+
+ public MergeEnumeration(Enumeration<URL> e1, Enumeration<URL> e2) {
+ this.e1 = e1;
+ this.e2 = e2;
+ }
+
+ @Override
+ public boolean hasMoreElements() {
+ return ((e1 != null && e1.hasMoreElements()) || (e2 != null && e2.hasMoreElements()));
+ }
+
+ @Override
+ public URL nextElement() {
+ URL ret = null;
+
+ if (e1 != null && e1.hasMoreElements()) {
+ ret = e1.nextElement();
+ } else if (e2 != null && e2.hasMoreElements()) {
+ ret = e2.nextElement();
+ }
+
+ return ret;
+ }
+ }
+
+ static class MyClassLoader extends ClassLoader {
+ public MyClassLoader(ClassLoader realClassLoader) {
+ super(realClassLoader);
+ }
+
+ @Override
+ public Class<?> findClass(String name) throws ClassNotFoundException { //NOPMD
+ return super.findClass(name);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderUtil.java
----------------------------------------------------------------------
diff --git a/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderUtil.java b/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderUtil.java
new file mode 100644
index 0000000..c3ec5e2
--- /dev/null
+++ b/plugin-classloader/src/main/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderUtil.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.plugin.classloader;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * AtlasPluginClassLoaderUtil used by AtlasPluginClassLoader.
+ */
+final class AtlasPluginClassLoaderUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasPluginClassLoaderUtil.class);
+
+ private static final String ATLAS_PLUGIN_LIBDIR = "atlas-%-plugin-impl";
+
+ private AtlasPluginClassLoaderUtil(){ }
+
+ public static URL[] getFilesInDirectories(String[] libDirs) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoaderUtil.getFilesInDirectories()");
+ }
+
+ List<URL> ret = new ArrayList<URL>();
+
+ for (String libDir : libDirs) {
+ getFilesInDirectory(libDir, ret);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoaderUtil.getFilesInDirectories(): " + ret.size() + " files");
+ }
+
+ return ret.toArray(new URL[]{});
+ }
+
+ private static void getFilesInDirectory(String dirPath, List<URL> files) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoaderUtil.getPluginFiles()");
+ }
+
+ if (dirPath != null) {
+ try {
+ File[] dirFiles = new File(dirPath).listFiles();
+
+ if (dirFiles != null) {
+ for (File dirFile : dirFiles) {
+ try {
+ URL jarPath = dirFile.toURI().toURL();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "getFilesInDirectory('" + dirPath + "'): adding " + dirFile.getAbsolutePath());
+ }
+
+ files.add(jarPath);
+ } catch (Exception excp) {
+ LOG.warn("getFilesInDirectory('" + dirPath + "'): failed to get URI for file " + dirFile
+ .getAbsolutePath(), excp);
+ }
+ }
+ }
+ } catch (Exception excp) {
+ LOG.warn("getFilesInDirectory('" + dirPath + "'): error", excp);
+ }
+ } else {
+ LOG.warn("getFilesInDirectory('" + dirPath + "'): could not find directory in path " + dirPath);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoaderUtil.getFilesInDirectory(" + dirPath + ")");
+ }
+ }
+
+ public static String getPluginImplLibPath(String pluginType, Class<?> pluginClass) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> AtlasPluginClassLoaderUtil.getPluginImplLibPath for Class (" + pluginClass.getName() + ")");
+ }
+
+ URI uri = pluginClass.getProtectionDomain().getCodeSource().getLocation().toURI();
+ Path path = Paths.get(URI.create(uri.toString()));
+ String ret = path.getParent().toString() + File.separatorChar + ATLAS_PLUGIN_LIBDIR.replaceAll("%", pluginType);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== AtlasPluginClassLoaderUtil.getPluginImplLibPath for Class " + pluginClass.getName() + "): "
+ + ret + ")");
+ }
+
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/plugin-classloader/src/test/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderTest.java
----------------------------------------------------------------------
diff --git a/plugin-classloader/src/test/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderTest.java b/plugin-classloader/src/test/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderTest.java
new file mode 100644
index 0000000..6139a00
--- /dev/null
+++ b/plugin-classloader/src/test/java/org/apache/atlas/plugin/classloader/AtlasPluginClassLoaderTest.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.plugin.classloader;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class AtlasPluginClassLoaderTest {
+
+ @Test
+ public void testClassLoader() throws Exception {
+ String cls = "org.apache.atlas.service.Services";
+
+ try {
+ loadClass(cls, null);
+ Assert.fail("Expected ClassNotFoundException");
+ } catch (ClassNotFoundException e) {
+ //expected
+ }
+
+ AtlasPluginClassLoader classLoader = new AtlasPluginClassLoader("../common/target");
+
+ classLoader.activate();
+
+ //org.apache.atlas.service.Services class should be loadable now
+ //should also load org.apache.atlas.service.Service
+ Class<?> servicesCls = loadClass(cls, null);
+ loadClass("org.apache.atlas.service.Service", servicesCls.getClassLoader());
+
+ //Fall back to current class loader should also work
+ loadClass(AtlasPluginClassLoaderUtil.class.getName(), null);
+
+ classLoader.deactivate();
+
+ //After disable, class loading should fail again
+ try {
+ loadClass(cls, null);
+ Assert.fail("Expected ClassNotFoundException");
+ } catch (ClassNotFoundException e) {
+ //expected
+ }
+ }
+
+ private Class<?> loadClass(String cls, ClassLoader classLoader) throws ClassNotFoundException {
+ if (classLoader == null) {
+ classLoader = Thread.currentThread().getContextClassLoader();
+ }
+ return Class.forName(cls, true, classLoader);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f119525..2224450 100755
--- a/pom.xml
+++ b/pom.xml
@@ -378,6 +378,7 @@
<log4j.version>1.2.17</log4j.version>
<akka.version>2.3.7</akka.version>
<spray.version>1.3.1</spray.version>
+ <gson.version>2.5</gson.version>
<guava.version>14.0</guava.version>
<fastutil.version>6.5.16</fastutil.version>
<guice.version>4.0</guice.version>
@@ -470,12 +471,17 @@
<module>catalog</module>
<module>dashboardv2</module>
<module>webapp</module>
-
<module>docs</module>
+
<module>addons/hdfs-model</module>
+ <module>plugin-classloader</module>
+ <module>addons/hive-bridge-shim</module>
<module>addons/hive-bridge</module>
+ <module>addons/falcon-bridge-shim</module>
<module>addons/falcon-bridge</module>
+ <module>addons/sqoop-bridge-shim</module>
<module>addons/sqoop-bridge</module>
+ <module>addons/storm-bridge-shim</module>
<module>addons/storm-bridge</module>
<module>distro</module>
@@ -1111,6 +1117,30 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>hive-bridge-shim</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>storm-bridge-shim</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>falcon-bridge-shim</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>sqoop-bridge-shim</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<!--Scala dependencies-->
<dependency>
<groupId>org.scala-lang</groupId>
@@ -1223,7 +1253,7 @@
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <version>2.5</version>
+ <version>${gson.version}</version>
</dependency>
<dependency>
@@ -1303,6 +1333,12 @@
</exclusion>
</exclusions>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-plugin-classloader</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/043d2aa6/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index d700596..8c9c1eb 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
+ATLAS-987 Atlas hooks should avoid adding dependent libraries to component CLASSPATH (madhan.neethiraj via shwethags)
ATLAS-993 If condition in DSL order by clause is not defined then dsl query fails (guptaneeru via shwethags)
ATLAS-968 Set group information from UGI for Ldap authentication (nixonrodrigues via shwethags)
ATLAS-584 Integrate CSRF prevention filter (kevalbhatt18 via shwethags)
[2/3] incubator-atlas git commit: ATLAS-902 Atlas throws exception
due to null definition in Hive create table statement (svimal2106 via
shwethags)
Posted by sh...@apache.org.
ATLAS-902 Atlas throws exception due to null definition in Hive create table statement (svimal2106 via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/7fc276f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/7fc276f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/7fc276f6
Branch: refs/heads/master
Commit: 7fc276f609280ea51b782426a72d979613ae6707
Parents: 043d2aa
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Fri Jul 8 11:32:13 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Fri Jul 8 11:32:13 2016 +0530
----------------------------------------------------------------------
.../org/apache/atlas/hive/hook/HiveHookIT.java | 9 ++++++++
release-log.txt | 1 +
.../atlas/typesystem/types/DataTypes.java | 8 ++++---
.../apache/atlas/typesystem/types/BaseTest.java | 15 ++++++++++++-
.../atlas/typesystem/types/StructTest.java | 23 ++++++++++++++++++++
.../typesystem/json/SerializationTest.scala | 8 +++----
6 files changed, 56 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7fc276f6/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 8ca47d9..e7ecbb1 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -368,6 +368,15 @@ public class HiveHookIT {
}
@Test
+ public void testEmptyStringAsValue() throws Exception{
+ String tableName = tableName();
+ String command = "create table " + tableName + "(id int, name string) row format delimited lines terminated by '\n' null defined as ''";
+ runCommand(command);
+ assertTableIsRegistered(DEFAULT_DB, tableName);
+
+ }
+
+ @Test
public void testDropAndRecreateCTASOutput() throws Exception {
String tableName = createTable();
String ctasTableName = "table" + random();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7fc276f6/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 8c9c1eb..00a7bf6 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
+ATLAS-902 Atlas throws exception due to null definition in Hive create table statement (svimal2106 via shwethags)
ATLAS-987 Atlas hooks should avoid adding dependent libraries to component CLASSPATH (madhan.neethiraj via shwethags)
ATLAS-993 If condition in DSL order by clause is not defined then dsl query fails (guptaneeru via shwethags)
ATLAS-968 Set group information from UGI for Ldap authentication (nixonrodrigues via shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7fc276f6/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
index 41b3427..3783c23 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
@@ -449,6 +449,10 @@ public class DataTypes {
if (val != null && (!(val instanceof String) || StringUtils.isNotEmpty((CharSequence) val))) {
return val.toString();
}
+
+ if (m.nullAllowed() && val != null){
+ return val.toString();
+ }
return convertNull(m);
}
@@ -584,9 +588,7 @@ public class DataTypes {
b.put(keyType.convert(e.getKey(),
TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
Multiplicity.REQUIRED),
- valueType.convert(e.getValue(),
- TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
- Multiplicity.REQUIRED));
+ valueType.convert(e.getValue(), Multiplicity.OPTIONAL));
}
return b.build();
} else {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7fc276f6/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
index 64dc751..95c99e4 100755
--- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
@@ -39,6 +39,7 @@ public abstract class BaseTest {
public static final String STRUCT_TYPE_1 = "t1";
public static final String STRUCT_TYPE_2 = "t2";
+ public static final String STRUCT_TYPE_3 = "t3";
public static final String TEST_DATE = "2014-12-11T02:35:58.440Z";
public static Struct createStruct() throws AtlasException {
@@ -62,6 +63,11 @@ public abstract class BaseTest {
hm.put("a", 1.0);
hm.put("b", 2.0);
s.set("o", hm);
+ s.set("p", "");
+ s.setNull("q");
+ Map<String, String> hm2 = Maps.newHashMap();
+ hm2.put("a", "");
+ s.set("r", hm2);
return s;
}
@@ -89,13 +95,20 @@ public abstract class BaseTest {
TypesUtil.createOptionalAttrDef("l", DataTypes.DATE_TYPE),
TypesUtil.createOptionalAttrDef("m", ts.defineArrayType(DataTypes.INT_TYPE)),
TypesUtil.createOptionalAttrDef("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- TypesUtil.createOptionalAttrDef("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)));
+ TypesUtil.createOptionalAttrDef("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+ TypesUtil.createOptionalAttrDef("p", DataTypes.STRING_TYPE),
+ TypesUtil.createOptionalAttrDef("q", DataTypes.STRING_TYPE),
+ TypesUtil.createOptionalAttrDef("r", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE)));
System.out.println("defined structType = " + structType);
StructType recursiveStructType =
ts.defineStructType(STRUCT_TYPE_2, true, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
TypesUtil.createOptionalAttrDef("s", STRUCT_TYPE_2));
System.out.println("defined recursiveStructType = " + recursiveStructType);
+
+ StructType invalidStructType =
+ ts.defineStructType(STRUCT_TYPE_3, true, TypesUtil.createRequiredAttrDef("a",DataTypes.STRING_TYPE));
+ System.out.println("defined structType = " + invalidStructType);
}
protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition<TraitType>... tDefs)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7fc276f6/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
index b49e5ce..e52962b 100755
--- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
@@ -19,6 +19,7 @@
package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList;
+import com.sun.source.tree.AssertTree;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Struct;
@@ -28,16 +29,21 @@ import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
public class StructTest extends TypeUpdateBaseTest {
StructType structType;
StructType recursiveStructType;
+ StructType invalidStructType;
@BeforeMethod
public void setup() throws Exception {
super.setup();
structType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_1);
recursiveStructType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_2);
+ invalidStructType = getTypeSystem().getDataType(StructType.class, STRUCT_TYPE_3);
}
@Test
@@ -60,10 +66,27 @@ public class StructTest extends TypeUpdateBaseTest {
"\tm : \t[1, 1]\n" +
"\tn : \t[1.1, 1.1]\n" +
"\to : \t{a=1.0, b=2.0}\n" +
+ "\tp : \t\n" +
+ "\tq : \t<null>\n"+
+ "\tr : \t{a=}\n" +
"}");
}
@Test
+ public void testStructWithEmptyString() throws AtlasException{
+ try {
+ assertTrue(getTypeSystem().getTypeNames().contains("t3"));
+ Struct s = new Struct(invalidStructType.getName());
+ s.set("a", "");
+ ITypedStruct ts = invalidStructType.convert(s, Multiplicity.REQUIRED);
+ }
+ catch (AtlasException e){
+ String err = "org.apache.atlas.typesystem.types.ValueConversionException: Cannot convert value 'org.apache.atlas.typesystem.Struct@1ba02' to datatype t3";
+ Assert.assertEquals(e.toString(), err);
+ }
+ }
+
+ @Test
public void testRecursive() throws AtlasException {
Struct s1 = new Struct(recursiveStructType.getName());
s1.set("a", 1);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7fc276f6/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
index 92489eb..931773d 100755
--- a/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
+++ b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/SerializationTest.scala
@@ -48,7 +48,7 @@ class SerializationTest extends BaseTest {
val s: Struct = BaseTest.createStruct()
val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
- Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{a=1.0, b=2.0}\n}")
+ Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{a=1.0, b=2.0}\n\tp : \t\n\tq : \t<null>\n\tr : \t{a=}\n}")
implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
new BigDecimalSerializer + new BigIntegerSerializer
@@ -56,10 +56,10 @@ class SerializationTest extends BaseTest {
//Json representation
val ser = swrite(ts)
val ser1 = swrite(ts.toString)
- Assert.assertEquals(ser1, "\"{\\n\\ta : \\t1\\n\\tb : \\ttrue\\n\\tc : \\t1\\n\\td : \\t2\\n\\te : \\t1\\n\\tf : \\t1\\n\\tg : \\t1\\n\\th : \\t1.0\\n\\ti : \\t1.0\\n\\tj : \\t1\\n\\tk : \\t1\\n\\tl : \\t" + BaseTest.TEST_DATE + "\\n\\tm : \\t[1, 1]\\n\\tn : \\t[1.1, 1.1]\\n\\to : \\t{a=1.0, b=2.0}\\n}\"");
+ Assert.assertEquals(ser1, "\"{\\n\\ta : \\t1\\n\\tb : \\ttrue\\n\\tc : \\t1\\n\\td : \\t2\\n\\te : \\t1\\n\\tf : \\t1\\n\\tg : \\t1\\n\\th : \\t1.0\\n\\ti : \\t1.0\\n\\tj : \\t1\\n\\tk : \\t1\\n\\tl : \\t" + BaseTest.TEST_DATE + "\\n\\tm : \\t[1, 1]\\n\\tn : \\t[1.1, 1.1]\\n\\to : \\t{a=1.0, b=2.0}\\n\\tp : \\t\\n\\tq : \\t<null>\\n\\tr : \\t{a=}\\n}\"");
// Typed Struct read back
val ts1 = read[StructInstance](ser)
- Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n}")
+ Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n\tp : \t\n\tq : \t<null>\n\tr : \t{a=}\n}")
}
@Test def test2 {
@@ -74,7 +74,7 @@ class SerializationTest extends BaseTest {
{"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
"c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"a":1.0,"b":2.0}}""")
// Typed Struct read from string
- Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n}")
+ Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{a=1.0, b=2.0}\n\tp : \t<null>\n\tq : \t<null>\n\tr : \t<null>\n}")
}
@Test def testTrait {
[3/3] incubator-atlas git commit: ATLAS-995 Atlas to setup ldap
authentication type as either LDAP / AD or None (nixonrodrigues via
shwethags)
Posted by sh...@apache.org.
ATLAS-995 Atlas to setup ldap authentication type as either LDAP / AD or None (nixonrodrigues via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/1ee2c1bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/1ee2c1bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/1ee2c1bc
Branch: refs/heads/master
Commit: 1ee2c1bc45599a19f6a9546ca2718366f46f47b5
Parents: 7fc276f
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Fri Jul 8 11:33:02 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Fri Jul 8 11:33:02 2016 +0530
----------------------------------------------------------------------
distro/src/conf/atlas-application.properties | 3 +-
release-log.txt | 1 +
.../security/AtlasAuthenticationProvider.java | 37 ++++++++------------
3 files changed, 17 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1ee2c1bc/distro/src/conf/atlas-application.properties
----------------------------------------------------------------------
diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties
index e50d6b9..79723f4 100755
--- a/distro/src/conf/atlas-application.properties
+++ b/distro/src/conf/atlas-application.properties
@@ -92,11 +92,10 @@ atlas.enableTLS=false
# Authentication config
atlas.authentication.method.kerberos=false
-atlas.authentication.method.ldap=false
atlas.authentication.method.file=true
#### ldap.type= LDAP or AD
-atlas.authentication.method.ldap.type=LDAP
+atlas.authentication.method.ldap.type=none
#### user credentials file
atlas.authentication.method.file.filename=${sys:atlas.home}/conf/users-credentials.properties
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1ee2c1bc/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 00a7bf6..b113f41 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
+ATLAS-995 Atlas to setup ldap authentication type as either LDAP / AD or None (nixonrodrigues via shwethags)
ATLAS-902 Atlas throws exception due to null definition in Hive create table statement (svimal2106 via shwethags)
ATLAS-987 Atlas hooks should avoid adding dependent libraries to component CLASSPATH (madhan.neethiraj via shwethags)
ATLAS-993 If condition in DSL order by clause is not defined then dsl query fails (guptaneeru via shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1ee2c1bc/webapp/src/main/java/org/apache/atlas/web/security/AtlasAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/security/AtlasAuthenticationProvider.java b/webapp/src/main/java/org/apache/atlas/web/security/AtlasAuthenticationProvider.java
index 7780191..b0fe6aa 100644
--- a/webapp/src/main/java/org/apache/atlas/web/security/AtlasAuthenticationProvider.java
+++ b/webapp/src/main/java/org/apache/atlas/web/security/AtlasAuthenticationProvider.java
@@ -35,10 +35,8 @@ public class AtlasAuthenticationProvider extends
.getLogger(AtlasAuthenticationProvider.class);
private boolean fileAuthenticationMethodEnabled = true;
- private boolean ldapAuthenticationMethodEnabled = false;
- private String ldapType = "UNKNOWN";
+ private String ldapType = "NONE";
public static final String FILE_AUTH_METHOD = "atlas.authentication.method.file";
- public static final String LDAP_AUTH_METHOD = "atlas.authentication.method.ldap";
public static final String LDAP_TYPE = "atlas.authentication.method.ldap.type";
@Autowired
@@ -57,9 +55,7 @@ public class AtlasAuthenticationProvider extends
this.fileAuthenticationMethodEnabled = configuration.getBoolean(
FILE_AUTH_METHOD, true);
- this.ldapAuthenticationMethodEnabled = configuration.getBoolean(
- LDAP_AUTH_METHOD, false);
- this.ldapType = configuration.getString(LDAP_TYPE, "UNKNOWN");
+ this.ldapType = configuration.getString(LDAP_TYPE, "NONE");
} catch (Exception e) {
LOG.error(
"Error while getting atlas.login.method application properties",
@@ -71,22 +67,19 @@ public class AtlasAuthenticationProvider extends
public Authentication authenticate(Authentication authentication)
throws AuthenticationException {
- if (ldapAuthenticationMethodEnabled) {
-
- if (ldapType.equalsIgnoreCase("LDAP")) {
- try {
- authentication = ldapAuthenticationProvider
- .authenticate(authentication);
- } catch (Exception ex) {
- LOG.error("Error while LDAP authentication", ex);
- }
- } else if (ldapType.equalsIgnoreCase("AD")) {
- try {
- authentication = adAuthenticationProvider
- .authenticate(authentication);
- } catch (Exception ex) {
- LOG.error("Error while AD authentication", ex);
- }
+ if (ldapType.equalsIgnoreCase("LDAP")) {
+ try {
+ authentication = ldapAuthenticationProvider
+ .authenticate(authentication);
+ } catch (Exception ex) {
+ LOG.error("Error while LDAP authentication", ex);
+ }
+ } else if (ldapType.equalsIgnoreCase("AD")) {
+ try {
+ authentication = adAuthenticationProvider
+ .authenticate(authentication);
+ } catch (Exception ex) {
+ LOG.error("Error while AD authentication", ex);
}
}