You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by su...@apache.org on 2016/07/19 00:33:45 UTC
incubator-atlas git commit: ATLAS-1030 Atlas hooks should use
properties from atlas-application.properties,
instead of component's configuration (mneethiraj via sumasai)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 3f47408e7 -> 038b6b315
ATLAS-1030 Atlas hooks should use properties from atlas-application.properties, instead of component's configuration (mneethiraj via sumasai)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/038b6b31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/038b6b31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/038b6b31
Branch: refs/heads/master
Commit: 038b6b315d8353e75fe3fd663ac034e025e541d2
Parents: 3f47408
Author: Suma Shivaprasad <su...@gmail.com>
Authored: Mon Jul 18 17:31:18 2016 -0700
Committer: Suma Shivaprasad <su...@gmail.com>
Committed: Mon Jul 18 17:32:54 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/atlas/falcon/hook/FalconHookIT.java | 2 +-
addons/falcon-bridge/src/test/resources/hive-site.xml | 5 -----
.../org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java | 10 +++++-----
.../main/java/org/apache/atlas/hive/hook/HiveHook.java | 2 +-
.../test/java/org/apache/atlas/hive/hook/HiveHookIT.java | 8 ++++----
addons/hive-bridge/src/test/resources/hive-site.xml | 5 -----
.../main/java/org/apache/atlas/sqoop/hook/SqoopHook.java | 3 +--
.../java/org/apache/atlas/sqoop/hook/SqoopHookIT.java | 2 +-
addons/sqoop-bridge/src/test/resources/hive-site.xml | 5 -----
.../java/org/apache/atlas/storm/hook/StormAtlasHook.java | 6 +-----
release-log.txt | 1 +
11 files changed, 15 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
index 0139bf5..f7184f9 100644
--- a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
+++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
@@ -89,7 +89,7 @@ public class FalconHookIT {
return;
}
- HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasClient);
+ HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
hiveMetaStoreBridge.registerHiveDataModel();
FalconDataModelGenerator dataModelGenerator = new FalconDataModelGenerator();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/falcon-bridge/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/test/resources/hive-site.xml b/addons/falcon-bridge/src/test/resources/hive-site.xml
index b106903..f058c2e 100644
--- a/addons/falcon-bridge/src/test/resources/hive-site.xml
+++ b/addons/falcon-bridge/src/test/resources/hive-site.xml
@@ -42,11 +42,6 @@
</property>
<property>
- <name>atlas.cluster.name</name>
- <value>test</value>
- </property>
-
- <property>
<name>fs.pfile.impl</name>
<value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index cd0e964..9a5e279 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -99,16 +99,16 @@ public class HiveMetaStoreBridge {
* Construct a HiveMetaStoreBridge.
* @param hiveConf {@link HiveConf} for Hive component in the cluster
*/
- public HiveMetaStoreBridge(HiveConf hiveConf) throws Exception {
- this(hiveConf.get(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), null);
+ public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf hiveConf) throws Exception {
+ this(atlasProperties, hiveConf, null);
}
/**
* Construct a HiveMetaStoreBridge.
* @param hiveConf {@link HiveConf} for Hive component in the cluster
*/
- public HiveMetaStoreBridge(HiveConf hiveConf, AtlasClient atlasClient) throws Exception {
- this(hiveConf.get(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClient);
+ public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf hiveConf, AtlasClient atlasClient) throws Exception {
+ this(atlasProperties.getString(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClient);
}
AtlasClient getAtlasClient() {
@@ -660,7 +660,7 @@ public class HiveMetaStoreBridge {
failOnError = true;
}
- HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasClient);
+ HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClient);
hiveMetaStoreBridge.registerHiveDataModel();
hiveMetaStoreBridge.importHiveMetadata(failOnError);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 99009ba..0e86333 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -200,7 +200,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
LOG.info("Entered Atlas hook for hook type {} operation {}", event.getHookType(), event.getOperation());
- HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(hiveConf);
+ HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(atlasProperties, hiveConf);
switch (event.getOperation()) {
case CREATEDATABASE:
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 734b606..a15aa49 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -87,7 +87,7 @@ public class HiveHookIT {
private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(HiveHookIT.class);
private static final String DGI_URL = "http://localhost:21000/";
- private static final String CLUSTER_NAME = "test";
+ private static final String CLUSTER_NAME = "primary";
public static final String DEFAULT_DB = "default";
private static final String PART_FILE = "2015-01-01";
@@ -115,7 +115,7 @@ public class HiveHookIT {
Configuration configuration = ApplicationProperties.get();
atlasClient = new AtlasClient(configuration.getString(HiveMetaStoreBridge.ATLAS_ENDPOINT, DGI_URL));
- hiveMetaStoreBridge = new HiveMetaStoreBridge(conf, atlasClient);
+ hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClient);
hiveMetaStoreBridge.registerHiveDataModel();
}
@@ -1624,7 +1624,7 @@ public class HiveHookIT {
verifyEntityProperties(entityType, entityName, expectedProps, false);
if (entityType != Entity.Type.DATABASE) {
- //Database unset properties doesnt work strangely - alter database %s unset DBPROPERTIES doesnt work
+ //Database unset properties doesnt work - alter database %s unset DBPROPERTIES doesnt work
//Unset all the props
StringBuilder sb = new StringBuilder("'");
query = String.format(fmtQuery, entityName, UNSET_OP, Joiner.on("','").skipNulls().appendTo(sb, expectedProps.keySet()).append('\''));
@@ -1928,7 +1928,7 @@ public class HiveHookIT {
fail("Assertions failed. Failing after waiting for timeout " + timeout + " msecs", e);
}
LOG.debug("Waiting up to " + (mustEnd - System.currentTimeMillis()) + " msec as assertion failed", e);
- Thread.sleep(400);
+ Thread.sleep(5000);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/hive-bridge/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/resources/hive-site.xml b/addons/hive-bridge/src/test/resources/hive-site.xml
index 3fe57df..876f41d 100644
--- a/addons/hive-bridge/src/test/resources/hive-site.xml
+++ b/addons/hive-bridge/src/test/resources/hive-site.xml
@@ -57,11 +57,6 @@
</property>
<property>
- <name>atlas.cluster.name</name>
- <value>test</value>
- </property>
-
- <property>
<name>fs.pfile.impl</name>
<value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
index c412658..e43fc86 100644
--- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
+++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
@@ -163,8 +163,7 @@ public class SqoopHook extends SqoopJobDataPublisher {
@Override
public void publish(SqoopJobDataPublisher.Data data) throws Exception {
Configuration atlasProperties = ApplicationProperties.get();
- org.apache.hadoop.conf.Configuration sqoopConf = new org.apache.hadoop.conf.Configuration();
- String clusterName = sqoopConf.get(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
+ String clusterName = atlasProperties.getString(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
Referenceable dbStoreRef = createDBStoreInstance(data);
Referenceable dbRef = createHiveDatabaseInstance(clusterName, data.getHiveDB());
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
index 72d9201..e965c7d 100644
--- a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
+++ b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
@@ -54,7 +54,7 @@ public class SqoopHookIT {
private void registerDataModels(AtlasClient client) throws Exception {
// Make sure hive model exists
- HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasClient);
+ HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
hiveMetaStoreBridge.registerHiveDataModel();
SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/sqoop-bridge/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/test/resources/hive-site.xml b/addons/sqoop-bridge/src/test/resources/hive-site.xml
index b106903..f058c2e 100644
--- a/addons/sqoop-bridge/src/test/resources/hive-site.xml
+++ b/addons/sqoop-bridge/src/test/resources/hive-site.xml
@@ -42,11 +42,6 @@
</property>
<property>
- <name>atlas.cluster.name</name>
- <value>test</value>
- </property>
-
- <property>
<name>fs.pfile.impl</name>
<value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
index 5bd5397..c4b4976 100644
--- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
+++ b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
@@ -370,10 +370,6 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
}
private String getClusterName(Map stormConf) {
- String clusterName = AtlasConstants.DEFAULT_CLUSTER_NAME;
- if (stormConf.containsKey(AtlasConstants.CLUSTER_NAME_KEY)) {
- clusterName = (String)stormConf.get(AtlasConstants.CLUSTER_NAME_KEY);
- }
- return clusterName;
+ return atlasProperties.getString(AtlasConstants.CLUSTER_NAME_KEY, AtlasConstants.DEFAULT_CLUSTER_NAME);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/038b6b31/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index f6128dc..eb7d458 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
+ATLAS-1027 Atlas hooks should use properties from atlas-application.properties, instead of component's configuration (mneethiraj via sumasai)
ATLAS-1030 Add instrumentation to measure performance: REST API (mneethiraj via sumasai)
ATLAS-996 DSL queries with comparsions of many primitive types fail (jnhagelb via shwethags)
ATLAS-971 UI not displaying results for this query - Eg: "hive_table as t where qualifiedName = 'default.input@cl1' select t" (kevalbhatt18 via shwethags)