You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2015/07/08 08:53:45 UTC
incubator-atlas git commit: ATLAS-54 Rename configs in hive hook
(shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master b273520e0 -> 864cc5bc3
ATLAS-54 Rename configs in hive hook (shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/864cc5bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/864cc5bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/864cc5bc
Branch: refs/heads/master
Commit: 864cc5bc30a03eaef6ffc4f3f93e40c42b30af3d
Parents: b273520
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Wed Jul 8 12:23:33 2015 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Wed Jul 8 12:23:33 2015 +0530
----------------------------------------------------------------------
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 6 +++---
.../java/org/apache/atlas/hive/hook/HiveHook.java | 11 +++++++----
addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki | 16 ++++++++--------
.../atlas/hive/hook/BaseSSLAndKerberosTest.java | 10 +---------
.../java/org/apache/atlas/hive/hook/HiveHookIT.java | 16 ++++++++++------
.../org/apache/atlas/hive/hook/SSLHiveHookIT.java | 10 +---------
release-log.txt | 1 +
7 files changed, 31 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index a8f6a67..4d1af4e 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -56,11 +56,11 @@ import java.util.Set;
*/
public class HiveMetaStoreBridge {
private static final String DEFAULT_DGI_URL = "http://localhost:21000/";
- public static final String HIVE_CLUSTER_NAME = "hive.cluster.name";
+ public static final String HIVE_CLUSTER_NAME = "atlas.cluster.name";
public static final String DEFAULT_CLUSTER_NAME = "primary";
private final String clusterName;
- public static final String DGI_URL_PROPERTY = "hive.hook.dgi.url";
+ public static final String ATLAS_ENDPOINT = "atlas.rest.address";
private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class);
@@ -78,7 +78,7 @@ public class HiveMetaStoreBridge {
public HiveMetaStoreBridge(HiveConf hiveConf, String doAsUser, UserGroupInformation ugi) throws Exception {
clusterName = hiveConf.get(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
hiveClient = Hive.get(hiveConf);
- atlasClient = new AtlasClient(hiveConf.get(DGI_URL_PROPERTY, DEFAULT_DGI_URL), ugi, doAsUser);
+ atlasClient = new AtlasClient(hiveConf.get(ATLAS_ENDPOINT, DEFAULT_DGI_URL), ugi, doAsUser);
}
public AtlasClient getAtlasClient() {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 3d55e85..fe26446 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -60,9 +60,12 @@ public class HiveHook implements ExecuteWithHookContext {
private static final int WAIT_TIME = 3;
private static ExecutorService executor;
- private static final String MIN_THREADS = "hive.hook.dgi.minThreads";
- private static final String MAX_THREADS = "hive.hook.dgi.maxThreads";
- private static final String KEEP_ALIVE_TIME = "hive.hook.dgi.keepAliveTime";
+ public static final String CONF_PREFIX = "atlas.hook.hive.";
+
+ private static final String MIN_THREADS = CONF_PREFIX + "minThreads";
+ private static final String MAX_THREADS = CONF_PREFIX + "maxThreads";
+ private static final String KEEP_ALIVE_TIME = CONF_PREFIX + "keepAliveTime";
+ public static final String CONF_SYNC = CONF_PREFIX + "synchronous";
private static final int minThreadsDefault = 5;
private static final int maxThreadsDefault = 5;
@@ -131,7 +134,7 @@ public class HiveHook implements ExecuteWithHookContext {
// clone to avoid concurrent access
final HiveEvent event = new HiveEvent();
final HiveConf conf = new HiveConf(hookContext.getConf());
- boolean debug = conf.get("hive.hook.dgi.synchronous", "false").equals("true");
+ boolean debug = conf.get(CONF_SYNC, "false").equals("true");
event.conf = conf;
event.inputs = hookContext.getInputs();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki b/addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki
index e5646b9..647af27 100644
--- a/addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki
+++ b/addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki
@@ -25,11 +25,11 @@ hive conf directory:
* Atlas endpoint - Add the following property with the Atlas endpoint for your set-up
<verbatim>
<property>
- <name>hive.hook.dgi.url</name>
+ <name>atlas.rest.address</name>
<value>http://localhost:21000/</value>
</property>
<property>
- <name>hive.cluster.name</name>
+ <name>atlas.cluster.name</name>
<value>primary</value>
</property>
</verbatim>
@@ -50,21 +50,21 @@ The hook submits the request to a thread pool executor to avoid blocking the com
* Add the following properties in hive-ste.xml with the Atlas endpoint for your set-up
<verbatim>
<property>
- <name>hive.hook.dgi.url</name>
+ <name>atlas.rest.address</name>
<value>http://localhost:21000/</value>
</property>
<property>
- <name>hive.cluster.name</name>
+ <name>atlas.cluster.name</name>
<value>primary</value>
</property>
</verbatim>
* Add 'export HIVE_AUX_JARS_PATH=<dgi package>/hook/hive' in hive-env.sh
The following properties in hive-site.xml control the thread pool details:
- * hive.hook.dgi.minThreads - core number of threads. default 5
- * hive.hook.dgi.maxThreads - maximum number of threads. default 5
- * hive.hook.dgi.keepAliveTime - keep alive time in msecs. default 10
- * hive.hook.dgi.synchronous - boolean, true to run the hook synchronously. default false
+ * atlas.hook.hive.minThreads - core number of threads. default 5
+ * atlas.hook.hive.maxThreads - maximum number of threads. default 5
+ * atlas.hook.hive.keepAliveTime - keep alive time in msecs. default 10
+ * atlas.hook.hive.synchronous - boolean, true to run the hook synchronously. default false
---++ Limitations
* Since database name, table name and column names are case insensitive in hive, the corresponding names in entities are lowercase. So, any search APIs should use lowercase while querying on the entity names
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java
index 4669577..5ae6fe2 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java
@@ -123,14 +123,6 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest {
}
protected HiveConf getHiveConf() {
- HiveConf hiveConf = new HiveConf(this.getClass());
- hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
- hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName());
- hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/atlas");
- hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL);
- hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
- hiveConf.set("hive.hook.dgi.synchronous", "true");
- return hiveConf;
+ return HiveHookIT.createHiveConf(DGI_URL);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 95f4eeb..2bf3aba 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -54,7 +54,7 @@ public class HiveHookIT {
@BeforeClass
public void setUp() throws Exception {
//Set-up hive session
- HiveConf conf = getHiveConf();
+ HiveConf conf = createHiveConf();
driver = new Driver(conf);
ss = new SessionState(conf, System.getProperty("user.name"));
ss = SessionState.start(ss);
@@ -63,15 +63,19 @@ public class HiveHookIT {
dgiCLient = new AtlasClient(DGI_URL);
}
- private HiveConf getHiveConf() {
- HiveConf hiveConf = new HiveConf(this.getClass());
+ public static HiveConf createHiveConf() {
+ return createHiveConf(DGI_URL);
+ }
+
+ public static HiveConf createHiveConf(String atlasEndpoint) {
+ HiveConf hiveConf = new HiveConf(HiveHookIT.class);
hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName());
hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/metastore");
- hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL);
- hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
- hiveConf.set("hive.hook.dgi.synchronous", "true");
+ hiveConf.set(HiveMetaStoreBridge.ATLAS_ENDPOINT, atlasEndpoint);
+ hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, "jdbc:derby:./target/metastore_db;create=true");
+ hiveConf.set(HiveHook.CONF_SYNC, "true");
hiveConf.set(HiveMetaStoreBridge.HIVE_CLUSTER_NAME, CLUSTER_NAME);
hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODE, true); //to not use hdfs
hiveConf.setVar(HiveConf.ConfVars.HIVETESTMODEPREFIX, "");
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java
index 5c17b64..54bfaf6 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java
@@ -187,15 +187,7 @@ public class SSLHiveHookIT {
}
private HiveConf getHiveConf() {
- HiveConf hiveConf = new HiveConf(this.getClass());
- hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, "");
- hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName());
- hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/atlas");
- hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL);
- hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true");
- hiveConf.set("hive.hook.dgi.synchronous", "true");
- return hiveConf;
+ return HiveHookIT.createHiveConf(DGI_URL);
}
private void runCommand(String cmd) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index f527845..39ff8c2 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -4,6 +4,7 @@ Apache Atlas Release Notes
--trunk - unreleased
INCOMPATIBLE CHANGES:
+ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags)
ALL CHANGES: