You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ke...@apache.org on 2012/12/15 00:28:08 UTC
svn commit: r1422146 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/ conf/
metastore/src/java/org/apache/hadoop/hive/metastore/
metastore/src/test/org/apache/hadoop/hive/metastore/
Author: kevinwilfong
Date: Fri Dec 14 23:28:04 2012
New Revision: 1422146
URL: http://svn.apache.org/viewvc?rev=1422146&view=rev
Log:
HIVE-3766. Enable adding hooks to hive meta store init. (Jean Xu via kevinwilfong)
Added:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml.template
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1422146&r1=1422145&r2=1422146&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Dec 14 23:28:04 2012
@@ -125,6 +125,7 @@ public class HiveConf extends Configurat
HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS,
HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS,
HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX,
+ HiveConf.ConfVars.METASTORE_INIT_HOOKS,
HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS,
HiveConf.ConfVars.HMSHANDLERATTEMPTS,
HiveConf.ConfVars.HMSHANDLERINTERVAL,
@@ -311,6 +312,9 @@ public class HiveConf extends Configurat
METASTORE_BATCH_RETRIEVE_MAX("hive.metastore.batch.retrieve.max", 300),
METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX(
"hive.metastore.batch.retrieve.table.partition.max", 1000),
+ // A comma separated list of hooks which implement MetaStoreInitListener and will be run at
+ // the beginning of HMSHandler initialization
+ METASTORE_INIT_HOOKS("hive.metastore.init.hooks", ""),
METASTORE_PRE_EVENT_LISTENERS("hive.metastore.pre.event.listeners", ""),
METASTORE_EVENT_LISTENERS("hive.metastore.event.listeners", ""),
// should we do checks against the storage (usually hdfs) for operations like drop_partition
Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1422146&r1=1422145&r2=1422146&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Fri Dec 14 23:28:04 2012
@@ -688,6 +688,12 @@
</property>
<property>
+ <name>hive.metastore.init.hooks</name>
+ <value></value>
+ <description>A comma separated list of hooks to be invoked at the beginning of HMSHandler initialization. Aninit hook is specified as the name of Java class which extends org.apache.hadoop.hive.metastore.MetaStoreInitListener.</description>
+</property>
+
+<property>
<name>hive.client.stats.publishers</name>
<value></value>
<description>Comma-separated list of statistics publishers to be invoked on counters on each job. A client stats publisher is specified as the name of a Java class which implements the org.apache.hadoop.hive.ql.stats.ClientStatsPublisher interface.</description>
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1422146&r1=1422145&r2=1422146&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Fri Dec 14 23:28:04 2012
@@ -294,6 +294,7 @@ public class HiveMetaStore extends Thrif
private List<MetaStorePreEventListener> preListeners;
private List<MetaStoreEventListener> listeners;
private List<MetaStoreEndFunctionListener> endFunctionListeners;
+ private List<MetaStoreInitListener> initListeners;
{
classLoader = Thread.currentThread().getContextClassLoader();
@@ -306,6 +307,14 @@ public class HiveMetaStore extends Thrif
rawStoreClassName = hiveConf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL);
checkForDefaultDb = hiveConf.getBoolean(
"hive.metastore.checkForDefaultDb", true);
+ initListeners = MetaStoreUtils.getMetaStoreListeners(
+ MetaStoreInitListener.class, hiveConf,
+ hiveConf.getVar(HiveConf.ConfVars.METASTORE_INIT_HOOKS));
+ for (MetaStoreInitListener singleInitListener: initListeners) {
+ MetaStoreInitContext context = new MetaStoreInitContext();
+ singleInitListener.onInit(context);
+ }
+
String alterHandlerName = hiveConf.get("hive.metastore.alter.impl",
HiveAlterHandler.class.getName());
alterHandler = (AlterHandler) ReflectionUtils.newInstance(MetaStoreUtils.getClass(
Added: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java?rev=1422146&view=auto
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java (added)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java Fri Dec 14 23:28:04 2012
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+/**
+ * Base class which provides context to implementations of MetaStoreInitListener
+ */
+
+public class MetaStoreInitContext {
+
+}
Added: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java?rev=1422146&view=auto
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java (added)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java Fri Dec 14 23:28:04 2012
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+/**
+ * This abstract class needs to be extended to provide implementation of actions
+ * that needs to be performed when HMSHandler is initialized
+ */
+
+public abstract class MetaStoreInitListener implements Configurable {
+
+ private Configuration conf;
+
+ public MetaStoreInitListener(Configuration config){
+ this.conf = config;
+ }
+
+ public abstract void onInit(MetaStoreInitContext context) throws MetaException;
+
+ @Override
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ @Override
+ public void setConf(Configuration config) {
+ this.conf = config;
+ }
+}
Added: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java?rev=1422146&view=auto
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java (added)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java Fri Dec 14 23:28:04 2012
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreInitContext;
+import org.apache.hadoop.hive.metastore.MetaStoreInitListener;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+/*
+ * An implementation of MetaStoreInitListener to verify onInit is called when
+ * HMSHandler is initialized
+ */
+public class DummyMetaStoreInitListener extends MetaStoreInitListener{
+
+ public static boolean wasCalled = false;
+ public DummyMetaStoreInitListener(Configuration config) {
+ super(config);
+ }
+
+ @Override
+ public void onInit(MetaStoreInitContext context) throws MetaException {
+ wasCalled = true;
+ }
+}
Added: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java?rev=1422146&view=auto
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java (added)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java Fri Dec 14 23:28:04 2012
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.ShimLoader;
+
+/**
+ * TestMetaStoreInitListener. Test case for
+ * {@link org.apache.hadoop.hive.metastore.MetaStoreInitListener}
+ */
+public class TestMetaStoreInitListener extends TestCase {
+ private HiveConf hiveConf;
+ private HiveMetaStoreClient msc;
+ private Driver driver;
+
+ @Override
+ protected void setUp() throws Exception {
+
+ super.setUp();
+ System.setProperty("hive.metastore.init.hooks",
+ DummyMetaStoreInitListener.class.getName());
+ int port = MetaStoreUtils.findFreePort();
+ MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ hiveConf = new HiveConf(this.getClass());
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
+ hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
+ hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
+ hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
+ hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+ SessionState.start(new CliSessionState(hiveConf));
+ msc = new HiveMetaStoreClient(hiveConf, null);
+ driver = new Driver(hiveConf);
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ public void testMetaStoreInitListener() throws Exception {
+ // DummyMataStoreInitListener's onInit will be called at HMSHandler
+ // initialization, and set this to true
+ assertTrue(DummyMetaStoreInitListener.wasCalled);
+ }
+
+}