You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/12/06 21:20:18 UTC
[03/12] hive git commit: HIVE-17980 Move HiveMetaStoreClient plus a
few remaining classes. This closes #272 (Alan Gates, reviewed by Daniel Dai)
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index d18ddc8..b46cc38 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -463,6 +463,9 @@ public class MetastoreConf {
"hive.metastore.event.message.factory",
"org.apache.hadoop.hive.metastore.messaging.json.JSONMessageFactory",
"Factory class for making encoding and decoding messages in the events generated."),
+ EVENT_DB_LISTENER_TTL("metastore.event.db.listener.timetolive",
+ "hive.metastore.event.db.listener.timetolive", 86400, TimeUnit.SECONDS,
+ "time after which events will be removed from the database listener queue"),
EVENT_DB_NOTIFICATION_API_AUTH("metastore.metastore.event.db.notification.api.auth",
"hive.metastore.event.db.notification.api.auth", true,
"Should metastore do authorization against database notification related APIs such as get_next_notification.\n" +
@@ -799,6 +802,19 @@ public class MetastoreConf {
"internal use only, true when in testing tez"),
// We need to track this as some listeners pass it through our config and we need to honor
// the system properties.
+ HIVE_AUTHORIZATION_MANAGER("hive.security.authorization.manager",
+ "hive.security.authorization.manager",
+ "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory",
+ "The Hive client authorization manager class name. The user defined authorization class should implement \n" +
+ "interface org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider."),
+ HIVE_METASTORE_AUTHENTICATOR_MANAGER("hive.security.metastore.authenticator.manager",
+ "hive.security.metastore.authenticator.manager",
+ "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
+ "authenticator manager class name to be used in the metastore for authentication. \n" +
+ "The user defined authenticator should implement interface org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider."),
+ HIVE_METASTORE_AUTHORIZATION_AUTH_READS("hive.security.metastore.authorization.auth.reads",
+ "hive.security.metastore.authorization.auth.reads", true,
+ "If this is true, metastore authorizer authorizes read actions on database, table"),
HIVE_METASTORE_AUTHORIZATION_MANAGER(NO_SUCH_KEY,
"hive.security.metastore.authorization.manager",
"org.apache.hadoop.hive.ql.security.authorization.DefaultHiveMetastoreAuthorizationProvider",
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java
new file mode 100644
index 0000000..7d8c1d4
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventUtils.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore.messaging;
+
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.messaging.event.filters.DatabaseAndTableFilter;
+import org.apache.thrift.TException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class EventUtils {
+
+ public interface NotificationFetcher {
+ int getBatchSize() throws IOException;
+ long getCurrentNotificationEventId() throws IOException;
+ long getDbNotificationEventsCount(long fromEventId, String dbName) throws IOException;
+ List<NotificationEvent> getNextNotificationEvents(
+ long pos, IMetaStoreClient.NotificationFilter filter) throws IOException;
+ }
+
+ // MetaStoreClient-based impl of NotificationFetcher
+ public static class MSClientNotificationFetcher implements NotificationFetcher{
+
+ private IMetaStoreClient msc = null;
+ private Integer batchSize = null;
+
+ public MSClientNotificationFetcher(IMetaStoreClient msc){
+ this.msc = msc;
+ }
+
+ @Override
+ public int getBatchSize() throws IOException {
+ if (batchSize == null){
+ try {
+ batchSize = Integer.parseInt(
+ msc.getConfigValue(MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX.toString(), "50"));
+ // TODO: we're asking the metastore what its configuration for this var is - we may
+ // want to revisit to pull from client side instead. The reason I have it this way
+ // is because the metastore is more likely to have a reasonable config for this than
+ // an arbitrary client.
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ }
+ return batchSize;
+ }
+
+ @Override
+ public long getCurrentNotificationEventId() throws IOException {
+ try {
+ return msc.getCurrentNotificationEventId().getEventId();
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public long getDbNotificationEventsCount(long fromEventId, String dbName) throws IOException {
+ try {
+ NotificationEventsCountRequest rqst
+ = new NotificationEventsCountRequest(fromEventId, dbName);
+ return msc.getNotificationEventsCount(rqst).getEventsCount();
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public List<NotificationEvent> getNextNotificationEvents(
+ long pos, IMetaStoreClient.NotificationFilter filter) throws IOException {
+ try {
+ return msc.getNextNotification(pos,getBatchSize(), filter).getEvents();
+ } catch (TException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ public static class NotificationEventIterator implements Iterator<NotificationEvent> {
+
+ private NotificationFetcher nfetcher;
+ private IMetaStoreClient.NotificationFilter filter;
+ private int maxEvents;
+
+ private Iterator<NotificationEvent> batchIter = null;
+ private List<NotificationEvent> batch = null;
+ private long pos;
+ private long maxPos;
+ private int eventCount;
+
+ public NotificationEventIterator(
+ NotificationFetcher nfetcher, long eventFrom, int maxEvents,
+ String dbName, String tableName) throws IOException {
+ init(nfetcher, eventFrom, maxEvents, new DatabaseAndTableFilter(dbName, tableName));
+ // using init(..) instead of this(..) because the EventUtils.getDbTblNotificationFilter
+ // is an operation that needs to run before delegating to the other ctor, and this messes up chaining
+ // ctors
+ }
+
+ public NotificationEventIterator(
+ NotificationFetcher nfetcher, long eventFrom, int maxEvents,
+ IMetaStoreClient.NotificationFilter filter) throws IOException {
+ init(nfetcher,eventFrom,maxEvents,filter);
+ }
+
+ private void init(
+ NotificationFetcher nfetcher, long eventFrom, int maxEvents,
+ IMetaStoreClient.NotificationFilter filter) throws IOException {
+ this.nfetcher = nfetcher;
+ this.filter = filter;
+ this.pos = eventFrom;
+ if (maxEvents < 1){
+ // 0 or -1 implies fetch everything
+ this.maxEvents = Integer.MAX_VALUE;
+ } else {
+ this.maxEvents = maxEvents;
+ }
+
+ this.eventCount = 0;
+ this.maxPos = nfetcher.getCurrentNotificationEventId();
+ }
+
+ private void fetchNextBatch() throws IOException {
+ batch = nfetcher.getNextNotificationEvents(pos, filter);
+ int batchSize = nfetcher.getBatchSize();
+ while ( ((batch == null) || (batch.isEmpty())) && (pos < maxPos) ){
+ // no valid events this batch, but we're still not done processing events
+ pos += batchSize;
+ batch = nfetcher.getNextNotificationEvents(pos,filter);
+ }
+
+ if (batch == null){
+ batch = new ArrayList<>();
+ // instantiate empty list so that we don't error out on iterator fetching.
+ // If we're here, then the next check of pos will show our caller that
+ // that we've exhausted our event supply
+ }
+ batchIter = batch.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (eventCount >= maxEvents){
+ // If we've already satisfied the number of events we were supposed to deliver, we end it.
+ return false;
+ }
+ if ((batchIter != null) && (batchIter.hasNext())){
+ // If we have a valid batchIter and it has more elements, return them.
+ return true;
+ }
+ // If we're here, we want more events, and either batchIter is null, or batchIter
+ // has reached the end of the current batch. Let's fetch the next batch.
+ try {
+ fetchNextBatch();
+ } catch (IOException e) {
+ // Regrettable that we have to wrap the IOException into a RuntimeException,
+ // but throwing the exception is the appropriate result here, and hasNext()
+ // signature will only allow RuntimeExceptions. Iterator.hasNext() really
+ // should have allowed IOExceptions
+ throw new RuntimeException(e);
+ }
+ // New batch has been fetched. If it's not empty, we have more elements to process.
+ return !batch.isEmpty();
+ }
+
+ @Override
+ public NotificationEvent next() {
+ eventCount++;
+ NotificationEvent ev = batchIter.next();
+ pos = ev.getEventId();
+ return ev;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("remove() not supported on NotificationEventIterator");
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/AndFilter.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/AndFilter.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/AndFilter.java
new file mode 100644
index 0000000..454d2cc
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/AndFilter.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.messaging.event.filters;
+
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+
+public class AndFilter implements IMetaStoreClient.NotificationFilter {
+ final IMetaStoreClient.NotificationFilter[] filters;
+
+ public AndFilter(final IMetaStoreClient.NotificationFilter... filters) {
+ this.filters = filters;
+ }
+
+ @Override
+ public boolean accept(final NotificationEvent event) {
+ for (IMetaStoreClient.NotificationFilter filter : filters) {
+ if (!filter.accept(event)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/BasicFilter.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/BasicFilter.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/BasicFilter.java
new file mode 100644
index 0000000..84302d6
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/BasicFilter.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.messaging.event.filters;
+
+import org.apache.hadoop.hive.metastore.IMetaStoreClient.NotificationFilter;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+
+public abstract class BasicFilter implements NotificationFilter {
+ @Override
+ public boolean accept(final NotificationEvent event) {
+ if (event == null) {
+ return false; // get rid of trivial case first, so that we can safely assume non-null
+ }
+ return shouldAccept(event);
+ }
+
+ abstract boolean shouldAccept(final NotificationEvent event);
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/DatabaseAndTableFilter.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/DatabaseAndTableFilter.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/DatabaseAndTableFilter.java
new file mode 100644
index 0000000..0852abd
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/DatabaseAndTableFilter.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.messaging.event.filters;
+
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+
+/**
+ * Utility function that constructs a notification filter to match a given db name and/or table name.
+ * If dbName == null, fetches all warehouse events.
+ * If dnName != null, but tableName == null, fetches all events for the db
+ * If dbName != null && tableName != null, fetches all events for the specified table
+ */
+public class DatabaseAndTableFilter extends BasicFilter {
+ private final String databaseName, tableName;
+
+ public DatabaseAndTableFilter(final String databaseName, final String tableName) {
+ this.databaseName = databaseName;
+ this.tableName = tableName;
+ }
+
+ @Override
+ boolean shouldAccept(final NotificationEvent event) {
+ if (databaseName == null) {
+ return true; // if our dbName is null, we're interested in all wh events
+ }
+ if (databaseName.equalsIgnoreCase(event.getDbName())) {
+ if ((tableName == null)
+ // if our dbName is equal, but tableName is blank, we're interested in this db-level event
+ || (tableName.equalsIgnoreCase(event.getTableName()))
+ // table level event that matches us
+ ) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/EventBoundaryFilter.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/EventBoundaryFilter.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/EventBoundaryFilter.java
new file mode 100644
index 0000000..988874d
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/EventBoundaryFilter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.messaging.event.filters;
+
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+
+public class EventBoundaryFilter extends BasicFilter {
+ private final long eventFrom, eventTo;
+
+ public EventBoundaryFilter(final long eventFrom, final long eventTo) {
+ this.eventFrom = eventFrom;
+ this.eventTo = eventTo;
+ }
+
+ @Override
+ boolean shouldAccept(final NotificationEvent event) {
+ return eventFrom <= event.getEventId() && event.getEventId() <= eventTo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/MessageFormatFilter.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/MessageFormatFilter.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/MessageFormatFilter.java
new file mode 100644
index 0000000..b0ce3b9
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/messaging/event/filters/MessageFormatFilter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.messaging.event.filters;
+
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+
+public class MessageFormatFilter extends BasicFilter {
+ private final String format;
+
+ public MessageFormatFilter(String format) {
+ this.format = format;
+ }
+
+ @Override
+ boolean shouldAccept(final NotificationEvent event) {
+ if (format == null) {
+ return true; // let's say that passing null in will not do any filtering.
+ }
+ return format.equalsIgnoreCase(event.getMessageFormat());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
new file mode 100644
index 0000000..f4eacd5
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.tools;
+
+import java.net.URI;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.ObjectStore;
+
+/**
+ * This class provides Hive admins a tool to
+ * - execute JDOQL against the metastore using DataNucleus
+ * - perform HA name node upgrade
+ */
+
+public class HiveMetaTool {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveMetaTool.class.getName());
+ private final Options cmdLineOptions = new Options();
+ private ObjectStore objStore;
+ private boolean isObjStoreInitialized;
+
+ public HiveMetaTool() {
+ this.isObjStoreInitialized = false;
+ }
+
+ @SuppressWarnings("static-access")
+ private void init() {
+
+ System.out.println("Initializing HiveMetaTool..");
+
+ Option help = new Option("help", "print this message");
+ Option listFSRoot = new Option("listFSRoot", "print the current FS root locations");
+ Option executeJDOQL =
+ OptionBuilder.withArgName("query-string")
+ .hasArgs()
+ .withDescription("execute the given JDOQL query")
+ .create("executeJDOQL");
+
+ /* Ideally we want to specify the different arguments to updateLocation as separate argNames.
+ * However if we did that, HelpFormatter swallows all but the last argument. Note that this is
+ * a know issue with the HelpFormatter class that has not been fixed. We specify all arguments
+ * with a single argName to workaround this HelpFormatter bug.
+ */
+ Option updateFSRootLoc =
+ OptionBuilder
+ .withArgName("new-loc> " + "<old-loc")
+ .hasArgs(2)
+ .withDescription(
+ "Update FS root location in the metastore to new location.Both new-loc and " +
+ "old-loc should be valid URIs with valid host names and schemes." +
+ "When run with the dryRun option changes are displayed but are not " +
+ "persisted. When run with the serdepropKey/tablePropKey option " +
+ "updateLocation looks for the serde-prop-key/table-prop-key that is " +
+ "specified and updates its value if found.")
+ .create("updateLocation");
+ Option dryRun = new Option("dryRun" , "Perform a dry run of updateLocation changes.When " +
+ "run with the dryRun option updateLocation changes are displayed but not persisted. " +
+ "dryRun is valid only with the updateLocation option.");
+ Option serdePropKey =
+ OptionBuilder.withArgName("serde-prop-key")
+ .hasArgs()
+ .withValueSeparator()
+ .withDescription("Specify the key for serde property to be updated. serdePropKey option " +
+ "is valid only with updateLocation option.")
+ .create("serdePropKey");
+ Option tablePropKey =
+ OptionBuilder.withArgName("table-prop-key")
+ .hasArg()
+ .withValueSeparator()
+ .withDescription("Specify the key for table property to be updated. tablePropKey option " +
+ "is valid only with updateLocation option.")
+ .create("tablePropKey");
+
+ cmdLineOptions.addOption(help);
+ cmdLineOptions.addOption(listFSRoot);
+ cmdLineOptions.addOption(executeJDOQL);
+ cmdLineOptions.addOption(updateFSRootLoc);
+ cmdLineOptions.addOption(dryRun);
+ cmdLineOptions.addOption(serdePropKey);
+ cmdLineOptions.addOption(tablePropKey);
+ }
+
+ private void initObjectStore(Configuration conf) {
+ if (!isObjStoreInitialized) {
+ objStore = new ObjectStore();
+ objStore.setConf(conf);
+ isObjStoreInitialized = true;
+ }
+ }
+
+ private void shutdownObjectStore() {
+ if (isObjStoreInitialized) {
+ objStore.shutdown();
+ isObjStoreInitialized = false;
+ }
+ }
+
+ private void listFSRoot() {
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ initObjectStore(conf);
+
+ Set<String> hdfsRoots = objStore.listFSRoots();
+ if (hdfsRoots != null) {
+ System.out.println("Listing FS Roots..");
+ for (String s : hdfsRoots) {
+ System.out.println(s);
+ }
+ } else {
+ System.err.println("Encountered error during listFSRoot - " +
+ "commit of JDO transaction failed");
+ }
+ }
+
+ private void executeJDOQLSelect(String query) {
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ initObjectStore(conf);
+
+ System.out.println("Executing query: " + query);
+ try (ObjectStore.QueryWrapper queryWrapper = new ObjectStore.QueryWrapper()) {
+ Collection<?> result = objStore.executeJDOQLSelect(query, queryWrapper);
+ if (result != null) {
+ Iterator<?> iter = result.iterator();
+ while (iter.hasNext()) {
+ Object o = iter.next();
+ System.out.println(o.toString());
+ }
+ } else {
+ System.err.println("Encountered error during executeJDOQLSelect -" +
+ "commit of JDO transaction failed.");
+ }
+ }
+ }
+
+ private void executeJDOQLUpdate(String query) {
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ initObjectStore(conf);
+
+ System.out.println("Executing query: " + query);
+ long numUpdated = objStore.executeJDOQLUpdate(query);
+ if (numUpdated >= 0) {
+ System.out.println("Number of records updated: " + numUpdated);
+ } else {
+ System.err.println("Encountered error during executeJDOQL -" +
+ "commit of JDO transaction failed.");
+ }
+ }
+
+ private int printUpdateLocations(Map<String, String> updateLocations) {
+ int count = 0;
+ for (String key: updateLocations.keySet()) {
+ String value = updateLocations.get(key);
+ System.out.println("old location: " + key + " new location: " + value);
+ count++;
+ }
+ return count;
+ }
+
+ private void printTblURIUpdateSummary(ObjectStore.UpdateMStorageDescriptorTblURIRetVal retVal,
+ boolean isDryRun) {
+ String tblName = "SDS";
+ String fieldName = "LOCATION";
+
+ if (retVal == null) {
+ System.err.println("Encountered error while executing updateMStorageDescriptorTblURI - " +
+ "commit of JDO transaction failed. Failed to update FSRoot locations in " +
+ fieldName + "field in " + tblName + " table.");
+ } else {
+ Map<String, String> updateLocations = retVal.getUpdateLocations();
+ if (isDryRun) {
+ System.out.println("Dry Run of updateLocation on table " + tblName + "..");
+ } else {
+ System.out.println("Successfully updated the following locations..");
+ }
+ int count = printUpdateLocations(updateLocations);
+ if (isDryRun) {
+ System.out.println("Found " + count + " records in " + tblName + " table to update");
+ } else {
+ System.out.println("Updated " + count + " records in " + tblName + " table");
+ }
+ List<String> badRecords = retVal.getBadRecords();
+ if (badRecords.size() > 0) {
+ System.err.println("Warning: Found records with bad " + fieldName + " in " +
+ tblName + " table.. ");
+ for (String badRecord:badRecords) {
+ System.err.println("bad location URI: " + badRecord);
+ }
+ }
+ int numNullRecords = retVal.getNumNullRecords();
+ if (numNullRecords != 0) {
+ LOG.debug("Number of NULL location URI: " + numNullRecords +
+ ". This can happen for View or Index.");
+ }
+ }
+ }
+
+ private void printDatabaseURIUpdateSummary(ObjectStore.UpdateMDatabaseURIRetVal retVal,
+ boolean isDryRun) {
+ String tblName = "DBS";
+ String fieldName = "LOCATION_URI";
+
+ if (retVal == null) {
+ System.err.println("Encountered error while executing updateMDatabaseURI - " +
+ "commit of JDO transaction failed. Failed to update FSRoot locations in " +
+ fieldName + "field in " + tblName + " table.");
+ } else {
+ Map<String, String> updateLocations = retVal.getUpdateLocations();
+ if (isDryRun) {
+ System.out.println("Dry Run of updateLocation on table " + tblName + "..");
+ } else {
+ System.out.println("Successfully updated the following locations..");
+ }
+ int count = printUpdateLocations(updateLocations);
+ if (isDryRun) {
+ System.out.println("Found " + count + " records in " + tblName + " table to update");
+ } else {
+ System.out.println("Updated " + count + " records in " + tblName + " table");
+ }
+ List<String> badRecords = retVal.getBadRecords();
+ if (badRecords.size() > 0) {
+ System.err.println("Warning: Found records with bad " + fieldName + " in " +
+ tblName + " table.. ");
+ for (String badRecord:badRecords) {
+ System.err.println("bad location URI: " + badRecord);
+ }
+ }
+ }
+ }
+
+ private void printPropURIUpdateSummary(ObjectStore.UpdatePropURIRetVal retVal, String
+ tablePropKey, boolean isDryRun, String tblName, String methodName) {
+ if (retVal == null) {
+ System.err.println("Encountered error while executing " + methodName + " - " +
+ "commit of JDO transaction failed. Failed to update FSRoot locations in " +
+ "value field corresponding to" + tablePropKey + " in " + tblName + " table.");
+ } else {
+ Map<String, String> updateLocations = retVal.getUpdateLocations();
+ if (isDryRun) {
+ System.out.println("Dry Run of updateLocation on table " + tblName + "..");
+ } else {
+ System.out.println("Successfully updated the following locations..");
+ }
+ int count = printUpdateLocations(updateLocations);
+ if (isDryRun) {
+ System.out.println("Found " + count + " records in " + tblName + " table to update");
+ } else {
+ System.out.println("Updated " + count + " records in " + tblName + " table");
+ }
+ List<String> badRecords = retVal.getBadRecords();
+ if (badRecords.size() > 0) {
+ System.err.println("Warning: Found records with bad " + tablePropKey + " key in " +
+ tblName + " table.. ");
+ for (String badRecord:badRecords) {
+ System.err.println("bad location URI: " + badRecord);
+ }
+ }
+ }
+ }
+
+ private void printSerdePropURIUpdateSummary(ObjectStore.UpdateSerdeURIRetVal retVal,
+ String serdePropKey, boolean isDryRun) {
+ String tblName = "SERDE_PARAMS";
+
+ if (retVal == null) {
+ System.err.println("Encountered error while executing updateSerdeURI - " +
+ "commit of JDO transaction failed. Failed to update FSRoot locations in " +
+ "value field corresponding to " + serdePropKey + " in " + tblName + " table.");
+ } else {
+ Map<String, String> updateLocations = retVal.getUpdateLocations();
+ if (isDryRun) {
+ System.out.println("Dry Run of updateLocation on table " + tblName + "..");
+ } else {
+ System.out.println("Successfully updated the following locations..");
+ }
+ int count = printUpdateLocations(updateLocations);
+ if (isDryRun) {
+ System.out.println("Found " + count + " records in " + tblName + " table to update");
+ } else {
+ System.out.println("Updated " + count + " records in " + tblName + " table");
+ }
+ List<String> badRecords = retVal.getBadRecords();
+ if (badRecords.size() > 0) {
+ System.err.println("Warning: Found records with bad " + serdePropKey + " key in " +
+ tblName + " table.. ");
+ for (String badRecord:badRecords) {
+ System.err.println("bad location URI: " + badRecord);
+ }
+ }
+ }
+ }
+
+ public void updateFSRootLocation(URI oldURI, URI newURI, String serdePropKey, String
+ tablePropKey, boolean isDryRun) {
+ Configuration conf = MetastoreConf.newMetastoreConf();
+ initObjectStore(conf);
+
+ System.out.println("Looking for LOCATION_URI field in DBS table to update..");
+ ObjectStore.UpdateMDatabaseURIRetVal updateMDBURIRetVal = objStore.updateMDatabaseURI(oldURI,
+ newURI, isDryRun);
+ printDatabaseURIUpdateSummary(updateMDBURIRetVal, isDryRun);
+
+ System.out.println("Looking for LOCATION field in SDS table to update..");
+ ObjectStore.UpdateMStorageDescriptorTblURIRetVal updateTblURIRetVal =
+ objStore.updateMStorageDescriptorTblURI(oldURI, newURI, isDryRun);
+ printTblURIUpdateSummary(updateTblURIRetVal, isDryRun);
+
+ if (tablePropKey != null) {
+ System.out.println("Looking for value of " + tablePropKey + " key in TABLE_PARAMS table " +
+ "to update..");
+ ObjectStore.UpdatePropURIRetVal updateTblPropURIRetVal =
+ objStore.updateTblPropURI(oldURI, newURI,
+ tablePropKey, isDryRun);
+ printPropURIUpdateSummary(updateTblPropURIRetVal, tablePropKey, isDryRun, "TABLE_PARAMS",
+ "updateTblPropURI");
+
+ System.out.println("Looking for value of " + tablePropKey + " key in SD_PARAMS table " +
+ "to update..");
+ ObjectStore.UpdatePropURIRetVal updatePropURIRetVal = objStore
+ .updateMStorageDescriptorTblPropURI(oldURI, newURI, tablePropKey, isDryRun);
+ printPropURIUpdateSummary(updatePropURIRetVal, tablePropKey, isDryRun, "SD_PARAMS",
+ "updateMStorageDescriptorTblPropURI");
+ }
+
+ if (serdePropKey != null) {
+ System.out.println("Looking for value of " + serdePropKey + " key in SERDE_PARAMS table " +
+ "to update..");
+ ObjectStore.UpdateSerdeURIRetVal updateSerdeURIretVal = objStore.updateSerdeURI(oldURI,
+ newURI, serdePropKey, isDryRun);
+ printSerdePropURIUpdateSummary(updateSerdeURIretVal, serdePropKey, isDryRun);
+ }
+ }
+
+ private static void printAndExit(HiveMetaTool metaTool) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("metatool", metaTool.cmdLineOptions);
+ System.exit(1);
+ }
+
+ public static void main(String[] args) {
+ HiveMetaTool metaTool = new HiveMetaTool();
+ metaTool.init();
+ CommandLineParser parser = new GnuParser();
+ CommandLine line = null;
+
+ try {
+ try {
+ line = parser.parse(metaTool.cmdLineOptions, args);
+ } catch (ParseException e) {
+ System.err.println("HiveMetaTool:Parsing failed. Reason: " + e.getLocalizedMessage());
+ printAndExit(metaTool);
+ }
+
+ if (line.hasOption("help")) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("metatool", metaTool.cmdLineOptions);
+ } else if (line.hasOption("listFSRoot")) {
+ if (line.hasOption("dryRun")) {
+ System.err.println("HiveMetaTool: dryRun is not valid with listFSRoot");
+ printAndExit(metaTool);
+ } else if (line.hasOption("serdePropKey")) {
+ System.err.println("HiveMetaTool: serdePropKey is not valid with listFSRoot");
+ printAndExit(metaTool);
+ } else if (line.hasOption("tablePropKey")) {
+ System.err.println("HiveMetaTool: tablePropKey is not valid with listFSRoot");
+ printAndExit(metaTool);
+ }
+ metaTool.listFSRoot();
+ } else if (line.hasOption("executeJDOQL")) {
+ String query = line.getOptionValue("executeJDOQL");
+ if (line.hasOption("dryRun")) {
+ System.err.println("HiveMetaTool: dryRun is not valid with executeJDOQL");
+ printAndExit(metaTool);
+ } else if (line.hasOption("serdePropKey")) {
+ System.err.println("HiveMetaTool: serdePropKey is not valid with executeJDOQL");
+ printAndExit(metaTool);
+ } else if (line.hasOption("tablePropKey")) {
+ System.err.println("HiveMetaTool: tablePropKey is not valid with executeJDOQL");
+ printAndExit(metaTool);
+ }
+ if (query.toLowerCase().trim().startsWith("select")) {
+ metaTool.executeJDOQLSelect(query);
+ } else if (query.toLowerCase().trim().startsWith("update")) {
+ metaTool.executeJDOQLUpdate(query);
+ } else {
+ System.err.println("HiveMetaTool:Unsupported statement type");
+ printAndExit(metaTool);
+ }
+ } else if (line.hasOption("updateLocation")) {
+ String[] loc = line.getOptionValues("updateLocation");
+ boolean isDryRun = false;
+ String serdepropKey = null;
+ String tablePropKey = null;
+
+ if (loc.length != 2 && loc.length != 3) {
+ System.err.println("HiveMetaTool:updateLocation takes in 2 required and 1 " +
+ "optional arguments but " +
+ "was passed " + loc.length + " arguments");
+ printAndExit(metaTool);
+ }
+
+ Path newPath = new Path(loc[0]);
+ Path oldPath = new Path(loc[1]);
+
+ URI oldURI = oldPath.toUri();
+ URI newURI = newPath.toUri();
+
+ if (line.hasOption("dryRun")) {
+ isDryRun = true;
+ }
+
+ if (line.hasOption("serdePropKey")) {
+ serdepropKey = line.getOptionValue("serdePropKey");
+ }
+
+ if (line.hasOption("tablePropKey")) {
+ tablePropKey = line.getOptionValue("tablePropKey");
+ }
+
+ /*
+ * validate input - Both new and old URI should contain valid host names and valid schemes.
+ * port is optional in both the URIs since HDFS HA NN URI doesn't have a port.
+ */
+ if (oldURI.getHost() == null || newURI.getHost() == null) {
+ System.err.println("HiveMetaTool:A valid host is required in both old-loc and new-loc");
+ } else if (oldURI.getScheme() == null || newURI.getScheme() == null) {
+ System.err.println("HiveMetaTool:A valid scheme is required in both old-loc and new-loc");
+ } else {
+ metaTool.updateFSRootLocation(oldURI, newURI, serdepropKey, tablePropKey, isDryRun);
+ }
+ } else {
+ if (line.hasOption("dryRun")) {
+ System.err.println("HiveMetaTool: dryRun is not a valid standalone option");
+ } else if (line.hasOption("serdePropKey")) {
+ System.err.println("HiveMetaTool: serdePropKey is not a valid standalone option");
+ } else if (line.hasOption("tablePropKey")) {
+ System.err.println("HiveMetaTool: tablePropKey is not a valid standalone option");
+ printAndExit(metaTool);
+ } else {
+ System.err.print("HiveMetaTool:Parsing failed. Reason: Invalid arguments: " );
+ for (String s : line.getArgs()) {
+ System.err.print(s + " ");
+ }
+ System.err.println();
+ }
+ printAndExit(metaTool);
+ }
+ } finally {
+ metaTool.shutdownObjectStore();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
index 5dcedcd..b44ff8c 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
@@ -53,6 +53,16 @@ public class FileUtils {
return !name.startsWith("_") && !name.startsWith(".");
}
};
+ /**
+ * Filter that filters out hidden files
+ */
+ private static final PathFilter hiddenFileFilter = new PathFilter() {
+ @Override
+ public boolean accept(Path p) {
+ String name = p.getName();
+ return !name.startsWith("_") && !name.startsWith(".");
+ }
+ };
/**
* Move a particular file or directory to the trash.
@@ -424,4 +434,23 @@ public class FileUtils {
throw new MetaException("Unable to : " + path);
}
}
+
+ /**
+ * Utility method that determines if a specified directory already has
+ * contents (non-hidden files) or not - useful to determine if an
+ * immutable table already has contents, for example.
+ *
+ * @param path
+ * @throws IOException
+ */
+ public static boolean isDirEmpty(FileSystem fs, Path path) throws IOException {
+
+ if (fs.exists(path)) {
+ FileStatus[] status = fs.globStatus(new Path(path, "*"), hiddenFileFilter);
+ if (status.length > 0) {
+ return false;
+ }
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
index beee86f..cde34bc 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.metastore.ColumnType;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
@@ -52,7 +53,6 @@ import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFa
import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMerger;
import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMergerFactory;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.metastore.events.EventCleanerTask;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.security.SaslRpcServer;
@@ -64,9 +64,13 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.io.File;
+import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.Charset;
@@ -81,6 +85,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
@@ -298,8 +303,8 @@ public class MetaStoreUtils {
}
}
- public static String getPartitionValWithInvalidCharacter(List<String> partVals,
- Pattern partitionValidationPattern) {
+ private static String getPartitionValWithInvalidCharacter(List<String> partVals,
+ Pattern partitionValidationPattern) {
if (partitionValidationPattern == null) {
return null;
}
@@ -499,8 +504,9 @@ public class MetaStoreUtils {
}
// check if stats need to be (re)calculated
- public static boolean requireCalStats(Configuration hiveConf, Partition oldPart,
- Partition newPart, Table tbl, EnvironmentContext environmentContext) {
+ public static boolean requireCalStats(Partition oldPart,
+ Partition newPart, Table tbl,
+ EnvironmentContext environmentContext) {
if (environmentContext != null
&& environmentContext.isSetProperties()
@@ -792,9 +798,11 @@ public class MetaStoreUtils {
Configuration.class).newInstance(conf);
listeners.add(listener);
} catch (InvocationTargetException ie) {
+ LOG.error("Got InvocationTargetException", ie);
throw new MetaException("Failed to instantiate listener named: "+
listenerImpl + ", reason: " + ie.getCause());
} catch (Exception e) {
+ LOG.error("Got Exception", e);
throw new MetaException("Failed to instantiate listener named: "+
listenerImpl + ", reason: " + e);
}
@@ -961,13 +969,20 @@ public class MetaStoreUtils {
ColumnStatisticsObj statsObjNew = csNew.getStatsObj().get(index);
ColumnStatisticsObj statsObjOld = map.get(statsObjNew.getColName());
if (statsObjOld != null) {
+ // because we already confirm that the stats is accurate
+ // it is impossible that the column types have been changed while the
+ // column stats is still accurate.
+ assert (statsObjNew.getStatsData().getSetField() == statsObjOld.getStatsData()
+ .getSetField());
// If statsObjOld is found, we can merge.
ColumnStatsMerger merger = ColumnStatsMergerFactory.getColumnStatsMerger(statsObjNew,
statsObjOld);
merger.merge(statsObjNew, statsObjOld);
}
+ // If statsObjOld is not found, we just use statsObjNew as it is accurate.
list.add(statsObjNew);
}
+ // in all the other cases, we can not merge
csNew.setStatsObj(list);
}
@@ -1064,15 +1079,485 @@ public class MetaStoreUtils {
return machineList.includes(ipAddress);
}
- // TODO This should be moved to MetaStoreTestUtils once it is moved into standalone-metastore.
/**
- * Setup a configuration file for standalone mode. There are a few config variables that have
- * defaults that require parts of Hive that aren't present in standalone mode. This method
- * sets them to something that will work without the rest of Hive.
- * @param conf Configuration object
+ * Convert FieldSchemas to Thrift DDL.
*/
- public static void setConfForStandloneMode(Configuration conf) {
- MetastoreConf.setVar(conf, MetastoreConf.ConfVars.TASK_THREADS_ALWAYS,
- EventCleanerTask.class.getName());
+ public static String getDDLFromFieldSchema(String structName,
+ List<FieldSchema> fieldSchemas) {
+ StringBuilder ddl = new StringBuilder();
+ ddl.append("struct ");
+ ddl.append(structName);
+ ddl.append(" { ");
+ boolean first = true;
+ for (FieldSchema col : fieldSchemas) {
+ if (first) {
+ first = false;
+ } else {
+ ddl.append(", ");
+ }
+ ddl.append(ColumnType.typeToThriftType(col.getType()));
+ ddl.append(' ');
+ ddl.append(col.getName());
+ }
+ ddl.append("}");
+
+ LOG.trace("DDL: {}", ddl);
+ return ddl.toString();
+ }
+
+ public static Properties getTableMetadata(
+ org.apache.hadoop.hive.metastore.api.Table table) {
+ return MetaStoreUtils.getSchema(table.getSd(), table.getSd(), table
+ .getParameters(), table.getDbName(), table.getTableName(), table.getPartitionKeys());
+ }
+
+ public static Properties getPartitionMetadata(
+ org.apache.hadoop.hive.metastore.api.Partition partition,
+ org.apache.hadoop.hive.metastore.api.Table table) {
+ return MetaStoreUtils
+ .getSchema(partition.getSd(), partition.getSd(), partition
+ .getParameters(), table.getDbName(), table.getTableName(),
+ table.getPartitionKeys());
+ }
+
+ public static Properties getSchema(
+ org.apache.hadoop.hive.metastore.api.Partition part,
+ org.apache.hadoop.hive.metastore.api.Table table) {
+ return MetaStoreUtils.getSchema(part.getSd(), table.getSd(), table
+ .getParameters(), table.getDbName(), table.getTableName(), table.getPartitionKeys());
+ }
+
+ /**
+ * Get partition level schema from table level schema.
+ * This function will use the same column names, column types and partition keys for
+ * each partition Properties. Their values are copied from the table Properties. This
+ * is mainly to save CPU and memory. CPU is saved because the first time the
+ * StorageDescriptor column names are accessed, JDO needs to execute a SQL query to
+ * retrieve the data. If we know the data will be the same as the table level schema
+ * and they are immutable, we should just reuse the table level schema objects.
+ *
+ * @param sd The Partition level Storage Descriptor.
+ * @param parameters partition level parameters
+ * @param tblSchema The table level schema from which this partition should be copied.
+ * @return the properties
+ */
+ public static Properties getPartSchemaFromTableSchema(
+ StorageDescriptor sd,
+ Map<String, String> parameters,
+ Properties tblSchema) {
+
+ // Inherent most properties from table level schema and overwrite some properties
+ // in the following code.
+ // This is mainly for saving CPU and memory to reuse the column names, types and
+ // partition columns in the table level schema.
+ Properties schema = (Properties) tblSchema.clone();
+
+ // InputFormat
+ String inputFormat = sd.getInputFormat();
+ if (inputFormat == null || inputFormat.length() == 0) {
+ String tblInput =
+ schema.getProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT);
+ if (tblInput == null) {
+ inputFormat = org.apache.hadoop.mapred.SequenceFileInputFormat.class.getName();
+ } else {
+ inputFormat = tblInput;
+ }
+ }
+ schema.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT,
+ inputFormat);
+
+ // OutputFormat
+ String outputFormat = sd.getOutputFormat();
+ if (outputFormat == null || outputFormat.length() == 0) {
+ String tblOutput =
+ schema.getProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_OUTPUT_FORMAT);
+ if (tblOutput == null) {
+ outputFormat = org.apache.hadoop.mapred.SequenceFileOutputFormat.class.getName();
+ } else {
+ outputFormat = tblOutput;
+ }
+ }
+ schema.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_OUTPUT_FORMAT,
+ outputFormat);
+
+ // Location
+ if (sd.getLocation() != null) {
+ schema.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION,
+ sd.getLocation());
+ }
+
+ // Bucket count
+ schema.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_COUNT,
+ Integer.toString(sd.getNumBuckets()));
+
+ if (sd.getBucketCols() != null && sd.getBucketCols().size() > 0) {
+ schema.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_FIELD_NAME,
+ sd.getBucketCols().get(0));
+ }
+
+ // SerdeInfo
+ if (sd.getSerdeInfo() != null) {
+
+ // We should not update the following 3 values if SerDeInfo contains these.
+ // This is to keep backward compatible with getSchema(), where these 3 keys
+ // are updated after SerDeInfo properties got copied.
+ String cols = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS;
+ String colTypes = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES;
+ String parts = org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS;
+
+ for (Map.Entry<String,String> param : sd.getSerdeInfo().getParameters().entrySet()) {
+ String key = param.getKey();
+ if (schema.get(key) != null &&
+ (key.equals(cols) || key.equals(colTypes) || key.equals(parts))) {
+ continue;
+ }
+ schema.put(key, (param.getValue() != null) ? param.getValue() : StringUtils.EMPTY);
+ }
+
+ if (sd.getSerdeInfo().getSerializationLib() != null) {
+ schema.setProperty(ColumnType.SERIALIZATION_LIB, sd.getSerdeInfo().getSerializationLib());
+ }
+ }
+
+ // skipping columns since partition level field schemas are the same as table level's
+ // skipping partition keys since it is the same as table level partition keys
+
+ if (parameters != null) {
+ for (Map.Entry<String, String> e : parameters.entrySet()) {
+ schema.setProperty(e.getKey(), e.getValue());
+ }
+ }
+
+ return schema;
+ }
+
+ private static Properties addCols(Properties schema, List<FieldSchema> cols) {
+
+ StringBuilder colNameBuf = new StringBuilder();
+ StringBuilder colTypeBuf = new StringBuilder();
+ StringBuilder colComment = new StringBuilder();
+
+ boolean first = true;
+ String columnNameDelimiter = getColumnNameDelimiter(cols);
+ for (FieldSchema col : cols) {
+ if (!first) {
+ colNameBuf.append(columnNameDelimiter);
+ colTypeBuf.append(":");
+ colComment.append('\0');
+ }
+ colNameBuf.append(col.getName());
+ colTypeBuf.append(col.getType());
+ colComment.append((null != col.getComment()) ? col.getComment() : StringUtils.EMPTY);
+ first = false;
+ }
+ schema.setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS,
+ colNameBuf.toString());
+ schema.setProperty(ColumnType.COLUMN_NAME_DELIMITER, columnNameDelimiter);
+ String colTypes = colTypeBuf.toString();
+ schema.setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES,
+ colTypes);
+ schema.setProperty("columns.comments", colComment.toString());
+
+ return schema;
+
+ }
+
+ public static Properties getSchemaWithoutCols(StorageDescriptor sd,
+ Map<String, String> parameters, String databaseName, String tableName,
+ List<FieldSchema> partitionKeys) {
+ Properties schema = new Properties();
+ String inputFormat = sd.getInputFormat();
+ if (inputFormat == null || inputFormat.length() == 0) {
+ inputFormat = org.apache.hadoop.mapred.SequenceFileInputFormat.class
+ .getName();
+ }
+ schema.setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT,
+ inputFormat);
+ String outputFormat = sd.getOutputFormat();
+ if (outputFormat == null || outputFormat.length() == 0) {
+ outputFormat = org.apache.hadoop.mapred.SequenceFileOutputFormat.class
+ .getName();
+ }
+ schema.setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_OUTPUT_FORMAT,
+ outputFormat);
+
+ schema.setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME,
+ databaseName + "." + tableName);
+
+ if (sd.getLocation() != null) {
+ schema.setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION,
+ sd.getLocation());
+ }
+ schema.setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_COUNT, Integer
+ .toString(sd.getNumBuckets()));
+ if (sd.getBucketCols() != null && sd.getBucketCols().size() > 0) {
+ schema.setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.BUCKET_FIELD_NAME, sd
+ .getBucketCols().get(0));
+ }
+ if (sd.getSerdeInfo() != null) {
+ for (Map.Entry<String,String> param : sd.getSerdeInfo().getParameters().entrySet()) {
+ schema.put(param.getKey(), (param.getValue() != null) ? param.getValue() : StringUtils.EMPTY);
+ }
+
+ if (sd.getSerdeInfo().getSerializationLib() != null) {
+ schema.setProperty(ColumnType.SERIALIZATION_LIB, sd .getSerdeInfo().getSerializationLib());
+ }
+ }
+
+ if (sd.getCols() != null) {
+ schema.setProperty(ColumnType.SERIALIZATION_DDL, getDDLFromFieldSchema(tableName, sd.getCols()));
+ }
+
+ String partString = StringUtils.EMPTY;
+ String partStringSep = StringUtils.EMPTY;
+ String partTypesString = StringUtils.EMPTY;
+ String partTypesStringSep = StringUtils.EMPTY;
+ for (FieldSchema partKey : partitionKeys) {
+ partString = partString.concat(partStringSep);
+ partString = partString.concat(partKey.getName());
+ partTypesString = partTypesString.concat(partTypesStringSep);
+ partTypesString = partTypesString.concat(partKey.getType());
+ if (partStringSep.length() == 0) {
+ partStringSep = "/";
+ partTypesStringSep = ":";
+ }
+ }
+ if (partString.length() > 0) {
+ schema
+ .setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS,
+ partString);
+ schema
+ .setProperty(
+ org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES,
+ partTypesString);
+ }
+
+ if (parameters != null) {
+ for (Map.Entry<String, String> e : parameters.entrySet()) {
+ // add non-null parameters to the schema
+ if ( e.getValue() != null) {
+ schema.setProperty(e.getKey(), e.getValue());
+ }
+ }
+ }
+
+ return schema;
+ }
+
+ public static Properties getSchema(
+ org.apache.hadoop.hive.metastore.api.StorageDescriptor sd,
+ org.apache.hadoop.hive.metastore.api.StorageDescriptor tblsd,
+ Map<String, String> parameters, String databaseName, String tableName,
+ List<FieldSchema> partitionKeys) {
+
+ return addCols(getSchemaWithoutCols(sd, parameters, databaseName, tableName, partitionKeys), tblsd.getCols());
+ }
+
+ public static String getColumnNameDelimiter(List<FieldSchema> fieldSchemas) {
+ // we first take a look if any fieldSchemas contain COMMA
+ for (int i = 0; i < fieldSchemas.size(); i++) {
+ if (fieldSchemas.get(i).getName().contains(",")) {
+ return String.valueOf(ColumnType.COLUMN_COMMENTS_DELIMITER);
+ }
+ }
+ return String.valueOf(',');
+ }
+
+ /**
+ * Convert FieldSchemas to columnNames.
+ */
+ public static String getColumnNamesFromFieldSchema(List<FieldSchema> fieldSchemas) {
+ String delimiter = getColumnNameDelimiter(fieldSchemas);
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < fieldSchemas.size(); i++) {
+ if (i > 0) {
+ sb.append(delimiter);
+ }
+ sb.append(fieldSchemas.get(i).getName());
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Convert FieldSchemas to columnTypes.
+ */
+ public static String getColumnTypesFromFieldSchema(
+ List<FieldSchema> fieldSchemas) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < fieldSchemas.size(); i++) {
+ if (i > 0) {
+ sb.append(",");
+ }
+ sb.append(fieldSchemas.get(i).getType());
+ }
+ return sb.toString();
+ }
+
+ public static String getColumnCommentsFromFieldSchema(List<FieldSchema> fieldSchemas) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < fieldSchemas.size(); i++) {
+ if (i > 0) {
+ sb.append(ColumnType.COLUMN_COMMENTS_DELIMITER);
+ }
+ sb.append(fieldSchemas.get(i).getComment());
+ }
+ return sb.toString();
+ }
+
+ public static int startMetaStore() throws Exception {
+ return startMetaStore(HadoopThriftAuthBridge.getBridge(), null);
+ }
+
+ public static int startMetaStore(final HadoopThriftAuthBridge bridge, Configuration conf) throws
+ Exception {
+ int port = findFreePort();
+ startMetaStore(port, bridge, conf);
+ return port;
+ }
+
+ public static int startMetaStore(Configuration conf) throws Exception {
+ return startMetaStore(HadoopThriftAuthBridge.getBridge(), conf);
+ }
+
+ public static void startMetaStore(final int port, final HadoopThriftAuthBridge bridge) throws Exception {
+ startMetaStore(port, bridge, null);
+ }
+
+ public static void startMetaStore(final int port,
+ final HadoopThriftAuthBridge bridge, Configuration hiveConf)
+ throws Exception{
+ if (hiveConf == null) {
+ hiveConf = MetastoreConf.newMetastoreConf();
+ }
+ final Configuration finalHiveConf = hiveConf;
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ HiveMetaStore.startMetaStore(port, bridge, finalHiveConf);
+ } catch (Throwable e) {
+ LOG.error("Metastore Thrift Server threw an exception...",e);
+ }
+ }
+ });
+ thread.setDaemon(true);
+ thread.start();
+ loopUntilHMSReady(port);
+ }
+
+ /**
+ * A simple connect test to make sure that the metastore is up
+ * @throws Exception
+ */
+ private static void loopUntilHMSReady(int port) throws Exception {
+ int retries = 0;
+ Exception exc;
+ while (true) {
+ try {
+ Socket socket = new Socket();
+ socket.connect(new InetSocketAddress(port), 5000);
+ socket.close();
+ return;
+ } catch (Exception e) {
+ if (retries++ > 60) { //give up
+ exc = e;
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ }
+ // something is preventing metastore from starting
+ // print the stack from all threads for debugging purposes
+ LOG.error("Unable to connect to metastore server: " + exc.getMessage());
+ LOG.info("Printing all thread stack traces for debugging before throwing exception.");
+ LOG.info(getAllThreadStacksAsString());
+ throw exc;
+ }
+
+ private static String getAllThreadStacksAsString() {
+ Map<Thread, StackTraceElement[]> threadStacks = Thread.getAllStackTraces();
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<Thread, StackTraceElement[]> entry : threadStacks.entrySet()) {
+ Thread t = entry.getKey();
+ sb.append(System.lineSeparator());
+ sb.append("Name: ").append(t.getName()).append(" State: ").append(t.getState());
+ addStackString(entry.getValue(), sb);
+ }
+ return sb.toString();
+ }
+
+ private static void addStackString(StackTraceElement[] stackElems, StringBuilder sb) {
+ sb.append(System.lineSeparator());
+ for (StackTraceElement stackElem : stackElems) {
+ sb.append(stackElem).append(System.lineSeparator());
+ }
+ }
+
+ /**
+ * Finds a free port on the machine.
+ *
+ * @return
+ * @throws IOException
+ */
+ public static int findFreePort() throws IOException {
+ ServerSocket socket= new ServerSocket(0);
+ int port = socket.getLocalPort();
+ socket.close();
+ return port;
+ }
+
+ /**
+ * Finds a free port on the machine, but allow the
+ * ability to specify a port number to not use, no matter what.
+ */
+ public static int findFreePortExcepting(int portToExclude) throws IOException {
+ ServerSocket socket1 = null;
+ ServerSocket socket2 = null;
+ try {
+ socket1 = new ServerSocket(0);
+ socket2 = new ServerSocket(0);
+ if (socket1.getLocalPort() != portToExclude) {
+ return socket1.getLocalPort();
+ }
+ // If we're here, then socket1.getLocalPort was the port to exclude
+ // Since both sockets were open together at a point in time, we're
+ // guaranteed that socket2.getLocalPort() is not the same.
+ return socket2.getLocalPort();
+ } finally {
+ if (socket1 != null){
+ socket1.close();
+ }
+ if (socket2 != null){
+ socket2.close();
+ }
+ }
+ }
+
+ public static String getIndexTableName(String dbName, String baseTblName, String indexName) {
+ return dbName + "__" + baseTblName + "_" + indexName + "__";
+ }
+
+ public static boolean isMaterializedViewTable(Table table) {
+ if (table == null) {
+ return false;
+ }
+ return TableType.MATERIALIZED_VIEW.toString().equals(table.getTableType());
+ }
+
+ public static List<String> getColumnNames(List<FieldSchema> schema) {
+ List<String> cols = new ArrayList<>(schema.size());
+ for (FieldSchema fs : schema) {
+ cols.add(fs.getName());
+ }
+ return cols;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
index 41a18cb..0b0cfbd 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
@@ -36,11 +36,15 @@ import org.apache.zookeeper.client.ZooKeeperSaslClient;
import javax.security.auth.login.AppConfigurationEntry;
import org.apache.thrift.transport.TSSLTransportFactory;
import org.apache.thrift.transport.TServerSocket;
+import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLServerSocket;
+import javax.net.ssl.SSLSocket;
import javax.security.auth.login.LoginException;
import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
@@ -284,4 +288,26 @@ public class SecurityUtils {
}
return thriftServerSocket;
}
+
+ public static TTransport getSSLSocket(String host, int port, int loginTimeout,
+ String trustStorePath, String trustStorePassWord) throws TTransportException {
+ TSSLTransportFactory.TSSLTransportParameters params =
+ new TSSLTransportFactory.TSSLTransportParameters();
+ params.setTrustStore(trustStorePath, trustStorePassWord);
+ params.requireClientAuth(true);
+ // The underlying SSLSocket object is bound to host:port with the given SO_TIMEOUT and
+ // SSLContext created with the given params
+ TSocket tSSLSocket = TSSLTransportFactory.getClientSocket(host, port, loginTimeout, params);
+ return getSSLSocketWithHttps(tSSLSocket);
+ }
+
+ // Using endpoint identification algorithm as HTTPS enables us to do
+ // CNAMEs/subjectAltName verification
+ private static TSocket getSSLSocketWithHttps(TSocket tSSLSocket) throws TTransportException {
+ SSLSocket sslSocket = (SSLSocket) tSSLSocket.getSocket();
+ SSLParameters sslParams = sslSocket.getSSLParameters();
+ sslParams.setEndpointIdentificationAlgorithm("HTTPS");
+ sslSocket.setSSLParameters(sslParams);
+ return new TSocket(sslSocket);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/AlternateFailurePreListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/AlternateFailurePreListener.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/AlternateFailurePreListener.java
new file mode 100644
index 0000000..bde29a1
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/AlternateFailurePreListener.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import javax.jdo.JDOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.events.PreEventContext;
+
+/**
+ *
+ * AlternateFailurePreListener.
+ *
+ * An implementation of MetaStorePreEventListener which fails every other time it's invoked,
+ * starting with the first time.
+ *
+ * It also records and makes available the number of times it's been invoked.
+ */
+public class AlternateFailurePreListener extends MetaStorePreEventListener {
+
+ private static int callCount = 0;
+ private static boolean throwException = true;
+
+ public AlternateFailurePreListener(Configuration config) {
+ super(config);
+ }
+
+ @Override
+ public void onEvent(PreEventContext context) throws MetaException, NoSuchObjectException,
+ InvalidOperationException {
+
+ callCount++;
+ if (throwException) {
+ throwException = false;
+ throw new JDOException();
+ }
+
+ throwException = true;
+ }
+
+ public static int getCallCount() {
+ return callCount;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyEndFunctionListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyEndFunctionListener.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyEndFunctionListener.java
new file mode 100644
index 0000000..ca1f10b
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyEndFunctionListener.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+
+
+/** A dummy implementation for
+ * {@link org.apache.hadoop.hive.metastore.MetaStoreEndFunctionListener}
+ * for testing purposes.
+ */
+public class DummyEndFunctionListener extends MetaStoreEndFunctionListener{
+
+ public static final List<String> funcNameList = new ArrayList<>();
+ public static final List<MetaStoreEndFunctionContext> contextList =
+ new ArrayList<>();
+
+ public DummyEndFunctionListener(Configuration config) {
+ super(config);
+ }
+
+ @Override
+ public void onEndFunction(String functionName, MetaStoreEndFunctionContext context) {
+ funcNameList.add(functionName);
+ contextList.add(context);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyListener.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyListener.java
new file mode 100644
index 0000000..baecd12
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyListener.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.events.AddIndexEvent;
+import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterIndexEvent;
+import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
+import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent;
+import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent;
+import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.DropFunctionEvent;
+import org.apache.hadoop.hive.metastore.events.DropIndexEvent;
+import org.apache.hadoop.hive.metastore.events.DropPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.DropTableEvent;
+import org.apache.hadoop.hive.metastore.events.ListenerEvent;
+import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent;
+
+/** A dummy implementation for
+ * {@link org.apache.hadoop.hive.metastore.MetaStoreEventListener}
+ * for testing purposes.
+ */
+public class DummyListener extends MetaStoreEventListener{
+
+ public static final List<ListenerEvent> notifyList = new ArrayList<>();
+
+ /**
+ * @return The last event received, or null if no event was received.
+ */
+ public static ListenerEvent getLastEvent() {
+ if (notifyList.isEmpty()) {
+ return null;
+ } else {
+ return notifyList.get(notifyList.size() - 1);
+ }
+ }
+
+ public DummyListener(Configuration config) {
+ super(config);
+ }
+
+ @Override
+ public void onConfigChange(ConfigChangeEvent configChange) {
+ addEvent(configChange);
+ }
+
+ @Override
+ public void onAddPartition(AddPartitionEvent partition) throws MetaException {
+ addEvent(partition);
+ }
+
+ @Override
+ public void onCreateDatabase(CreateDatabaseEvent db) throws MetaException {
+ addEvent(db);
+ }
+
+ @Override
+ public void onCreateTable(CreateTableEvent table) throws MetaException {
+ addEvent(table);
+ }
+
+ @Override
+ public void onDropDatabase(DropDatabaseEvent db) throws MetaException {
+ addEvent(db);
+ }
+
+ @Override
+ public void onDropPartition(DropPartitionEvent partition) throws MetaException {
+ addEvent(partition);
+ }
+
+ @Override
+ public void onDropTable(DropTableEvent table) throws MetaException {
+ addEvent(table);
+ }
+
+ @Override
+ public void onAlterTable(AlterTableEvent event) throws MetaException {
+ addEvent(event);
+ }
+
+ @Override
+ public void onAlterPartition(AlterPartitionEvent event) throws MetaException {
+ addEvent(event);
+ }
+
+ @Override
+ public void onLoadPartitionDone(LoadPartitionDoneEvent partEvent) throws MetaException {
+ addEvent(partEvent);
+ }
+
+ @Override
+ public void onAddIndex(AddIndexEvent indexEvent) throws MetaException {
+ addEvent(indexEvent);
+ }
+
+ @Override
+ public void onDropIndex(DropIndexEvent indexEvent) throws MetaException {
+ addEvent(indexEvent);
+ }
+
+ @Override
+ public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException {
+ addEvent(indexEvent);
+ }
+
+ @Override
+ public void onCreateFunction (CreateFunctionEvent fnEvent) throws MetaException {
+ addEvent(fnEvent);
+ }
+
+ @Override
+ public void onDropFunction (DropFunctionEvent fnEvent) throws MetaException {
+ addEvent(fnEvent);
+ }
+
+ private void addEvent(ListenerEvent event) {
+ notifyList.add(event);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java
new file mode 100644
index 0000000..0f2a3c7
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+/*
+ * An implementation of MetaStoreInitListener to verify onInit is called when
+ * HMSHandler is initialized
+ */
+public class DummyMetaStoreInitListener extends MetaStoreInitListener{
+
+ public static boolean wasCalled = false;
+ public DummyMetaStoreInitListener(Configuration config) {
+ super(config);
+ }
+
+ @Override
+ public void onInit(MetaStoreInitContext context) throws MetaException {
+ wasCalled = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d79c4595/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyPreListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyPreListener.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyPreListener.java
new file mode 100644
index 0000000..0a68bac
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyPreListener.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.events.PreEventContext;
+
+/**
+ *
+ * DummyPreListener.
+ *
+ * An implementation of MetaStorePreEventListener which stores the Events it's seen in a list.
+ */
+public class DummyPreListener extends MetaStorePreEventListener {
+
+ public static final List<PreEventContext> notifyList = new ArrayList<>();
+
+ public DummyPreListener(Configuration config) {
+ super(config);
+ }
+
+ @Override
+ public void onEvent(PreEventContext context) throws MetaException, NoSuchObjectException,
+ InvalidOperationException {
+ notifyList.add(context);
+ }
+
+}