You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2012/09/04 11:22:33 UTC
svn commit: r1380500 - in /hive/trunk: ./ bin/ bin/ext/ eclipse-templates/
metastore/ metastore/src/java/org/apache/hadoop/hive/metastore/
metastore/src/java/org/apache/hadoop/hive/metastore/tools/
metastore/src/test/org/apache/hadoop/hive/metastore/
Author: cws
Date: Tue Sep 4 09:22:32 2012
New Revision: 1380500
URL: http://svn.apache.org/viewvc?rev=1380500&view=rev
Log:
HIVE-3056. Ability to bulk update location field in Db/Table/Partition records (Shreepadma Venugopalan via cws)
Added:
hive/trunk/bin/ext/metatool.sh (with props)
hive/trunk/bin/metatool
hive/trunk/eclipse-templates/TestHiveMetaTool.launchtemplate
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java (with props)
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java (with props)
Modified:
hive/trunk/build.xml
hive/trunk/metastore/ivy.xml
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
Added: hive/trunk/bin/ext/metatool.sh
URL: http://svn.apache.org/viewvc/hive/trunk/bin/ext/metatool.sh?rev=1380500&view=auto
==============================================================================
--- hive/trunk/bin/ext/metatool.sh (added)
+++ hive/trunk/bin/ext/metatool.sh Tue Sep 4 09:22:32 2012
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+THISSERVICE=metatool
+export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} "
+
+metatool () {
+
+ CLASS=org.apache.hadoop.hive.metastore.tools.HiveMetaTool
+ execHiveCmd $CLASS "$@"
+}
+
+metatool_help () {
+ CLASS=org.apache.hadoop.hive.metastore.tools.HiveMetaTool
+ execHiveCmd $CLASS "--help"
+}
Propchange: hive/trunk/bin/ext/metatool.sh
------------------------------------------------------------------------------
svn:eol-style = native
Added: hive/trunk/bin/metatool
URL: http://svn.apache.org/viewvc/hive/trunk/bin/metatool?rev=1380500&view=auto
==============================================================================
--- hive/trunk/bin/metatool (added)
+++ hive/trunk/bin/metatool Tue Sep 4 09:22:32 2012
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hive --service metatool "$@"
Modified: hive/trunk/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build.xml?rev=1380500&r1=1380499&r2=1380500&view=diff
==============================================================================
--- hive/trunk/build.xml (original)
+++ hive/trunk/build.xml Tue Sep 4 09:22:32 2012
@@ -419,6 +419,7 @@
<mkdir dir="${target.lib.dir}/py"/>
<mkdir dir="${target.lib.dir}/php"/>
<copy file="${hive.root}/bin/hive" todir="${target.bin.dir}"/>
+ <copy file="${hive.root}/bin/metatool" todir="${target.bin.dir}"/>
<echo message="metastore.excludes = ${metastore.excludes}"/>
<copy todir="${target.scripts.dir}/metastore/upgrade">
<fileset dir="${hive.root}/metastore/scripts/upgrade" excludes="${metastore.excludes}"/>
Added: hive/trunk/eclipse-templates/TestHiveMetaTool.launchtemplate
URL: http://svn.apache.org/viewvc/hive/trunk/eclipse-templates/TestHiveMetaTool.launchtemplate?rev=1380500&view=auto
==============================================================================
--- hive/trunk/eclipse-templates/TestHiveMetaTool.launchtemplate (added)
+++ hive/trunk/eclipse-templates/TestHiveMetaTool.launchtemplate Tue Sep 4 09:22:32 2012
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<launchConfiguration type="org.eclipse.jdt.junit.launchconfig">
+ <booleanAttribute key="org.eclipse.debug.core.appendEnvironmentVariables" value="false"/>
+ <mapAttribute key="org.eclipse.debug.core.environmentVariables">
+ <mapEntry key="JAVA_HOME" value="${system_property:java.home}"/>
+ <mapEntry key="HIVE_HADOOP_TEST_CLASSPATH" value="@HIVE_HADOOP_TEST_CLASSPATH@"/>
+ </mapAttribute>
+ <stringAttribute key="org.eclipse.jdt.junit.CONTAINER" value=""/>
+ <booleanAttribute key="org.eclipse.jdt.junit.KEEPRUNNING_ATTR" value="false"/>
+ <stringAttribute key="org.eclipse.jdt.junit.TESTNAME" value=""/>
+ <stringAttribute key="org.eclipse.jdt.junit.TEST_KIND" value="org.eclipse.jdt.junit.loader.junit3"/>
+ <listAttribute key="org.eclipse.jdt.launching.CLASSPATH">
+ <listEntry value="<?xml version="1.0" encoding="UTF-8"?> <runtimeClasspathEntry containerPath="org.eclipse.jdt.launching.JRE_CONTAINER" javaProject="@PROJECT@" path="1" type="4"/> "/>
+ <listEntry value="<?xml version="1.0" encoding="UTF-8"?> <runtimeClasspathEntry internalArchive="/@PROJECT@/build/metastore/hive-metastore-@HIVE_VERSION@.jar" path="3" type="2"/> "/>
+ <listEntry value="<?xml version="1.0" encoding="UTF-8"?> <runtimeClasspathEntry internalArchive="/@PROJECT@/metastore/src/model" path="3" type="2"/> "/>
+ <listEntry value="<?xml version="1.0" encoding="UTF-8"?> <runtimeClasspathEntry internalArchive="/@PROJECT@/data/conf" path="3" type="2"/> "/>
+ <listEntry value="<?xml version="1.0" encoding="UTF-8"?> <runtimeClasspathEntry id="org.eclipse.jdt.launching.classpathentry.defaultClasspath"> <memento exportedEntriesOnly="false" project="@PROJECT@"/> </runtimeClasspathEntry> "/>
+ <listEntry value="<?xml version="1.0" encoding="UTF-8"?> <runtimeClasspathEntry internalArchive="/@PROJECT@/conf" path="3" type="2"/> "/>
+ </listAttribute>
+ <booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>
+ <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.apache.hadoop.hive.metastore.TestHiveMetaTool"/>
+ <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@PROJECT@"/>
+ <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS"
+ value="@JVM_ARGS@ -Dhive.root.logger=INFO,console -Dhadoop.bin.path=@HADOOP_BIN_PATH@ -Dtest.tmp.dir="${workspace_loc:@PROJECT@}/build/ql/tmp" -Dtest.warehouse.dir="pfile://${workspace_loc:@PROJECT@}/build/test/data/warehouse" -Dbuild.dir="${workspace_loc:@PROJECT@}/build/ql" -Dbuild.dir.hive="${workspace_loc:@PROJECT@}/build" -Dversion="@HIVE_VERSION@""/>
+ <stringAttribute key="org.eclipse.jdt.launching.WORKING_DIRECTORY" value="${workspace_loc:@PROJECT@}/ql"/>
+</launchConfiguration>
Modified: hive/trunk/metastore/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/ivy.xml?rev=1380500&r1=1380499&r2=1380500&view=diff
==============================================================================
--- hive/trunk/metastore/ivy.xml (original)
+++ hive/trunk/metastore/ivy.xml Tue Sep 4 09:22:32 2012
@@ -45,7 +45,7 @@
<dependency org="commons-pool" name="commons-pool" rev="${commons-pool.version}"/>
<dependency org="org.datanucleus" name="datanucleus-connectionpool" rev="${datanucleus-connectionpool.version}"
transitive="false"/>
- <dependency org="org.datanucleus" name="datanucleus-core" rev="${datanucleus-core.version}"
+ <dependency org="org.datanucleus" name="datanucleus-core" rev="${datanucleus-core.version}"
transitive="false"/>
<dependency org="org.datanucleus" name="datanucleus-enhancer" rev="${datanucleus-enhancer.version}"
transitive="false"/>
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1380500&r1=1380499&r2=1380500&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Tue Sep 4 09:22:32 2012
@@ -20,6 +20,8 @@ package org.apache.hadoop.hive.metastore
import static org.apache.commons.lang.StringUtils.join;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -3974,6 +3976,285 @@ public class ObjectStore implements RawS
return join(storedVals,',');
}
+ /** The following API
+ *
+ * - executeJDOQLSelect
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ @SuppressWarnings("finally")
+ public Collection<?> executeJDOQLSelect(String query) {
+ boolean committed = false;
+ Collection<?> result = null;
+
+ LOG.info("Executing query: " + query);
+
+ try {
+ openTransaction();
+ Query q = pm.newQuery(query);
+ result = (Collection<?>) q.execute();
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ return null;
+ } else {
+ return result;
+ }
+ }
+ }
+
+ /** The following API
+ *
+ * - executeJDOQLUpdate
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ @SuppressWarnings("finally")
+ public long executeJDOQLUpdate(String query) {
+ boolean committed = false;
+ long numUpdated = 0;
+
+ LOG.info("Executing query: " + query);
+
+ try {
+ openTransaction();
+ Query q = pm.newQuery(query);
+ numUpdated = (Long) q.execute();
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ return -1;
+ } else {
+ return numUpdated;
+ }
+ }
+ }
+
+ /** The following API
+ *
+ * - listFSRoots
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ @SuppressWarnings("finally")
+ public Set<String> listFSRoots() {
+ boolean committed = false;
+ Set<String> fsRoots = new HashSet<String>();
+
+ try {
+ openTransaction();
+ Query query = pm.newQuery(MDatabase.class);
+ List<MDatabase> mDBs = (List<MDatabase>) query.execute();
+ pm.retrieveAll(mDBs);
+
+ for (MDatabase mDB:mDBs) {
+ fsRoots.add(mDB.getLocationUri());
+ }
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ return null;
+ } else {
+ return fsRoots;
+ }
+ }
+ }
+
+ private boolean shouldUpdateURI(URI onDiskUri, URI inputUri) {
+ String onDiskHost = onDiskUri.getHost();
+ String inputHost = inputUri.getHost();
+
+ int onDiskPort = onDiskUri.getPort();
+ int inputPort = inputUri.getPort();
+
+ String onDiskScheme = onDiskUri.getScheme();
+ String inputScheme = inputUri.getScheme();
+
+ //compare ports
+ if (inputPort != -1) {
+ if (inputPort != onDiskPort) {
+ return false;
+ }
+ }
+ //compare schemes
+ if (inputScheme != null) {
+ if (onDiskScheme == null) {
+ return false;
+ }
+ if (!inputScheme.equalsIgnoreCase(onDiskScheme)) {
+ return false;
+ }
+ }
+ //compare hosts
+ if (onDiskHost != null) {
+ if (!inputHost.equalsIgnoreCase(onDiskHost)) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ return true;
+ }
+
+ private int updateMDatabaseURI(URI oldLoc, URI newLoc,
+ HashMap<String, String> updateLocations, boolean dryRun) {
+ int count = 0;
+ Query query = pm.newQuery(MDatabase.class);
+ List<MDatabase> mDBs = (List<MDatabase>) query.execute();
+ pm.retrieveAll(mDBs);
+
+ LOG.info("Looking for location in DB_LOCATION_URI field in DBS table...");
+
+ for(MDatabase mDB:mDBs) {
+ URI locationURI = null;
+ try {
+ locationURI = new URI(mDB.getLocationUri());
+ } catch(URISyntaxException e) {
+ LOG.error("Encountered error while validating location URI"
+ + e.getLocalizedMessage());
+ }
+ // locationURI is a valid URI
+ if (locationURI != null) {
+ if (shouldUpdateURI(locationURI, oldLoc)) {
+ String dbLoc = mDB.getLocationUri().replaceAll(oldLoc.toString(), newLoc.toString());
+ if (dryRun) {
+ updateLocations.put(locationURI.toString(), dbLoc);
+ } else {
+ mDB.setLocationUri(dbLoc);
+ }
+ count++;
+ }
+ }
+ }
+
+ LOG.info("Found " + count + " records to update");
+ return count;
+ }
+
+ private int updateMStorageDescriptorURI(URI oldLoc, URI newLoc,
+ HashMap<String, String> updateLocations, boolean dryRun) {
+ int count = 0;
+ Query query = pm.newQuery(MStorageDescriptor.class);
+ List<MStorageDescriptor> mSDSs = (List<MStorageDescriptor>) query.execute();
+ pm.retrieveAll(mSDSs);
+
+ LOG.info("Looking for location in LOCATION field in SDS table...");
+
+ for(MStorageDescriptor mSDS:mSDSs) {
+ URI locationURI = null;
+ try {
+ locationURI = new URI(mSDS.getLocation());
+ } catch (URISyntaxException e) {
+ LOG.error("Encountered error while validating location URI"
+ + e.getLocalizedMessage());
+ }
+ // locationURI is a valid URI
+ if (locationURI != null) {
+ if (shouldUpdateURI(locationURI, oldLoc)) {
+ String tblLoc = mSDS.getLocation().replaceAll(oldLoc.toString(), newLoc.toString());
+ if (dryRun) {
+ updateLocations.put(locationURI.toString(), tblLoc);
+ } else {
+ mSDS.setLocation(tblLoc);
+ }
+ count++;
+ }
+ }
+ }
+
+ LOG.info("Found " + count + " records to update");
+ return count;
+ }
+
+ private int updateAvroSerdeURI(URI oldLoc, URI newLoc,
+ HashMap<String, String> updateLocations, boolean dryRun) {
+ int count = 0;
+ Query query = pm.newQuery(MSerDeInfo.class);
+ List<MSerDeInfo> mSerdes = (List<MSerDeInfo>) query.execute();
+ pm.retrieveAll(mSerdes);
+
+ LOG.info("Looking for location in the value field of schema.url key in SERDES table...");
+
+ for(MSerDeInfo mSerde:mSerdes) {
+ String key = new String("schema.url");
+ String schemaLoc = mSerde.getParameters().get(key);
+ if (schemaLoc != null) {
+ URI schemaLocURI = null;
+ try {
+ schemaLocURI = new URI(schemaLoc);
+ } catch (URISyntaxException e) {
+ LOG.error("Encountered error while validating location URI"
+ + e.getLocalizedMessage());
+ }
+ // schemaLocURI is a valid URI
+ if (schemaLocURI != null) {
+ if (shouldUpdateURI(schemaLocURI, oldLoc)) {
+ String newSchemaLoc = schemaLoc.replaceAll(oldLoc.toString(), newLoc.toString());
+ if (dryRun) {
+ updateLocations.put(schemaLocURI.toString(), newSchemaLoc);
+ } else {
+ mSerde.getParameters().put(key, newSchemaLoc);
+ }
+ count++;
+ }
+ }
+ }
+ }
+
+ LOG.info("Found " + count + " records to update");
+ return count;
+ }
+
+ /** The following APIs
+ *
+ * - updateFSRootLocation
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ @SuppressWarnings("finally")
+ public int updateFSRootLocation(URI oldLoc, URI newLoc,
+ HashMap<String, String> updateLocations, boolean dryRun) {
+ boolean committed = false;
+ int count = 0;
+ int totalCount = 0;
+
+ LOG.info("Old FS root location: " + oldLoc.toString() +
+ " New FS root location: " + newLoc.toString());
+ LOG.info("Updating FS root location...");
+
+ try {
+ openTransaction();
+
+ // update locationURI in mDatabase
+ count = updateMDatabaseURI(oldLoc, newLoc, updateLocations, dryRun);
+ totalCount += count;
+
+ // update location in mStorageDescriptor
+ count = updateMStorageDescriptorURI(oldLoc, newLoc, updateLocations, dryRun);
+ totalCount += count;
+
+ // upgrade schema.url for avro serde
+ count = updateAvroSerdeURI(oldLoc, newLoc, updateLocations, dryRun);
+ totalCount += count;
+
+ committed = commitTransaction();
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ return -1;
+ } else {
+ return totalCount;
+ }
+ }
+ }
+
@Override
public long cleanupEvents() {
boolean commited = false;
Added: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java?rev=1380500&view=auto
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java (added)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java Tue Sep 4 09:22:32 2012
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.tools;
+
+import java.net.URI;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.ObjectStore;
+
+/**
+ * This class provides Hive admins a tool to
+ * - execute JDOQL against the metastore using DataNucleus
+ * - perform HA name node upgrade
+ */
+
+public class HiveMetaTool {
+
+ private static final Log LOG = LogFactory.getLog(HiveMetaTool.class.getName());
+ private final Options cmdLineOptions = new Options();
+ private ObjectStore objStore;
+
+
+ public HiveMetaTool() {
+ }
+
+ @SuppressWarnings("static-access")
+ private void init() {
+
+ System.out.println("Initializing HiveMetaTool..");
+
+ Option help = new Option("help", "print this message");
+ Option listFSRoot = new Option("listFSRoot", "print the current FS root locations");
+ Option executeJDOQL =
+ OptionBuilder.withArgName("query-string")
+ .hasArgs()
+ .withDescription("execute the given JDOQL query")
+ .create("executeJDOQL");
+
+ /* Ideally we want to specify the different arguments to updateLocation as separate argNames.
+ * However if we did that, HelpFormatter swallows all but the last argument. Note that this is
+ * a know issue with the HelpFormatter class that has not been fixed. We specify all arguments
+ * with a single argName to workaround this HelpFormatter bug.
+ */
+ Option updateFSRootLoc =
+ OptionBuilder
+ .withArgName("new-loc> " + "<old-loc")
+ .hasArgs(3)
+ .withDescription(
+ "update FS root location in the metastore to new location. Both new-loc and" +
+ " old-loc should be valid URIs with valid host names and schemes. " +
+ "when run with the dryRun option changes are displayed but are not persisted.")
+ .create("updateLocation");
+ Option dryRun = new Option("dryRun" , "dryRun is valid only with updateLocation option. when " +
+ "run with the dryRun option updateLocation changes are displayed but are not persisted.");
+
+ cmdLineOptions.addOption(help);
+ cmdLineOptions.addOption(listFSRoot);
+ cmdLineOptions.addOption(executeJDOQL);
+ cmdLineOptions.addOption(updateFSRootLoc);
+ cmdLineOptions.addOption(dryRun);
+
+ }
+
+ private void initObjectStore(HiveConf hiveConf) {
+ objStore = new ObjectStore();
+ objStore.setConf(hiveConf);
+ }
+
+ private void shutdownObjectStore() {
+ if (objStore != null) {
+ objStore.shutdown();
+ }
+ }
+
+ private void listFSRoot() {
+ Set<String> hdfsRoots = objStore.listFSRoots();
+ if (hdfsRoots != null) {
+ System.out.println("HiveMetaTool:Listing FS Roots..");
+ for (String s : hdfsRoots) {
+ System.out.println(s);
+ }
+ } else {
+ System.err.println("HiveMetaTool:Encountered error during listFSRoot - " +
+ "commit of JDO transaction failed");
+ }
+ }
+
+ private void executeJDOQLSelect(String query) {
+ Collection<?> result = objStore.executeJDOQLSelect(query);
+ if (result != null) {
+ Iterator<?> iter = result.iterator();
+ while (iter.hasNext()) {
+ Object o = iter.next();
+ System.out.println(o.toString());
+ }
+ } else {
+ System.err.println("HiveMetaTool:Encountered error during executeJDOQLSelect -" +
+ "commit of JDO transaction failed.");
+ }
+ }
+
+ private void executeJDOQLUpdate(String query) {
+ long numUpdated = objStore.executeJDOQLUpdate(query);
+ if (numUpdated >= 0) {
+ System.out.println("HiveMetaTool:Number of records updated: " + numUpdated);
+ } else {
+ System.err.println("HiveMetaTool:Encountered error during executeJDOQL -" +
+ "commit of JDO transaction failed.");
+ }
+ }
+
+ private void printUpdateLocations(HashMap<String, String> updateLocations) {
+ for (String key: updateLocations.keySet()) {
+ String value = updateLocations.get(key);
+ System.out.println("current location: " + key + " new location: " + value);
+ }
+ }
+
+ private void updateFSRootLocation(URI oldURI, URI newURI, boolean dryRun) {
+ HashMap<String, String> updateLocations = new HashMap<String, String>();
+ int count = objStore.updateFSRootLocation(oldURI, newURI, updateLocations, dryRun);
+ if (count == -1) {
+ System.err.println("HiveMetaTool:Encountered error while executing updateFSRootLocation - " +
+ "commit of JDO transaction failed, failed to update FS Root locations.");
+ } else {
+ if (!dryRun) {
+ System.out.println("HiveMetaTool: Successfully updated " + count + "FS Root locations");
+ } else {
+ printUpdateLocations(updateLocations);
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+
+ HiveConf hiveConf = new HiveConf(HiveMetaTool.class);
+ HiveMetaTool metaTool = new HiveMetaTool();
+ metaTool.init();
+ CommandLineParser parser = new GnuParser();
+ CommandLine line = null;
+
+ try {
+ try {
+ line = parser.parse(metaTool.cmdLineOptions, args);
+ } catch (ParseException e) {
+ System.err.println("HiveMetaTool:Parsing failed. Reason: " + e.getLocalizedMessage());
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("metatool", metaTool.cmdLineOptions);
+ System.exit(1);
+ }
+
+ if (line.hasOption("help")) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("metatool", metaTool.cmdLineOptions);
+ } else if (line.hasOption("listFSRoot")) {
+ metaTool.initObjectStore(hiveConf);
+ metaTool.listFSRoot();
+ } else if (line.hasOption("executeJDOQL")) {
+ String query = line.getOptionValue("executeJDOQL");
+ metaTool.initObjectStore(hiveConf);
+ if (query.toLowerCase().trim().startsWith("select")) {
+ metaTool.executeJDOQLSelect(query);
+ } else if (query.toLowerCase().trim().startsWith("update")) {
+ metaTool.executeJDOQLUpdate(query);
+ } else {
+ System.err.println("HiveMetaTool:Unsupported statement type");
+ }
+ } else if (line.hasOption("updateLocation")) {
+ String[] loc = line.getOptionValues("updateLocation");
+ boolean dryRun = false;
+
+ if (loc.length != 2 && loc.length != 3) {
+ System.err.println("HiveMetaTool:updateLocation takes in 2 required and 1 " +
+ "optional arguements but " +
+ "was passed " + loc.length + " arguements");
+ }
+
+ Path newPath = new Path(loc[0]);
+ Path oldPath = new Path(loc[1]);
+
+ URI oldURI = oldPath.toUri();
+ URI newURI = newPath.toUri();
+
+ if (line.hasOption("dryRun")) {
+ dryRun = true;
+ }
+
+ /*
+ * validate input - if the old uri contains a valid port, the new uri
+ * should contain a valid port as well. Both new and old uri should
+ * contain valid host names and valid schemes.
+ */
+ if (oldURI.getHost() == null || newURI.getHost() == null) {
+ System.err.println("HiveMetaTool:A valid host is required in both old-loc and new-loc");
+ } else if (oldURI.getPort() > 0 && newURI.getPort() < 0) {
+ System.err.println("HiveMetaTool:old-loc has a valid port, new-loc should " +
+ "also contain a valid port");
+ } else if (oldURI.getScheme() == null || newURI.getScheme() == null) {
+ System.err.println("HiveMetaTool:A valid scheme is required in both old-loc and new-loc");
+ } else {
+ metaTool.initObjectStore(hiveConf);
+ metaTool.updateFSRootLocation(oldURI, newURI, dryRun);
+ }
+ } else {
+ System.err.print("HiveMetaTool:Invalid option:" + line.getOptions());
+ for (String s : line.getArgs()) {
+ System.err.print(s + " ");
+ }
+ System.err.println();
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("metatool", metaTool.cmdLineOptions);
+ }
+ } finally {
+ metaTool.shutdownObjectStore();
+ }
+ }
+}
Propchange: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java?rev=1380500&r1=1380499&r2=1380500&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java Tue Sep 4 09:22:32 2012
@@ -1573,161 +1573,163 @@ public abstract class TestHiveMetaStore
* Tests for list partition by filter functionality.
* @throws Exception
*/
+
public void testPartitionFilter() throws Exception {
- String dbName = "filterdb";
- String tblName = "filtertbl";
+ String dbName = "filterdb";
+ String tblName = "filtertbl";
- List<String> vals = new ArrayList<String>(3);
- vals.add("p11");
- vals.add("p21");
- vals.add("p31");
- List <String> vals2 = new ArrayList<String>(3);
- vals2.add("p11");
- vals2.add("p22");
- vals2.add("p31");
- List <String> vals3 = new ArrayList<String>(3);
- vals3.add("p12");
- vals3.add("p21");
- vals3.add("p31");
- List <String> vals4 = new ArrayList<String>(3);
- vals4.add("p12");
- vals4.add("p23");
- vals4.add("p31");
- List <String> vals5 = new ArrayList<String>(3);
- vals5.add("p13");
- vals5.add("p24");
- vals5.add("p31");
- List <String> vals6 = new ArrayList<String>(3);
- vals6.add("p13");
- vals6.add("p25");
- vals6.add("p31");
+ List<String> vals = new ArrayList<String>(3);
+ vals.add("p11");
+ vals.add("p21");
+ vals.add("p31");
+ List <String> vals2 = new ArrayList<String>(3);
+ vals2.add("p11");
+ vals2.add("p22");
+ vals2.add("p31");
+ List <String> vals3 = new ArrayList<String>(3);
+ vals3.add("p12");
+ vals3.add("p21");
+ vals3.add("p31");
+ List <String> vals4 = new ArrayList<String>(3);
+ vals4.add("p12");
+ vals4.add("p23");
+ vals4.add("p31");
+ List <String> vals5 = new ArrayList<String>(3);
+ vals5.add("p13");
+ vals5.add("p24");
+ vals5.add("p31");
+ List <String> vals6 = new ArrayList<String>(3);
+ vals6.add("p13");
+ vals6.add("p25");
+ vals6.add("p31");
- silentDropDatabase(dbName);
+ silentDropDatabase(dbName);
- Database db = new Database();
- db.setName(dbName);
- client.createDatabase(db);
+ Database db = new Database();
+ db.setName(dbName);
+ client.createDatabase(db);
- ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2);
- cols.add(new FieldSchema("c1", Constants.STRING_TYPE_NAME, ""));
- cols.add(new FieldSchema("c2", Constants.INT_TYPE_NAME, ""));
+ ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2);
+ cols.add(new FieldSchema("c1", Constants.STRING_TYPE_NAME, ""));
+ cols.add(new FieldSchema("c2", Constants.INT_TYPE_NAME, ""));
- ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(3);
- partCols.add(new FieldSchema("p1", Constants.STRING_TYPE_NAME, ""));
- partCols.add(new FieldSchema("p2", Constants.STRING_TYPE_NAME, ""));
- partCols.add(new FieldSchema("p3", Constants.INT_TYPE_NAME, ""));
+ ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(3);
+ partCols.add(new FieldSchema("p1", Constants.STRING_TYPE_NAME, ""));
+ partCols.add(new FieldSchema("p2", Constants.STRING_TYPE_NAME, ""));
+ partCols.add(new FieldSchema("p3", Constants.INT_TYPE_NAME, ""));
- Table tbl = new Table();
- tbl.setDbName(dbName);
- tbl.setTableName(tblName);
- StorageDescriptor sd = new StorageDescriptor();
- tbl.setSd(sd);
- sd.setCols(cols);
- sd.setCompressed(false);
- sd.setNumBuckets(1);
- sd.setParameters(new HashMap<String, String>());
- sd.setBucketCols(new ArrayList<String>());
- sd.setSerdeInfo(new SerDeInfo());
- sd.getSerdeInfo().setName(tbl.getTableName());
- sd.getSerdeInfo().setParameters(new HashMap<String, String>());
- sd.getSerdeInfo().getParameters()
- .put(Constants.SERIALIZATION_FORMAT, "1");
- sd.setSortCols(new ArrayList<Order>());
+ Table tbl = new Table();
+ tbl.setDbName(dbName);
+ tbl.setTableName(tblName);
+ StorageDescriptor sd = new StorageDescriptor();
+ tbl.setSd(sd);
+ sd.setCols(cols);
+ sd.setCompressed(false);
+ sd.setNumBuckets(1);
+ sd.setParameters(new HashMap<String, String>());
+ sd.setBucketCols(new ArrayList<String>());
+ sd.setSerdeInfo(new SerDeInfo());
+ sd.getSerdeInfo().setName(tbl.getTableName());
+ sd.getSerdeInfo().setParameters(new HashMap<String, String>());
+ sd.getSerdeInfo().getParameters()
+ .put(Constants.SERIALIZATION_FORMAT, "1");
+ sd.setSortCols(new ArrayList<Order>());
- tbl.setPartitionKeys(partCols);
- client.createTable(tbl);
+ tbl.setPartitionKeys(partCols);
+ client.createTable(tbl);
- tbl = client.getTable(dbName, tblName);
+ tbl = client.getTable(dbName, tblName);
- add_partition(client, tbl, vals, "part1");
- add_partition(client, tbl, vals2, "part2");
- add_partition(client, tbl, vals3, "part3");
- add_partition(client, tbl, vals4, "part4");
- add_partition(client, tbl, vals5, "part5");
- add_partition(client, tbl, vals6, "part6");
+ add_partition(client, tbl, vals, "part1");
+ add_partition(client, tbl, vals2, "part2");
+ add_partition(client, tbl, vals3, "part3");
+ add_partition(client, tbl, vals4, "part4");
+ add_partition(client, tbl, vals5, "part5");
+ add_partition(client, tbl, vals6, "part6");
+
+ checkFilter(client, dbName, tblName, "p1 = \"p11\"", 2);
+ checkFilter(client, dbName, tblName, "p1 = \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p2 = \"p21\"", 2);
+ checkFilter(client, dbName, tblName, "p2 = \"p23\"", 1);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" and p2=\"p22\"", 1);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" or p2=\"p23\"", 3);
+ checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4);
+
+ checkFilter(client, dbName, tblName,
+ "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\")", 3);
+ checkFilter(client, dbName, tblName,
+ "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\") Or " +
+ "(p1=\"p13\" aNd p2=\"p24\")", 4);
+ //test for and or precedence
+ checkFilter(client, dbName, tblName,
+ "p1=\"p12\" and (p2=\"p27\" Or p2=\"p21\")", 1);
+ checkFilter(client, dbName, tblName,
+ "p1=\"p12\" and p2=\"p27\" Or p2=\"p21\"", 2);
+
+ checkFilter(client, dbName, tblName, "p1 > \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p1 >= \"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 < \"p12\"", 2);
+ checkFilter(client, dbName, tblName, "p1 <= \"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 <> \"p12\"", 4);
+ checkFilter(client, dbName, tblName, "p1 like \"p1.*\"", 6);
+ checkFilter(client, dbName, tblName, "p2 like \"p.*3\"", 1);
- checkFilter(client, dbName, tblName, "p1 = \"p11\"", 2);
- checkFilter(client, dbName, tblName, "p1 = \"p12\"", 2);
- checkFilter(client, dbName, tblName, "p2 = \"p21\"", 2);
- checkFilter(client, dbName, tblName, "p2 = \"p23\"", 1);
- checkFilter(client, dbName, tblName, "p1 = \"p11\" and p2=\"p22\"", 1);
- checkFilter(client, dbName, tblName, "p1 = \"p11\" or p2=\"p23\"", 3);
- checkFilter(client, dbName, tblName, "p1 = \"p11\" or p1=\"p12\"", 4);
-
- checkFilter(client, dbName, tblName,
- "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\")", 3);
- checkFilter(client, dbName, tblName,
- "p1 = \"p11\" or (p1=\"p12\" and p2=\"p21\") Or " +
- "(p1=\"p13\" aNd p2=\"p24\")", 4);
- //test for and or precedence
- checkFilter(client, dbName, tblName,
- "p1=\"p12\" and (p2=\"p27\" Or p2=\"p21\")", 1);
- checkFilter(client, dbName, tblName,
- "p1=\"p12\" and p2=\"p27\" Or p2=\"p21\"", 2);
-
- checkFilter(client, dbName, tblName, "p1 > \"p12\"", 2);
- checkFilter(client, dbName, tblName, "p1 >= \"p12\"", 4);
- checkFilter(client, dbName, tblName, "p1 < \"p12\"", 2);
- checkFilter(client, dbName, tblName, "p1 <= \"p12\"", 4);
- checkFilter(client, dbName, tblName, "p1 <> \"p12\"", 4);
- checkFilter(client, dbName, tblName, "p1 like \"p1.*\"", 6);
- checkFilter(client, dbName, tblName, "p2 like \"p.*3\"", 1);
-
- //Test for setting the maximum partition count
- List<Partition> partitions = client.listPartitionsByFilter(dbName,
- tblName, "p1 >= \"p12\"", (short) 2);
- assertEquals("User specified row limit for partitions",
- 2, partitions.size());
+ //Test for setting the maximum partition count
+ List<Partition> partitions = client.listPartitionsByFilter(dbName,
+ tblName, "p1 >= \"p12\"", (short) 2);
+ assertEquals("User specified row limit for partitions",
+ 2, partitions.size());
- //Negative tests
- Exception me = null;
- try {
- client.listPartitionsByFilter(dbName,
- tblName, "p3 >= \"p12\"", (short) -1);
- } catch(MetaException e) {
- me = e;
- }
- assertNotNull(me);
- assertTrue("Filter on int partition key", me.getMessage().contains(
- "Filtering is supported only on partition keys of type string"));
+ //Negative tests
+ Exception me = null;
+ try {
+ client.listPartitionsByFilter(dbName,
+ tblName, "p3 >= \"p12\"", (short) -1);
+ } catch(MetaException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("Filter on int partition key", me.getMessage().contains(
+ "Filtering is supported only on partition keys of type string"));
- me = null;
- try {
- client.listPartitionsByFilter(dbName,
- tblName, "c1 >= \"p12\"", (short) -1);
- } catch(MetaException e) {
- me = e;
- }
- assertNotNull(me);
- assertTrue("Filter on invalid key", me.getMessage().contains(
- "<c1> is not a partitioning key for the table"));
+ me = null;
+ try {
+ client.listPartitionsByFilter(dbName,
+ tblName, "c1 >= \"p12\"", (short) -1);
+ } catch(MetaException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("Filter on invalid key", me.getMessage().contains(
+ "<c1> is not a partitioning key for the table"));
- me = null;
- try {
- client.listPartitionsByFilter(dbName,
- tblName, "c1 >= ", (short) -1);
- } catch(MetaException e) {
- me = e;
- }
- assertNotNull(me);
- assertTrue("Invalid filter string", me.getMessage().contains(
- "Error parsing partition filter"));
+ me = null;
+ try {
+ client.listPartitionsByFilter(dbName,
+ tblName, "c1 >= ", (short) -1);
+ } catch(MetaException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("Invalid filter string", me.getMessage().contains(
+ "Error parsing partition filter"));
- me = null;
- try {
- client.listPartitionsByFilter("invDBName",
- "invTableName", "p1 = \"p11\"", (short) -1);
- } catch(NoSuchObjectException e) {
- me = e;
- }
- assertNotNull(me);
- assertTrue("NoSuchObject exception", me.getMessage().contains(
- "database/table does not exist"));
+ me = null;
+ try {
+ client.listPartitionsByFilter("invDBName",
+ "invTableName", "p1 = \"p11\"", (short) -1);
+ } catch(NoSuchObjectException e) {
+ me = e;
+ }
+ assertNotNull(me);
+ assertTrue("NoSuchObject exception", me.getMessage().contains(
+ "database/table does not exist"));
- client.dropTable(dbName, tblName);
- client.dropDatabase(dbName);
+ client.dropTable(dbName, tblName);
+ client.dropDatabase(dbName);
}
+
/**
* Test filtering on table with single partition
* @throws Exception
@@ -2299,5 +2301,4 @@ public abstract class TestHiveMetaStore
createPartitions(dbName, tbl, values);
}
-
}
Added: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java?rev=1380500&view=auto
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java (added)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java Tue Sep 4 09:22:32 2012
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.Type;
+import org.apache.hadoop.hive.metastore.tools.HiveMetaTool;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.util.StringUtils;
+
+public class TestHiveMetaTool extends TestCase {
+
+ private HiveMetaStoreClient client;
+
+ private PrintStream originalOut;
+ private OutputStream os;
+ private PrintStream ps;
+ private String locationUri;
+
+
+ private void dropDatabase(String dbName) throws Exception {
+ try {
+ client.dropDatabase(dbName);
+ } catch (NoSuchObjectException e) {
+ } catch (InvalidOperationException e) {
+ } catch (Exception e) {
+ throw e;
+ }
+ }
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+
+ try {
+ HiveConf hiveConf = new HiveConf(HiveMetaTool.class);
+ client = new HiveMetaStoreClient(hiveConf, null);
+
+ // Setup output stream to redirect output to
+ os = new ByteArrayOutputStream();
+ ps = new PrintStream(os);
+
+ // create a dummy database and a couple of dummy tables
+ String dbName = "testDB";
+ String typeName = "Person";
+ String tblName = "simpleTbl";
+
+ Database db = new Database();
+ db.setName(dbName);
+ client.dropTable(dbName, tblName);
+ dropDatabase(dbName);
+ client.createDatabase(db);
+ locationUri = db.getLocationUri();
+
+ client.dropType(typeName);
+ Type typ1 = new Type();
+ typ1.setName(typeName);
+ typ1.setFields(new ArrayList<FieldSchema>(2));
+ typ1.getFields().add(
+ new FieldSchema("name", Constants.STRING_TYPE_NAME, ""));
+ typ1.getFields().add(
+ new FieldSchema("income", Constants.INT_TYPE_NAME, ""));
+ client.createType(typ1);
+
+ Table tbl = new Table();
+ tbl.setDbName(dbName);
+ tbl.setTableName(tblName);
+ StorageDescriptor sd = new StorageDescriptor();
+ tbl.setSd(sd);
+ sd.setCols(typ1.getFields());
+ sd.setCompressed(false);
+ sd.setNumBuckets(1);
+ sd.setParameters(new HashMap<String, String>());
+ sd.getParameters().put("test_param_1", "Use this for comments etc");
+ sd.setBucketCols(new ArrayList<String>(2));
+ sd.getBucketCols().add("name");
+ sd.setSerdeInfo(new SerDeInfo());
+ sd.getSerdeInfo().setName(tbl.getTableName());
+ sd.getSerdeInfo().setParameters(new HashMap<String, String>());
+ sd.getSerdeInfo().getParameters().put(
+ org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
+ sd.getSerdeInfo().setSerializationLib(
+ org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
+ tbl.setPartitionKeys(new ArrayList<FieldSchema>());
+
+ client.createTable(tbl);
+ client.close();
+ } catch (Exception e) {
+ System.err.println("Unable to setup the hive metatool test");
+ System.err.println(StringUtils.stringifyException(e));
+ throw new Exception(e);
+ }
+ }
+
+ private void redirectOutputStream() {
+
+ originalOut = System.out;
+ System.setOut(ps);
+
+ }
+
+ private void restoreOutputStream() {
+
+ System.setOut(originalOut);
+ }
+
+ public void testListFSRoot() throws Exception {
+
+ redirectOutputStream();
+ String[] args = new String[1];
+ args[0] = new String("-listFSRoot");
+
+ try {
+ HiveMetaTool.main(args);
+ String out = os.toString();
+ boolean b = out.contains(locationUri);
+ assertTrue(b);
+ } finally {
+ restoreOutputStream();
+ System.out.println("Completed testListFSRoot");
+ }
+ }
+
+ public void testExecuteJDOQL() throws Exception {
+
+ redirectOutputStream();
+ String[] args = new String[2];
+ args[0] = new String("-executeJDOQL");
+ args[1] = new String("select locationUri from org.apache.hadoop.hive.metastore.model.MDatabase");
+
+ try {
+ HiveMetaTool.main(args);
+ String out = os.toString();
+ boolean b = out.contains(locationUri);
+ assertTrue(b);
+ } finally {
+ restoreOutputStream();
+ System.out.println("Completed testExecuteJDOQL");
+ }
+ }
+
+ public void testUpdateFSRootLocation() throws Exception {
+
+ redirectOutputStream();
+ String newLocationUri = "hdfs://nn-ha-uri/user/hive/warehouse";
+ String[] args = new String[3];
+ args[0] = new String("-updateLocation");
+ args[1] = new String(newLocationUri);
+ args[2] = new String(locationUri);
+
+ String[] args2 = new String[1];
+ args2[0] = new String("-listFSRoot");
+
+ try {
+
+ // perform HA upgrade
+ HiveMetaTool.main(args);
+
+ // obtain new HDFS root
+ HiveMetaTool.main(args2);
+
+ String out = os.toString();
+ boolean b = out.contains(newLocationUri);
+
+ if (b) {
+ System.out.println("updateFSRootLocation successful");
+ } else {
+ System.out.println("updateFSRootLocation failed");
+ }
+ // restore the original HDFS root if needed
+ if (b) {
+ args[1] = new String(locationUri);
+ args[2] = new String(newLocationUri);
+ HiveMetaTool.main(args);
+ }
+ } finally {
+ restoreOutputStream();
+ System.out.println("Completed testUpdateFSRootLocation..");
+ }
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ try {
+ super.tearDown();
+
+ } catch (Throwable e) {
+ System.err.println("Unable to close metastore");
+ System.err.println(StringUtils.stringifyException(e));
+ throw new Exception(e);
+ }
+ }
+}
Propchange: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java
------------------------------------------------------------------------------
svn:eol-style = native