You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2012/09/17 21:27:35 UTC
svn commit: r1386782 - in /hive/trunk/metastore/src:
java/org/apache/hadoop/hive/metastore/ObjectStore.java
java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java
Author: cws
Date: Mon Sep 17 19:27:35 2012
New Revision: 1386782
URL: http://svn.apache.org/viewvc?rev=1386782&view=rev
Log:
HIVE-3443. Hive Metatool should take serde_param_key from the user to allow for changes to avro serde's schema url key (Shreepadma Venugopalan via cws)
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1386782&r1=1386781&r2=1386782&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Mon Sep 17 19:27:35 2012
@@ -4009,24 +4009,23 @@ public class ObjectStore implements RawS
* is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
*
*/
- @SuppressWarnings("finally")
public Collection<?> executeJDOQLSelect(String query) {
boolean committed = false;
Collection<?> result = null;
- LOG.info("Executing query: " + query);
-
try {
openTransaction();
Query q = pm.newQuery(query);
result = (Collection<?>) q.execute();
committed = commitTransaction();
+ if (committed) {
+ return result;
+ } else {
+ return null;
+ }
} finally {
if (!committed) {
rollbackTransaction();
- return null;
- } else {
- return result;
}
}
}
@@ -4038,24 +4037,23 @@ public class ObjectStore implements RawS
* is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
*
*/
- @SuppressWarnings("finally")
public long executeJDOQLUpdate(String query) {
boolean committed = false;
long numUpdated = 0;
- LOG.info("Executing query: " + query);
-
try {
openTransaction();
Query q = pm.newQuery(query);
numUpdated = (Long) q.execute();
committed = commitTransaction();
+ if (committed) {
+ return numUpdated;
+ } else {
+ return -1;
+ }
} finally {
if (!committed) {
rollbackTransaction();
- return -1;
- } else {
- return numUpdated;
}
}
}
@@ -4067,7 +4065,6 @@ public class ObjectStore implements RawS
* is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
*
*/
- @SuppressWarnings("finally")
public Set<String> listFSRoots() {
boolean committed = false;
Set<String> fsRoots = new HashSet<String>();
@@ -4082,12 +4079,14 @@ public class ObjectStore implements RawS
fsRoots.add(mDB.getLocationUri());
}
committed = commitTransaction();
+ if (committed) {
+ return fsRoots;
+ } else {
+ return null;
+ }
} finally {
if (!committed) {
rollbackTransaction();
- return null;
- } else {
- return fsRoots;
}
}
}
@@ -4128,155 +4127,322 @@ public class ObjectStore implements RawS
return true;
}
- private int updateMDatabaseURI(URI oldLoc, URI newLoc,
- HashMap<String, String> updateLocations, boolean dryRun) {
- int count = 0;
- Query query = pm.newQuery(MDatabase.class);
- List<MDatabase> mDBs = (List<MDatabase>) query.execute();
- pm.retrieveAll(mDBs);
-
- LOG.info("Looking for location in DB_LOCATION_URI field in DBS table...");
-
- for(MDatabase mDB:mDBs) {
- URI locationURI = null;
- try {
- locationURI = new URI(mDB.getLocationUri());
- } catch(URISyntaxException e) {
- LOG.error("Encountered error while validating location URI"
- + e.getLocalizedMessage());
- }
- // locationURI is a valid URI
- if (locationURI != null) {
- if (shouldUpdateURI(locationURI, oldLoc)) {
- String dbLoc = mDB.getLocationUri().replaceAll(oldLoc.toString(), newLoc.toString());
- if (dryRun) {
+ public class UpdateMDatabaseURIRetVal {
+ private List<String> badRecords;
+ private Map<String, String> updateLocations;
+
+ UpdateMDatabaseURIRetVal(List<String> badRecords, Map<String, String> updateLocations) {
+ this.badRecords = badRecords;
+ this.updateLocations = updateLocations;
+ }
+
+ public List<String> getBadRecords() {
+ return badRecords;
+ }
+
+ public void setBadRecords(List<String> badRecords) {
+ this.badRecords = badRecords;
+ }
+
+ public Map<String, String> getUpdateLocations() {
+ return updateLocations;
+ }
+
+ public void setUpdateLocations(Map<String, String> updateLocations) {
+ this.updateLocations = updateLocations;
+ }
+ }
+
+ /** The following APIs
+ *
+ * - updateMDatabaseURI
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ public UpdateMDatabaseURIRetVal updateMDatabaseURI(URI oldLoc, URI newLoc, boolean dryRun) {
+ boolean committed = false;
+ Map<String, String> updateLocations = new HashMap<String, String>();
+ List<String> badRecords = new ArrayList<String>();
+ UpdateMDatabaseURIRetVal retVal = null;
+
+ try {
+ openTransaction();
+ Query query = pm.newQuery(MDatabase.class);
+ List<MDatabase> mDBs = (List<MDatabase>) query.execute();
+ pm.retrieveAll(mDBs);
+
+ for(MDatabase mDB:mDBs) {
+ URI locationURI = null;
+ String location = mDB.getLocationUri();
+ try {
+ locationURI = new URI(location);
+ } catch(URISyntaxException e) {
+ badRecords.add(location);
+ }
+ if (locationURI == null) {
+ badRecords.add(location);
+ } else {
+ if (shouldUpdateURI(locationURI, oldLoc)) {
+ String dbLoc = mDB.getLocationUri().replaceAll(oldLoc.toString(), newLoc.toString());
updateLocations.put(locationURI.toString(), dbLoc);
- } else {
- mDB.setLocationUri(dbLoc);
+ if (!dryRun) {
+ mDB.setLocationUri(dbLoc);
+ }
}
- count++;
}
}
+ committed = commitTransaction();
+ if (committed) {
+ retVal = new UpdateMDatabaseURIRetVal(badRecords, updateLocations);
+ }
+ return retVal;
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ }
+ }
+ }
+
+ public class UpdateMStorageDescriptorTblPropURIRetVal {
+ private List<String> badRecords;
+ private Map<String, String> updateLocations;
+
+ UpdateMStorageDescriptorTblPropURIRetVal(List<String> badRecords,
+ Map<String, String> updateLocations) {
+ this.badRecords = badRecords;
+ this.updateLocations = updateLocations;
+ }
+
+ public List<String> getBadRecords() {
+ return badRecords;
+ }
+
+ public void setBadRecords(List<String> badRecords) {
+ this.badRecords = badRecords;
}
- LOG.info("Found " + count + " records to update");
- return count;
+ public Map<String, String> getUpdateLocations() {
+ return updateLocations;
+ }
+
+ public void setUpdateLocations(Map<String, String> updateLocations) {
+ this.updateLocations = updateLocations;
+ }
}
- private int updateMStorageDescriptorURI(URI oldLoc, URI newLoc,
- HashMap<String, String> updateLocations, boolean dryRun) {
- int count = 0;
- Query query = pm.newQuery(MStorageDescriptor.class);
- List<MStorageDescriptor> mSDSs = (List<MStorageDescriptor>) query.execute();
- pm.retrieveAll(mSDSs);
-
- LOG.info("Looking for location in LOCATION field in SDS table...");
-
- for(MStorageDescriptor mSDS:mSDSs) {
- URI locationURI = null;
- try {
- locationURI = new URI(mSDS.getLocation());
- } catch (URISyntaxException e) {
- LOG.error("Encountered error while validating location URI"
- + e.getLocalizedMessage());
- }
- // locationURI is a valid URI
- if (locationURI != null) {
- if (shouldUpdateURI(locationURI, oldLoc)) {
- String tblLoc = mSDS.getLocation().replaceAll(oldLoc.toString(), newLoc.toString());
- if (dryRun) {
- updateLocations.put(locationURI.toString(), tblLoc);
- } else {
- mSDS.setLocation(tblLoc);
+ /** The following APIs
+ *
+ * - updateMStorageDescriptorTblPropURI
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ public UpdateMStorageDescriptorTblPropURIRetVal updateMStorageDescriptorTblPropURI(URI oldLoc,
+ URI newLoc, String tblPropKey, boolean isDryRun) {
+ boolean committed = false;
+ Map<String, String> updateLocations = new HashMap<String, String>();
+ List<String> badRecords = new ArrayList<String>();
+ UpdateMStorageDescriptorTblPropURIRetVal retVal = null;
+
+ try {
+ openTransaction();
+ Query query = pm.newQuery(MStorageDescriptor.class);
+ List<MStorageDescriptor> mSDSs = (List<MStorageDescriptor>) query.execute();
+ pm.retrieveAll(mSDSs);
+
+ for(MStorageDescriptor mSDS:mSDSs) {
+ URI tablePropLocationURI = null;
+ String tablePropLocation = mSDS.getParameters().get(tblPropKey);
+ if (tablePropLocation != null) {
+ try {
+ tablePropLocationURI = new URI(tablePropLocation);
+ } catch (URISyntaxException e) {
+ badRecords.add(tablePropLocation);
+ }
}
- count++;
+ // if tablePropKey that was passed in lead to a valid URI resolution, update it if
+ //parts of it match the old-NN-loc, else add to badRecords
+ if (tablePropLocationURI == null) {
+ badRecords.add(tablePropLocation);
+ } else {
+ if (shouldUpdateURI(tablePropLocationURI, oldLoc)) {
+ String tblPropLoc = mSDS.getParameters().get(tblPropKey).replaceAll(oldLoc.toString(),
+ newLoc.toString());
+ updateLocations.put(tablePropLocationURI.toString(), tblPropLoc);
+ if (!isDryRun) {
+ mSDS.getParameters().put(tblPropKey, tblPropLoc);
+ }
+ }
+ }
+ }
+ committed = commitTransaction();
+ if (committed) {
+ retVal = new UpdateMStorageDescriptorTblPropURIRetVal(badRecords, updateLocations);
+ }
+ return retVal;
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
}
- }
+ }
+ }
+
+ public class UpdateMStorageDescriptorTblURIRetVal {
+ private List<String> badRecords;
+ private Map<String, String> updateLocations;
+
+ UpdateMStorageDescriptorTblURIRetVal(List<String> badRecords,
+ Map<String, String> updateLocations) {
+ this.badRecords = badRecords;
+ this.updateLocations = updateLocations;
+ }
+
+ public List<String> getBadRecords() {
+ return badRecords;
+ }
+
+ public void setBadRecords(List<String> badRecords) {
+ this.badRecords = badRecords;
+ }
+
+ public Map<String, String> getUpdateLocations() {
+ return updateLocations;
}
- LOG.info("Found " + count + " records to update");
- return count;
+ public void setUpdateLocations(Map<String, String> updateLocations) {
+ this.updateLocations = updateLocations;
+ }
}
- private int updateAvroSerdeURI(URI oldLoc, URI newLoc,
- HashMap<String, String> updateLocations, boolean dryRun) {
- int count = 0;
- Query query = pm.newQuery(MSerDeInfo.class);
- List<MSerDeInfo> mSerdes = (List<MSerDeInfo>) query.execute();
- pm.retrieveAll(mSerdes);
-
- LOG.info("Looking for location in the value field of schema.url key in SERDES table...");
-
- for(MSerDeInfo mSerde:mSerdes) {
- String key = new String("schema.url");
- String schemaLoc = mSerde.getParameters().get(key);
- if (schemaLoc != null) {
- URI schemaLocURI = null;
+ /** The following APIs
+ *
+ * - updateMStorageDescriptorTblURI
+ *
+ * is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
+ *
+ */
+ public UpdateMStorageDescriptorTblURIRetVal updateMStorageDescriptorTblURI(URI oldLoc, URI newLoc,
+ boolean isDryRun) {
+ boolean committed = false;
+ Map<String, String> updateLocations = new HashMap<String, String>();
+ List<String> badRecords = new ArrayList<String>();
+ UpdateMStorageDescriptorTblURIRetVal retVal = null;
+
+ try {
+ openTransaction();
+ Query query = pm.newQuery(MStorageDescriptor.class);
+ List<MStorageDescriptor> mSDSs = (List<MStorageDescriptor>) query.execute();
+ pm.retrieveAll(mSDSs);
+
+ for(MStorageDescriptor mSDS:mSDSs) {
+ URI locationURI = null;
+ String location = mSDS.getLocation();
try {
- schemaLocURI = new URI(schemaLoc);
+ locationURI = new URI(location);
} catch (URISyntaxException e) {
- LOG.error("Encountered error while validating location URI"
- + e.getLocalizedMessage());
+ badRecords.add(location);
}
- // schemaLocURI is a valid URI
- if (schemaLocURI != null) {
- if (shouldUpdateURI(schemaLocURI, oldLoc)) {
- String newSchemaLoc = schemaLoc.replaceAll(oldLoc.toString(), newLoc.toString());
- if (dryRun) {
- updateLocations.put(schemaLocURI.toString(), newSchemaLoc);
- } else {
- mSerde.getParameters().put(key, newSchemaLoc);
+ if (locationURI == null) {
+ badRecords.add(location);
+ } else {
+ if (shouldUpdateURI(locationURI, oldLoc)) {
+ String tblLoc = mSDS.getLocation().replaceAll(oldLoc.toString(), newLoc.toString());
+ updateLocations.put(locationURI.toString(), tblLoc);
+ if (!isDryRun) {
+ mSDS.setLocation(tblLoc);
}
- count++;
}
}
}
+ committed = commitTransaction();
+ if (committed) {
+ retVal = new UpdateMStorageDescriptorTblURIRetVal(badRecords, updateLocations);
+ }
+ return retVal;
+ } finally {
+ if (!committed) {
+ rollbackTransaction();
+ }
+ }
+ }
+
+ public class UpdateSerdeURIRetVal {
+ private List<String> badRecords;
+ private Map<String, String> updateLocations;
+
+ UpdateSerdeURIRetVal(List<String> badRecords, Map<String, String> updateLocations) {
+ this.badRecords = badRecords;
+ this.updateLocations = updateLocations;
+ }
+
+ public List<String> getBadRecords() {
+ return badRecords;
+ }
+
+ public void setBadRecords(List<String> badRecords) {
+ this.badRecords = badRecords;
}
- LOG.info("Found " + count + " records to update");
- return count;
+ public Map<String, String> getUpdateLocations() {
+ return updateLocations;
+ }
+
+ public void setUpdateLocations(Map<String, String> updateLocations) {
+ this.updateLocations = updateLocations;
+ }
}
/** The following APIs
*
- * - updateFSRootLocation
+ * - updateSerdeURI
*
* is used by HiveMetaTool. This API **shouldn't** be exposed via Thrift.
*
*/
- @SuppressWarnings("finally")
- public int updateFSRootLocation(URI oldLoc, URI newLoc,
- HashMap<String, String> updateLocations, boolean dryRun) {
+ public UpdateSerdeURIRetVal updateSerdeURI(URI oldLoc, URI newLoc, String serdeProp,
+ boolean isDryRun) {
boolean committed = false;
- int count = 0;
- int totalCount = 0;
-
- LOG.info("Old FS root location: " + oldLoc.toString() +
- " New FS root location: " + newLoc.toString());
- LOG.info("Updating FS root location...");
+ Map<String, String> updateLocations = new HashMap<String, String>();
+ List<String> badRecords = new ArrayList<String>();
+ UpdateSerdeURIRetVal retVal = null;
try {
openTransaction();
-
- // update locationURI in mDatabase
- count = updateMDatabaseURI(oldLoc, newLoc, updateLocations, dryRun);
- totalCount += count;
-
- // update location in mStorageDescriptor
- count = updateMStorageDescriptorURI(oldLoc, newLoc, updateLocations, dryRun);
- totalCount += count;
-
- // upgrade schema.url for avro serde
- count = updateAvroSerdeURI(oldLoc, newLoc, updateLocations, dryRun);
- totalCount += count;
-
+ Query query = pm.newQuery(MSerDeInfo.class);
+ List<MSerDeInfo> mSerdes = (List<MSerDeInfo>) query.execute();
+ pm.retrieveAll(mSerdes);
+
+ for(MSerDeInfo mSerde:mSerdes) {
+ String schemaLoc = mSerde.getParameters().get(serdeProp);
+ if (schemaLoc != null) {
+ URI schemaLocURI = null;
+ try {
+ schemaLocURI = new URI(schemaLoc);
+ } catch (URISyntaxException e) {
+ badRecords.add(schemaLoc);
+ }
+ if (schemaLocURI == null) {
+ badRecords.add(schemaLoc);
+ } else {
+ if (shouldUpdateURI(schemaLocURI, oldLoc)) {
+ String newSchemaLoc = schemaLoc.replaceAll(oldLoc.toString(), newLoc.toString());
+ updateLocations.put(schemaLocURI.toString(), newSchemaLoc);
+ if (!isDryRun) {
+ mSerde.getParameters().put(serdeProp, newSchemaLoc);
+ }
+ }
+ }
+ }
+ }
committed = commitTransaction();
+ if (committed) {
+ retVal = new UpdateSerdeURIRetVal(badRecords, updateLocations);
+ }
+ return retVal;
} finally {
if (!committed) {
rollbackTransaction();
- return -1;
- } else {
- return totalCount;
}
}
}
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java?rev=1386782&r1=1386781&r2=1386782&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/tools/HiveMetaTool.java Mon Sep 17 19:27:35 2012
@@ -20,8 +20,9 @@ package org.apache.hadoop.hive.metastore
import java.net.URI;
import java.util.Collection;
-import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.commons.cli.CommandLine;
@@ -49,9 +50,10 @@ public class HiveMetaTool {
private static final Log LOG = LogFactory.getLog(HiveMetaTool.class.getName());
private final Options cmdLineOptions = new Options();
private ObjectStore objStore;
-
+ private boolean isObjStoreInitialized;
public HiveMetaTool() {
+ this.isObjStoreInitialized = false;
}
@SuppressWarnings("static-access")
@@ -75,48 +77,78 @@ public class HiveMetaTool {
Option updateFSRootLoc =
OptionBuilder
.withArgName("new-loc> " + "<old-loc")
- .hasArgs(3)
+ .hasArgs(2)
.withDescription(
- "update FS root location in the metastore to new location. Both new-loc and" +
- " old-loc should be valid URIs with valid host names and schemes. " +
- "when run with the dryRun option changes are displayed but are not persisted.")
- .create("updateLocation");
- Option dryRun = new Option("dryRun" , "dryRun is valid only with updateLocation option. when " +
- "run with the dryRun option updateLocation changes are displayed but are not persisted.");
+ "Update FS root location in the metastore to new location.Both new-loc and " +
+ "old-loc should be valid URIs with valid host names and schemes." +
+ "When run with the dryRun option changes are displayed but are not " +
+ "persisted. When run with the serdepropKey/tablePropKey option " +
+ "updateLocation looks for the serde-prop-key/table-prop-key that is " +
+ "specified and updates its value if found.")
+ .create("updateLocation");
+ Option dryRun = new Option("dryRun" , "Perform a dry run of updateLocation changes.When " +
+ "run with the dryRun option updateLocation changes are displayed but not persisted. " +
+ "dryRun is valid only with the updateLocation option.");
+ Option serdePropKey =
+ OptionBuilder.withArgName("serde-prop-key")
+ .hasArgs()
+ .withValueSeparator()
+ .withDescription("Specify the key for serde property to be updated. serdePropKey option " +
+ "is valid only with updateLocation option.")
+ .create("serdePropKey");
+ Option tablePropKey =
+ OptionBuilder.withArgName("table-prop-key")
+ .hasArg()
+ .withValueSeparator()
+ .withDescription("Specify the key for table property to be updated. tablePropKey option " +
+ "is valid only with updateLocation option.")
+ .create("tablePropKey");
cmdLineOptions.addOption(help);
cmdLineOptions.addOption(listFSRoot);
cmdLineOptions.addOption(executeJDOQL);
cmdLineOptions.addOption(updateFSRootLoc);
cmdLineOptions.addOption(dryRun);
-
+ cmdLineOptions.addOption(serdePropKey);
+ cmdLineOptions.addOption(tablePropKey);
}
private void initObjectStore(HiveConf hiveConf) {
- objStore = new ObjectStore();
- objStore.setConf(hiveConf);
+ if (!isObjStoreInitialized) {
+ objStore = new ObjectStore();
+ objStore.setConf(hiveConf);
+ isObjStoreInitialized = true;
+ }
}
private void shutdownObjectStore() {
- if (objStore != null) {
+ if (isObjStoreInitialized) {
objStore.shutdown();
+ isObjStoreInitialized = false;
}
}
private void listFSRoot() {
+ HiveConf hiveConf = new HiveConf(HiveMetaTool.class);
+ initObjectStore(hiveConf);
+
Set<String> hdfsRoots = objStore.listFSRoots();
if (hdfsRoots != null) {
- System.out.println("HiveMetaTool:Listing FS Roots..");
+ System.out.println("Listing FS Roots..");
for (String s : hdfsRoots) {
System.out.println(s);
}
} else {
- System.err.println("HiveMetaTool:Encountered error during listFSRoot - " +
+ System.err.println("Encountered error during listFSRoot - " +
"commit of JDO transaction failed");
}
}
private void executeJDOQLSelect(String query) {
+ HiveConf hiveConf = new HiveConf(HiveMetaTool.class);
+ initObjectStore(hiveConf);
+
+ System.out.println("Executing query: " + query);
Collection<?> result = objStore.executeJDOQLSelect(query);
if (result != null) {
Iterator<?> iter = result.iterator();
@@ -125,46 +157,205 @@ public class HiveMetaTool {
System.out.println(o.toString());
}
} else {
- System.err.println("HiveMetaTool:Encountered error during executeJDOQLSelect -" +
+ System.err.println("Encountered error during executeJDOQLSelect -" +
"commit of JDO transaction failed.");
}
}
- private void executeJDOQLUpdate(String query) {
+ private long executeJDOQLUpdate(String query) {
+ HiveConf hiveConf = new HiveConf(HiveMetaTool.class);
+ initObjectStore(hiveConf);
+
+ System.out.println("Executing query: " + query);
long numUpdated = objStore.executeJDOQLUpdate(query);
if (numUpdated >= 0) {
- System.out.println("HiveMetaTool:Number of records updated: " + numUpdated);
+ System.out.println("Number of records updated: " + numUpdated);
} else {
- System.err.println("HiveMetaTool:Encountered error during executeJDOQL -" +
+ System.err.println("Encountered error during executeJDOQL -" +
"commit of JDO transaction failed.");
}
+ return numUpdated;
}
- private void printUpdateLocations(HashMap<String, String> updateLocations) {
+ private int printUpdateLocations(Map<String, String> updateLocations) {
+ int count = 0;
for (String key: updateLocations.keySet()) {
String value = updateLocations.get(key);
- System.out.println("current location: " + key + " new location: " + value);
+ System.out.println("old location: " + key + " new location: " + value);
+ count++;
}
+ return count;
}
- private void updateFSRootLocation(URI oldURI, URI newURI, boolean dryRun) {
- HashMap<String, String> updateLocations = new HashMap<String, String>();
- int count = objStore.updateFSRootLocation(oldURI, newURI, updateLocations, dryRun);
- if (count == -1) {
- System.err.println("HiveMetaTool:Encountered error while executing updateFSRootLocation - " +
- "commit of JDO transaction failed, failed to update FS Root locations.");
+ private void printTblURIUpdateSummary(ObjectStore.UpdateMStorageDescriptorTblURIRetVal retVal,
+ boolean isDryRun) {
+ String tblName = new String("SDS");
+ String fieldName = new String("LOCATION");
+
+ if (retVal == null) {
+ System.err.println("Encountered error while executing updateMStorageDescriptorTblURI - " +
+ "commit of JDO transaction failed. Failed to update FSRoot locations in " +
+ fieldName + "field in " + tblName + " table.");
} else {
- if (!dryRun) {
- System.out.println("HiveMetaTool: Successfully updated " + count + "FS Root locations");
+ Map<String, String> updateLocations = retVal.getUpdateLocations();
+ if (isDryRun) {
+ System.out.println("Dry Run of updateLocation on table " + tblName + "..");
+ } else {
+ System.out.println("Successfully updated the following locations..");
+ }
+ int count = printUpdateLocations(updateLocations);
+ if (isDryRun) {
+ System.out.println("Found " + count + " records in " + tblName + " table to update");
} else {
- printUpdateLocations(updateLocations);
+ System.out.println("Updated " + count + " records in " + tblName + " table");
+ }
+ List<String> badRecords = retVal.getBadRecords();
+ if (badRecords.size() > 0) {
+ System.err.println("Found records with bad " + fieldName + " in " + tblName + " table.. ");
+ for (String badRecord:badRecords) {
+ System.err.println("bad location URI: " + badRecord);
+ }
}
}
}
- public static void main(String[] args) {
+ private void printDatabaseURIUpdateSummary(ObjectStore.UpdateMDatabaseURIRetVal retVal,
+ boolean isDryRun) {
+ String tblName = new String("DBS");
+ String fieldName = new String("LOCATION_URI");
+
+ if (retVal == null) {
+ System.err.println("Encountered error while executing updateMDatabaseURI - " +
+ "commit of JDO transaction failed. Failed to update FSRoot locations in " +
+ fieldName + "field in " + tblName + " table.");
+ } else {
+ Map<String, String> updateLocations = retVal.getUpdateLocations();
+ if (isDryRun) {
+ System.out.println("Dry Run of updateLocation on table " + tblName + "..");
+ } else {
+ System.out.println("Successfully updated the following locations..");
+ }
+ int count = printUpdateLocations(updateLocations);
+ if (isDryRun) {
+ System.out.println("Found " + count + " records in " + tblName + " table to update");
+ } else {
+ System.out.println("Updated " + count + " records in " + tblName + " table");
+ }
+ List<String> badRecords = retVal.getBadRecords();
+ if (badRecords.size() > 0) {
+ System.err.println("Found records with bad " + fieldName + " in " + tblName + " table.. ");
+ for (String badRecord:badRecords) {
+ System.err.println("bad location URI: " + badRecord);
+ }
+ }
+ }
+ }
+ private void printTblPropURIUpdateSummary(
+ ObjectStore.UpdateMStorageDescriptorTblPropURIRetVal retVal, String tablePropKey,
+ boolean isDryRun) {
+ String tblName = new String("SD_PARAMS");
+
+ if (retVal == null) {
+ System.err.println("Encountered error while executing updateMStorageDescriptorTblPropURI - " +
+ "commit of JDO transaction failed. Failed to update FSRoot locations in " +
+ "value field corresponding to" + tablePropKey + " in " + tblName + " table.");
+ } else {
+ Map<String, String> updateLocations = retVal.getUpdateLocations();
+ if (isDryRun) {
+ System.out.println("Dry Run of updateLocation on table " + tblName + "..");
+ } else {
+ System.out.println("Successfully updated the following locations..");
+ }
+ int count = printUpdateLocations(updateLocations);
+ if (isDryRun) {
+ System.out.println("Found " + count + " records in " + tblName + " table to update");
+ } else {
+ System.out.println("Updated " + count + " records in " + tblName + " table");
+ }
+ List<String> badRecords = retVal.getBadRecords();
+ if (badRecords.size() > 0) {
+ System.err.println("Found records with bad " + tablePropKey + " key in " +
+ tblName + " table.. ");
+ for (String badRecord:badRecords) {
+ System.err.println("bad location URI: " + badRecord);
+ }
+ }
+ }
+ }
+
+ private void printSerdePropURIUpdateSummary(ObjectStore.UpdateSerdeURIRetVal retVal,
+ String serdePropKey, boolean isDryRun) {
+ String tblName = new String("SERDE_PARAMS");
+
+ if (retVal == null) {
+ System.err.println("Encountered error while executing updateSerdeURI - " +
+ "commit of JDO transaction failed. Failed to update FSRoot locations in " +
+ "value field corresponding to " + serdePropKey + " in " + tblName + " table.");
+ } else {
+ Map<String, String> updateLocations = retVal.getUpdateLocations();
+ if (isDryRun) {
+ System.out.println("Dry Run of updateLocation on table " + tblName + "..");
+ } else {
+ System.out.println("Successfully updated the following locations..");
+ }
+ int count = printUpdateLocations(updateLocations);
+ if (isDryRun) {
+ System.out.println("Found " + count + " records in " + tblName + " table to update");
+ } else {
+ System.out.println("Updated " + count + " records in " + tblName + " table");
+ }
+ List<String> badRecords = retVal.getBadRecords();
+ if (badRecords.size() > 0) {
+ System.err.println("Found records with bad " + serdePropKey + " key in " +
+ tblName + " table.. ");
+ for (String badRecord:badRecords) {
+ System.err.println("bad location URI: " + badRecord);
+ }
+ }
+ }
+ }
+
+ public void updateFSRootLocation(URI oldURI, URI newURI, String serdePropKey,
+ String tablePropKey, boolean isDryRun) {
HiveConf hiveConf = new HiveConf(HiveMetaTool.class);
+ initObjectStore(hiveConf);
+
+ System.out.println("Looking for LOCATION_URI field in DBS table to update..");
+ ObjectStore.UpdateMDatabaseURIRetVal updateMDBURIRetVal = objStore.updateMDatabaseURI(oldURI,
+ newURI, isDryRun);
+ printDatabaseURIUpdateSummary(updateMDBURIRetVal, isDryRun);
+
+ System.out.println("Looking for LOCATION field in SDS table to update..");
+ ObjectStore.UpdateMStorageDescriptorTblURIRetVal updateTblURIRetVal =
+ objStore.updateMStorageDescriptorTblURI(oldURI, newURI, isDryRun);
+ printTblURIUpdateSummary(updateTblURIRetVal, isDryRun);
+
+ if (tablePropKey != null) {
+ System.out.println("Looking for value of " + tablePropKey + " key in SD_PARAMS table " +
+ "to update..");
+ ObjectStore.UpdateMStorageDescriptorTblPropURIRetVal updateTblPropURIRetVal =
+ objStore.updateMStorageDescriptorTblPropURI(oldURI, newURI,
+ tablePropKey, isDryRun);
+ printTblPropURIUpdateSummary(updateTblPropURIRetVal, tablePropKey, isDryRun);
+ }
+
+ if (serdePropKey != null) {
+ System.out.println("Looking for value of " + serdePropKey + " key in SERDE_PARAMS table " +
+ "to update..");
+ ObjectStore.UpdateSerdeURIRetVal updateSerdeURIretVal = objStore.updateSerdeURI(oldURI,
+ newURI, serdePropKey, isDryRun);
+ printSerdePropURIUpdateSummary(updateSerdeURIretVal, serdePropKey, isDryRun);
+ }
+ }
+
+ private static void printAndExit(HiveMetaTool metaTool) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("metatool", metaTool.cmdLineOptions);
+ System.exit(1);
+ }
+
+ public static void main(String[] args) {
HiveMetaTool metaTool = new HiveMetaTool();
metaTool.init();
CommandLineParser parser = new GnuParser();
@@ -175,35 +366,55 @@ public class HiveMetaTool {
line = parser.parse(metaTool.cmdLineOptions, args);
} catch (ParseException e) {
System.err.println("HiveMetaTool:Parsing failed. Reason: " + e.getLocalizedMessage());
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("metatool", metaTool.cmdLineOptions);
- System.exit(1);
+ printAndExit(metaTool);
}
if (line.hasOption("help")) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("metatool", metaTool.cmdLineOptions);
} else if (line.hasOption("listFSRoot")) {
- metaTool.initObjectStore(hiveConf);
+ if (line.hasOption("dryRun")) {
+ System.err.println("HiveMetaTool: dryRun is not valid with listFSRoot");
+ printAndExit(metaTool);
+ } else if (line.hasOption("serdePropKey")) {
+ System.err.println("HiveMetaTool: serdePropKey is not valid with listFSRoot");
+ printAndExit(metaTool);
+ } else if (line.hasOption("tablePropKey")) {
+ System.err.println("HiveMetaTool: tablePropKey is not valid with listFSRoot");
+ printAndExit(metaTool);
+ }
metaTool.listFSRoot();
} else if (line.hasOption("executeJDOQL")) {
String query = line.getOptionValue("executeJDOQL");
- metaTool.initObjectStore(hiveConf);
+ if (line.hasOption("dryRun")) {
+ System.err.println("HiveMetaTool: dryRun is not valid with executeJDOQL");
+ printAndExit(metaTool);
+ } else if (line.hasOption("serdePropKey")) {
+ System.err.println("HiveMetaTool: serdePropKey is not valid with executeJDOQL");
+ printAndExit(metaTool);
+ } else if (line.hasOption("tablePropKey")) {
+ System.err.println("HiveMetaTool: tablePropKey is not valid with executeJDOQL");
+ printAndExit(metaTool);
+ }
if (query.toLowerCase().trim().startsWith("select")) {
metaTool.executeJDOQLSelect(query);
} else if (query.toLowerCase().trim().startsWith("update")) {
metaTool.executeJDOQLUpdate(query);
} else {
System.err.println("HiveMetaTool:Unsupported statement type");
+ printAndExit(metaTool);
}
} else if (line.hasOption("updateLocation")) {
String[] loc = line.getOptionValues("updateLocation");
- boolean dryRun = false;
+ boolean isDryRun = false;
+ String serdepropKey = null;
+ String tablePropKey = null;
if (loc.length != 2 && loc.length != 3) {
System.err.println("HiveMetaTool:updateLocation takes in 2 required and 1 " +
- "optional arguements but " +
- "was passed " + loc.length + " arguements");
+ "optional arguments but " +
+ "was passed " + loc.length + " arguments");
+ printAndExit(metaTool);
}
Path newPath = new Path(loc[0]);
@@ -213,7 +424,15 @@ public class HiveMetaTool {
URI newURI = newPath.toUri();
if (line.hasOption("dryRun")) {
- dryRun = true;
+ isDryRun = true;
+ }
+
+ if (line.hasOption("serdePropKey")) {
+ serdepropKey = line.getOptionValue("serdePropKey");
+ }
+
+ if (line.hasOption("tablePropKey")) {
+ tablePropKey = line.getOptionValue("tablePropKey");
}
/*
@@ -229,20 +448,27 @@ public class HiveMetaTool {
} else if (oldURI.getScheme() == null || newURI.getScheme() == null) {
System.err.println("HiveMetaTool:A valid scheme is required in both old-loc and new-loc");
} else {
- metaTool.initObjectStore(hiveConf);
- metaTool.updateFSRootLocation(oldURI, newURI, dryRun);
+ metaTool.updateFSRootLocation(oldURI, newURI, serdepropKey, tablePropKey, isDryRun);
}
} else {
- System.err.print("HiveMetaTool:Invalid option:" + line.getOptions());
- for (String s : line.getArgs()) {
- System.err.print(s + " ");
+ if (line.hasOption("dryRun")) {
+ System.err.println("HiveMetaTool: dryRun is not a valid standalone option");
+ } else if (line.hasOption("serdePropKey")) {
+ System.err.println("HiveMetaTool: serdePropKey is not a valid standalone option");
+ } else if (line.hasOption("tablePropKey")) {
+ System.err.println("HiveMetaTool: tablePropKey is not a valid standalone option");
+ printAndExit(metaTool);
+ } else {
+ System.err.print("HiveMetaTool:Parsing failed. Reason: Invalid arguments: " );
+ for (String s : line.getArgs()) {
+ System.err.print(s + " ");
+ }
+ System.err.println();
}
- System.err.println();
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("metatool", metaTool.cmdLineOptions);
+ printAndExit(metaTool);
}
} finally {
metaTool.shutdownObjectStore();
}
}
-}
+}
\ No newline at end of file
Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java?rev=1386782&r1=1386781&r2=1386782&view=diff
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java (original)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaTool.java Mon Sep 17 19:27:35 2012
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Type;
import org.apache.hadoop.hive.metastore.tools.HiveMetaTool;
import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils;
import org.apache.hadoop.util.StringUtils;
public class TestHiveMetaTool extends TestCase {
@@ -72,16 +73,20 @@ public class TestHiveMetaTool extends Te
ps = new PrintStream(os);
// create a dummy database and a couple of dummy tables
- String dbName = "testDB";
+ String dbName = "TestHiveMetaToolDB";
String typeName = "Person";
String tblName = "simpleTbl";
+ String badTblName = "badSimpleTbl";
Database db = new Database();
db.setName(dbName);
client.dropTable(dbName, tblName);
+ client.dropTable(dbName, badTblName);
dropDatabase(dbName);
client.createDatabase(db);
locationUri = db.getLocationUri();
+ String avroUri = "hdfs://nn.example.com/warehouse/hive/ab.avsc";
+ String badAvroUri = new String("hdfs:/hive");
client.dropType(typeName);
Type typ1 = new Type();
@@ -110,10 +115,34 @@ public class TestHiveMetaTool extends Te
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.getSerdeInfo().getParameters().put(
org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
+ sd.getParameters().put(AvroSerdeUtils.SCHEMA_URL, avroUri);
sd.getSerdeInfo().setSerializationLib(
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
+ org.apache.hadoop.hive.serde2.avro.AvroSerDe.class.getName());
tbl.setPartitionKeys(new ArrayList<FieldSchema>());
+ client.createTable(tbl);
+ //create a table with bad avro uri
+ tbl = new Table();
+ tbl.setDbName(dbName);
+ tbl.setTableName(badTblName);
+ sd = new StorageDescriptor();
+ tbl.setSd(sd);
+ sd.setCols(typ1.getFields());
+ sd.setCompressed(false);
+ sd.setNumBuckets(1);
+ sd.setParameters(new HashMap<String, String>());
+ sd.getParameters().put("test_param_1", "Use this for comments etc");
+ sd.setBucketCols(new ArrayList<String>(2));
+ sd.getBucketCols().add("name");
+ sd.setSerdeInfo(new SerDeInfo());
+ sd.getSerdeInfo().setName(tbl.getTableName());
+ sd.getSerdeInfo().setParameters(new HashMap<String, String>());
+ sd.getSerdeInfo().getParameters().put(
+ org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
+ sd.getParameters().put(AvroSerdeUtils.SCHEMA_URL, badAvroUri);
+ sd.getSerdeInfo().setSerializationLib(
+ org.apache.hadoop.hive.serde2.avro.AvroSerDe.class.getName());
+ tbl.setPartitionKeys(new ArrayList<FieldSchema>());
client.createTable(tbl);
client.close();
} catch (Exception e) {
@@ -173,37 +202,29 @@ public class TestHiveMetaTool extends Te
public void testUpdateFSRootLocation() throws Exception {
redirectOutputStream();
- String newLocationUri = "hdfs://nn-ha-uri/user/hive/warehouse";
- String[] args = new String[3];
+ String oldLocationUri = "hdfs://nn.example.com/";
+ String newLocationUri = "hdfs://nn-ha-uri/";
+ String[] args = new String[5];
args[0] = new String("-updateLocation");
args[1] = new String(newLocationUri);
- args[2] = new String(locationUri);
-
- String[] args2 = new String[1];
- args2[0] = new String("-listFSRoot");
+ args[2] = new String(oldLocationUri);
+ args[3] = new String("-tablePropKey");
+ args[4] = new String("avro.schema.url");
try {
-
// perform HA upgrade
HiveMetaTool.main(args);
-
- // obtain new HDFS root
- HiveMetaTool.main(args2);
-
String out = os.toString();
boolean b = out.contains(newLocationUri);
+ restoreOutputStream();
+ assertTrue(b);
- if (b) {
- System.out.println("updateFSRootLocation successful");
- } else {
- System.out.println("updateFSRootLocation failed");
- }
- // restore the original HDFS root if needed
- if (b) {
- args[1] = new String(locationUri);
- args[2] = new String(newLocationUri);
- HiveMetaTool.main(args);
- }
+ //restore the original HDFS root
+ args[1] = new String(oldLocationUri);
+ args[2] = new String(newLocationUri);
+ redirectOutputStream();
+ HiveMetaTool.main(args);
+ restoreOutputStream();
} finally {
restoreOutputStream();
System.out.println("Completed testUpdateFSRootLocation..");
@@ -214,7 +235,6 @@ public class TestHiveMetaTool extends Te
protected void tearDown() throws Exception {
try {
super.tearDown();
-
} catch (Throwable e) {
System.err.println("Unable to close metastore");
System.err.println(StringUtils.stringifyException(e));