You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pa...@apache.org on 2011/07/11 23:43:29 UTC
svn commit: r1145368 - in /hive/trunk:
metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
Author: pauly
Date: Mon Jul 11 21:43:29 2011
New Revision: 1145368
URL: http://svn.apache.org/viewvc?rev=1145368&view=rev
Log:
HIVE-2275. Revert HIVE-2219 and apply correct patch to improve the efficiency of dropping multiple partitions (Sohan Jain via pauly)
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1145368&r1=1145367&r2=1145368&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Mon Jul 11 21:43:29 2011
@@ -27,7 +27,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -277,20 +276,17 @@ public class Warehouse {
}
/**
- * A common function for constructing a partition name
- * @param spec the partition spec
- * @param addTrailingSeparator whether to add a trailing separator at the end
- * @param isSorted whether the partition name should be sorted by key
- * @return a partition name
+ * Makes a partition name from a specification
+ * @param spec
+ * @param addTrailingSeperator if true, adds a trailing separator e.g. 'ds=1/'
+ * @return
* @throws MetaException
*/
- private static String makePartNameCommon(Map<String, String> spec,
- boolean addTrailingSeparator, boolean isSorted) throws MetaException {
+ public static String makePartName(Map<String, String> spec,
+ boolean addTrailingSeperator)
+ throws MetaException {
StringBuilder suffixBuf = new StringBuilder();
int i = 0;
- if (isSorted) {
- spec = new TreeMap<String, String>(spec);
- }
for (Entry<String, String> e : spec.entrySet()) {
if (e.getValue() == null || e.getValue().length() == 0) {
throw new MetaException("Partition spec is incorrect. " + spec);
@@ -303,39 +299,11 @@ public class Warehouse {
suffixBuf.append(escapePathName(e.getValue()));
i++;
}
- if (addTrailingSeparator) {
+ if (addTrailingSeperator) {
suffixBuf.append(Path.SEPARATOR);
}
return suffixBuf.toString();
}
-
- /**
- * Makes a partition name from a specification
- * @param spec
- * @param addTrailingSeperator if true, adds a trailing separator e.g. 'ds=1/'
- * @return
- * @throws MetaException
- */
- public static String makePartName(Map<String, String> spec,
- boolean addTrailingSeperator)
- throws MetaException {
- return makePartNameCommon(spec, addTrailingSeperator, false);
- }
-
- /**
- * Makes a partition name from a specification. The keys/value pairs in the partition
- * name are sorted by key name. E.g., created=4/ds=1/hr=3. Sorting the keys is useful
- * for comparison of partition names.
- * @param spec the partition spec
- * @param addTrailingSeparator whether to add a trailing separator at the end of the name
- * @return a partition name in which the keys are sorted
- * @throws MetaException
- */
- public static String makeSortedPartName(Map<String, String> spec,
- boolean addTrailingSeparator) throws MetaException {
- return makePartNameCommon(spec, addTrailingSeparator, true);
- }
-
/**
* Given a dynamic partition specification, return the path corresponding to the
* static part of partition specification. This is basically a copy of makePartName
@@ -452,4 +420,5 @@ public class Warehouse {
values.addAll(partSpec.values());
return values;
}
+
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1145368&r1=1145367&r2=1145368&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Mon Jul 11 21:43:29 2011
@@ -34,7 +34,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
-import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -2928,43 +2927,16 @@ public class DDLTask extends Task<DDLWor
dropTbl.getExpectView());
}
- // get all partition names of the table
- List<String> partitionNames =
- db.getPartitionNames(dropTbl.getTableName(), (short) -1);
- //Build a hashtable of a "sorted partition name" to the original partition spec
- //We need a sorted partition name so we can compare partitions by their names
- Hashtable<String,Map<String,String>> partitions = new Hashtable<String,Map<String,String>>();
- for (String partitionName : partitionNames) {
- try {
- Map<String, String> spec = Warehouse.makeSpecFromName(partitionName);
- partitions.put(Warehouse.makeSortedPartName(spec, false), spec);
- } catch (MetaException e) {
- LOG.warn("Unrecognized partition name from metastore: " + partitionName);
- }
- }
- // drop partitions in the list
List<Partition> partsToDelete = new ArrayList<Partition>();
for (Map<String, String> partSpec : dropTbl.getPartSpecs()) {
- String partSpecStr;
- try {
- partSpecStr = Warehouse.makeSortedPartName(partSpec, false);
- } catch (MetaException e) {
- //If the partition spec to remove is empty or null, just move on gracefully
- LOG.warn(e.getMessage());
- continue;
- }
- Map<String, String> origPart = null;
- //If the partition we wish to delete exists in our hashtable of all partition names,
- //get the partition and add it to our list of partitions to delete
- if ((origPart = partitions.get(partSpecStr)) != null) {
- Partition p = db.getPartition(tbl, origPart, false);
+ List<Partition> partitions = db.getPartitions(tbl, partSpec);
+ for (Partition p : partitions) {
if (!p.canDrop()) {
throw new HiveException("Table " + tbl.getTableName() +
" Partition " + p.getName() +
" is protected from being dropped");
}
partsToDelete.add(p);
- partitions.remove(partSpecStr);
}
}