You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2014/04/25 23:00:01 UTC
svn commit: r1590142 - in
/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore:
MetaStoreDirectSql.java ObjectStore.java Warehouse.java
Author: sershe
Date: Fri Apr 25 21:00:01 2014
New Revision: 1590142
URL: http://svn.apache.org/r1590142
Log:
HIVE-6945 : issues with dropping partitions on Oracle (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1590142&r1=1590141&r2=1590142&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java Fri Apr 25 21:00:01 2014
@@ -309,7 +309,7 @@ class MetaStoreDirectSql {
StringBuilder partSb = new StringBuilder(sbCapacity);
// Assume db and table names are the same for all partition, that's what we're selecting for.
for (Object partitionId : sqlResult) {
- partSb.append((Long)partitionId).append(",");
+ partSb.append(extractSqlLong(partitionId)).append(",");
}
String partIds = trimCommaList(partSb);
timingTrace(doTrace, queryText, start, queryTime);
@@ -346,10 +346,10 @@ class MetaStoreDirectSql {
dbName = dbName.toLowerCase();
for (Object[] fields : sqlResult2) {
// Here comes the ugly part...
- long partitionId = (Long)fields[0];
- Long sdId = (Long)fields[1];
- Long colId = (Long)fields[2];
- Long serdeId = (Long)fields[3];
+ long partitionId = extractSqlLong(fields[0]);
+ Long sdId = extractSqlLong(fields[1]);
+ Long colId = extractSqlLong(fields[2]);
+ Long serdeId = extractSqlLong(fields[3]);
// A partition must have either everything set, or nothing set if it's a view.
if (sdId == null || colId == null || serdeId == null) {
if (isView == null) {
@@ -502,7 +502,7 @@ class MetaStoreDirectSql {
loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() {
private Long currentListId;
private List<String> currentList;
- public void apply(StorageDescriptor t, Object[] fields) {
+ public void apply(StorageDescriptor t, Object[] fields) throws MetaException {
if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo());
// Note that this is not a typical list accumulator - there's no call to finalize
// the last list. Instead we add list to SD first, as well as locally to add elements.
@@ -511,7 +511,7 @@ class MetaStoreDirectSql {
currentListId = null;
t.getSkewedInfo().addToSkewedColValues(new ArrayList<String>());
} else {
- long fieldsListId = (Long)fields[1];
+ long fieldsListId = extractSqlLong(fields[1]);
if (currentListId == null || fieldsListId != currentListId) {
currentList = new ArrayList<String>();
currentListId = fieldsListId;
@@ -539,7 +539,7 @@ class MetaStoreDirectSql {
loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() {
private Long currentListId;
private List<String> currentList;
- public void apply(StorageDescriptor t, Object[] fields) {
+ public void apply(StorageDescriptor t, Object[] fields) throws MetaException {
if (!t.isSetSkewedInfo()) {
SkewedInfo skewedInfo = new SkewedInfo();
skewedInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
@@ -552,7 +552,7 @@ class MetaStoreDirectSql {
currentList = new ArrayList<String>(); // left outer join produced a list with no values
currentListId = null;
} else {
- long fieldsListId = (Long)fields[1];
+ long fieldsListId = extractSqlLong(fields[1]);
if (currentListId == null || fieldsListId != currentListId) {
currentList = new ArrayList<String>();
currentListId = fieldsListId;
@@ -589,6 +589,14 @@ class MetaStoreDirectSql {
return orderedResult;
}
+ private Long extractSqlLong(Object obj) throws MetaException {
+ if (obj == null) return null;
+ if (!(obj instanceof Number)) {
+ throw new MetaException("Expected numeric type but got " + obj.getClass().getName());
+ }
+ return ((Number)obj).longValue();
+ }
+
private void timingTrace(boolean doTrace, String queryText, long start, long queryTime) {
if (!doTrace) return;
LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " +
@@ -604,9 +612,10 @@ class MetaStoreDirectSql {
if (value instanceof String && ((String)value).length() == 1) {
c = ((String)value).charAt(0);
}
+ if (c == null) return null;
if (c == 'Y') return true;
if (c == 'N') return false;
- throw new MetaException("Cannot extrace boolean from column value " + value);
+ throw new MetaException("Cannot extract boolean from column value " + value);
}
private int extractSqlInt(Object field) {
@@ -621,7 +630,7 @@ class MetaStoreDirectSql {
}
private abstract class ApplyFunc<Target> {
- public abstract void apply(Target t, Object[] fields);
+ public abstract void apply(Target t, Object[] fields) throws MetaException;
}
/**
@@ -655,7 +664,7 @@ class MetaStoreDirectSql {
if (fields == null) {
fields = iter.next();
}
- long nestedId = (Long)fields[keyIndex];
+ long nestedId = extractSqlLong(fields[keyIndex]);
if (nestedId < id) throw new MetaException("Found entries for unknown ID " + nestedId);
if (nestedId > id) break; // fields belong to one of the next entries
func.apply(entry.getValue(), fields);
@@ -943,7 +952,7 @@ class MetaStoreDirectSql {
+ "\"MAX_COL_LEN\", \"NUM_TRUES\", \"NUM_FALSES\", \"LAST_ANALYZED\" ";
private ColumnStatistics makeColumnStats(
- List<Object[]> list, ColumnStatisticsDesc csd, int offset) {
+ List<Object[]> list, ColumnStatisticsDesc csd, int offset) throws MetaException {
ColumnStatistics result = new ColumnStatistics();
result.setStatsDesc(csd);
List<ColumnStatisticsObj> csos = new ArrayList<ColumnStatisticsObj>(list.size());
@@ -951,8 +960,8 @@ class MetaStoreDirectSql {
// LastAnalyzed is stored per column but thrift has it per several;
// get the lowest for now as nobody actually uses this field.
Object laObj = row[offset + 14];
- if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > (Long)laObj)) {
- csd.setLastAnalyzed((Long)laObj);
+ if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > extractSqlLong(laObj))) {
+ csd.setLastAnalyzed(extractSqlLong(laObj));
}
ColumnStatisticsData data = new ColumnStatisticsData();
// see STATS_COLLIST
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1590142&r1=1590141&r2=1590142&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Apr 25 21:00:01 2014
@@ -133,6 +133,7 @@ import org.apache.thrift.TException;
import org.datanucleus.store.rdbms.exceptions.MissingTableException;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
@@ -946,6 +947,16 @@ public class ObjectStore implements RawS
return tables;
}
+ /** Makes shallow copy of a list to avoid DataNucleus mucking with our objects. */
+ private <T> List<T> convertList(List<T> dnList) {
+ return (dnList == null) ? null : Lists.newArrayList(dnList);
+ }
+
+ /** Makes shallow copy of a map to avoid DataNucleus mucking with our objects. */
+ private <K, V> Map<K, V> convertMap(Map<K, V> dnMap) {
+ return (dnMap == null) ? null : Maps.newHashMap(dnMap);
+ }
+
private Table convertToTable(MTable mtbl) throws MetaException {
if (mtbl == null) {
return null;
@@ -964,9 +975,8 @@ public class ObjectStore implements RawS
return new Table(mtbl.getTableName(), mtbl.getDatabase().getName(), mtbl
.getOwner(), mtbl.getCreateTime(), mtbl.getLastAccessTime(), mtbl
.getRetention(), convertToStorageDescriptor(mtbl.getSd()),
- convertToFieldSchemas(mtbl.getPartitionKeys()), mtbl.getParameters(),
- mtbl.getViewOriginalText(), mtbl.getViewExpandedText(),
- tableType);
+ convertToFieldSchemas(mtbl.getPartitionKeys()), convertMap(mtbl.getParameters()),
+ mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), tableType);
}
private MTable convertToMTable(Table tbl) throws InvalidObjectException,
@@ -1057,8 +1067,7 @@ public class ObjectStore implements RawS
if (ms == null) {
throw new MetaException("Invalid SerDeInfo object");
}
- return new SerDeInfo(ms.getName(), ms.getSerializationLib(), ms
- .getParameters());
+ return new SerDeInfo(ms.getName(), ms.getSerializationLib(), convertMap(ms.getParameters()));
}
private MSerDeInfo converToMSerDeInfo(SerDeInfo ms) throws MetaException {
@@ -1094,9 +1103,9 @@ public class ObjectStore implements RawS
StorageDescriptor sd = new StorageDescriptor(noFS ? null : convertToFieldSchemas(mFieldSchemas),
msd.getLocation(), msd.getInputFormat(), msd.getOutputFormat(), msd
.isCompressed(), msd.getNumBuckets(), converToSerDeInfo(msd
- .getSerDeInfo()), msd.getBucketCols(), convertToOrders(msd
- .getSortCols()), msd.getParameters());
- SkewedInfo skewedInfo = new SkewedInfo(msd.getSkewedColNames(),
+ .getSerDeInfo()), convertList(msd.getBucketCols()), convertToOrders(msd
+ .getSortCols()), convertMap(msd.getParameters()));
+ SkewedInfo skewedInfo = new SkewedInfo(convertList(msd.getSkewedColNames()),
convertToSkewedValues(msd.getSkewedColValues()),
covertToSkewedMap(msd.getSkewedColValueLocationMaps()));
sd.setSkewedInfo(skewedInfo);
@@ -1415,10 +1424,10 @@ public class ObjectStore implements RawS
if (mpart == null) {
return null;
}
- return new Partition(mpart.getValues(), mpart.getTable().getDatabase()
+ return new Partition(convertList(mpart.getValues()), mpart.getTable().getDatabase()
.getName(), mpart.getTable().getTableName(), mpart.getCreateTime(),
mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd()),
- mpart.getParameters());
+ convertMap(mpart.getParameters()));
}
private Partition convertToPart(String dbName, String tblName, MPartition mpart)
@@ -1426,9 +1435,9 @@ public class ObjectStore implements RawS
if (mpart == null) {
return null;
}
- return new Partition(mpart.getValues(), dbName, tblName, mpart.getCreateTime(),
- mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd(), false),
- mpart.getParameters());
+ return new Partition(convertList(mpart.getValues()), dbName, tblName,
+ mpart.getCreateTime(), mpart.getLastAccessTime(),
+ convertToStorageDescriptor(mpart.getSd(), false), convertMap(mpart.getParameters()));
}
@Override
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1590142&r1=1590141&r2=1590142&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Fri Apr 25 21:00:01 2014
@@ -556,7 +556,15 @@ public class Warehouse {
public static String makePartName(List<FieldSchema> partCols,
List<String> vals, String defaultStr) throws MetaException {
if ((partCols.size() != vals.size()) || (partCols.size() == 0)) {
- throw new MetaException("Invalid partition key & values");
+ String errorStr = "Invalid partition key & values; keys [";
+ for (FieldSchema fs : partCols) {
+ errorStr += (fs.getName() + ", ");
+ }
+ errorStr += "], values [";
+ for (String val : vals) {
+ errorStr += (val + ", ");
+ }
+ throw new MetaException(errorStr + "]");
}
List<String> colNames = new ArrayList<String>();
for (FieldSchema col: partCols) {