You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/04/04 22:36:55 UTC
[20/50] [abbrv] hive git commit: HIVE-13115: MetaStore Direct SQL
getPartitions call fail when the columns schemas for a partition are null
(Ratandeep Ratti reviewed by Carl Steinbach)
HIVE-13115: MetaStore Direct SQL getPartitions call fail when the columns schemas for a partition are null (Ratandeep Ratti reviewed by Carl Steinbach)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/69cfd357
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/69cfd357
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/69cfd357
Branch: refs/heads/llap
Commit: 69cfd357eb482c426161aa3c4a00d574ee29416e
Parents: 9686209
Author: Carl Steinbach <cw...@apache.org>
Authored: Sun Mar 27 15:41:38 2016 -0700
Committer: Carl Steinbach <cw...@apache.org>
Committed: Sun Mar 27 15:41:38 2016 -0700
----------------------------------------------------------------------
.../hive/metastore/TestHiveMetaStore.java | 20 +++++++++++-
.../hive/metastore/MetaStoreDirectSql.java | 32 +++++++++++---------
2 files changed, 37 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/69cfd357/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 5da4165..83fb15c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -280,6 +280,24 @@ public abstract class TestHiveMetaStore extends TestCase {
}
assertTrue("Partitions are not same", part.equals(part_get));
+ // check null cols schemas for a partition
+ List<String> vals6 = makeVals("2016-02-22 00:00:00", "16");
+ Partition part6 = makePartitionObject(dbName, tblName, vals6, tbl, "/part5");
+ part6.getSd().setCols(null);
+ LOG.info("Creating partition will null field schema");
+ client.add_partition(part6);
+ LOG.info("Listing all partitions for table " + dbName + "." + tblName);
+ final List<Partition> partitions = client.listPartitions(dbName, tblName, (short) -1);
+ boolean foundPart = false;
+ for (Partition p : partitions) {
+ if (p.getValues().equals(vals6)) {
+ assertNull(p.getSd().getCols());
+ LOG.info("Found partition " + p + " having null field schema");
+ foundPart = true;
+ }
+ }
+ assertTrue(foundPart);
+
String partName = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=14";
String part2Name = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=15";
String part3Name = "ds=" + FileUtils.escapePathName("2008-07-02 14:13:12") + "/hr=15";
@@ -313,7 +331,7 @@ public abstract class TestHiveMetaStore extends TestCase {
partialVals.clear();
partialVals.add("");
partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1);
- assertTrue("Should have returned 4 partition names", partialNames.size() == 4);
+ assertTrue("Should have returned 5 partition names", partialNames.size() == 5);
assertTrue("Not all part names returned", partialNames.containsAll(partNames));
// Test partition listing with a partial spec - hr is specified but ds is not
http://git-wip-us.apache.org/repos/asf/hive/blob/69cfd357/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index d51f58d..06e9f78 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -557,14 +557,14 @@ class MetaStoreDirectSql {
Long sdId = extractSqlLong(fields[1]);
Long colId = extractSqlLong(fields[2]);
Long serdeId = extractSqlLong(fields[3]);
- // A partition must have either everything set, or nothing set if it's a view.
- if (sdId == null || colId == null || serdeId == null) {
+ // A partition must have at least sdId and serdeId set, or nothing set if it's a view.
+ if (sdId == null || serdeId == null) {
if (isView == null) {
isView = isViewTable(dbName, tblName);
}
if ((sdId != null || colId != null || serdeId != null) || !isView) {
- throw new MetaException("Unexpected null for one of the IDs, SD " + sdId + ", column "
- + colId + ", serde " + serdeId + " for a " + (isView ? "" : "non-") + " view");
+ throw new MetaException("Unexpected null for one of the IDs, SD " + sdId +
+ ", serde " + serdeId + " for a " + (isView ? "" : "non-") + " view");
}
}
@@ -580,7 +580,7 @@ class MetaStoreDirectSql {
partitions.put(partitionId, part);
if (sdId == null) continue; // Probably a view.
- assert colId != null && serdeId != null;
+ assert serdeId != null;
// We assume each partition has an unique SD.
StorageDescriptor sd = new StorageDescriptor();
@@ -605,14 +605,16 @@ class MetaStoreDirectSql {
sdSb.append(sdId).append(",");
part.setSd(sd);
- List<FieldSchema> cols = colss.get(colId);
- // We expect that colId will be the same for all (or many) SDs.
- if (cols == null) {
- cols = new ArrayList<FieldSchema>();
- colss.put(colId, cols);
- colsSb.append(colId).append(",");
+ if (colId != null) {
+ List<FieldSchema> cols = colss.get(colId);
+ // We expect that colId will be the same for all (or many) SDs.
+ if (cols == null) {
+ cols = new ArrayList<FieldSchema>();
+ colss.put(colId, cols);
+ colsSb.append(colId).append(",");
+ }
+ sd.setCols(cols);
}
- sd.setCols(cols);
// We assume each SD has an unique serde.
SerDeInfo serde = new SerDeInfo();
@@ -658,8 +660,10 @@ class MetaStoreDirectSql {
assert serdeSb.length() == 0 && colsSb.length() == 0;
return orderedResult; // No SDs, probably a view.
}
- String sdIds = trimCommaList(sdSb), serdeIds = trimCommaList(serdeSb),
- colIds = trimCommaList(colsSb);
+
+ String sdIds = trimCommaList(sdSb);
+ String serdeIds = trimCommaList(serdeSb);
+ String colIds = trimCommaList(colsSb);
// Get all the stuff for SD. Don't do empty-list check - we expect partitions do have SDs.
queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from \"SD_PARAMS\""