You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2015/07/27 18:29:27 UTC
hive git commit: HIVE-11344 : HIVE-9845 makes HCatSplit.write modify
the split so that PartInfo objects are unusable after it (Sushanth Sowmyan,
reviewed by Mithun Radhakrishnan)
Repository: hive
Updated Branches:
refs/heads/master 92e98858e -> 9f474263b
HIVE-11344 : HIVE-9845 makes HCatSplit.write modify the split so that PartInfo objects are unusable after it (Sushanth Sowmyan, reviewed by Mithun Radhakrishnan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f474263
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f474263
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f474263
Branch: refs/heads/master
Commit: 9f474263b5c47dc25c096a0b54794ebcda146bed
Parents: 92e9885
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Mon Jul 27 09:21:54 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Mon Jul 27 09:29:18 2015 -0700
----------------------------------------------------------------------
.../hive/hcatalog/mapreduce/PartInfo.java | 32 +++++++++++++++-----
1 file changed, 25 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9f474263/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
index fca0a92..5b88505 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/PartInfo.java
@@ -173,7 +173,16 @@ public class PartInfo implements Serializable {
void setTableInfo(HCatTableInfo thatTableInfo) {
this.tableInfo = thatTableInfo;
+ restoreLocalInfoFromTableInfo();
+ }
+ /**
+ * Undoes the effects of compression( dedupWithTableInfo() ) during serialization,
+ * and restores PartInfo fields to return original data.
+ * Can be called idempotently, repeatably.
+ */
+ private void restoreLocalInfoFromTableInfo() {
+ assert tableInfo != null : "TableInfo can't be null at this point.";
if (partitionSchema == null) {
partitionSchema = tableInfo.getDataColumns();
}
@@ -196,15 +205,10 @@ public class PartInfo implements Serializable {
}
/**
- * Serialization method. Suppresses serialization of redundant information that's already
- * available from TableInfo.
+ * Finds commonalities with TableInfo, and suppresses (nulls) fields if they are identical
*/
- private void writeObject(ObjectOutputStream oos)
- throws IOException {
- // Suppress commonality with TableInfo.
-
+ private void dedupWithTableInfo() {
assert tableInfo != null : "TableInfo can't be null at this point.";
-
if (partitionSchema != null) {
if (partitionSchema.equals(tableInfo.getDataColumns())) {
partitionSchema = null;
@@ -260,7 +264,21 @@ public class PartInfo implements Serializable {
}
}
}
+ }
+ /**
+ * Serialization method used by java serialization.
+ * Suppresses serialization of redundant information that's already available from
+ * TableInfo before writing out, so as to minimize amount of serialized space but
+ * restore it back before returning, so that PartInfo object is still usable afterwards
+ * (See HIVE-8485 and HIVE-11344 for details.)
+ */
+ private void writeObject(ObjectOutputStream oos)
+ throws IOException {
+ dedupWithTableInfo();
oos.defaultWriteObject();
+ restoreLocalInfoFromTableInfo();
}
+
+
}