You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gs...@apache.org on 2022/11/10 16:57:42 UTC
[hive] branch master updated: HIVE-26667: Incompatible expression deserialization against latest HMS (Zhihua Deng reviewed by Sai Hemanth)
This is an automated email from the ASF dual-hosted git repository.
gsaihemanth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 7a238e67ad4 HIVE-26667: Incompatible expression deserialization against latest HMS (Zhihua Deng reviewed by Sai Hemanth)
7a238e67ad4 is described below
commit 7a238e67ad46cf8b2d9ca1d69672b8f4e169fe8e
Author: dengzh <de...@gmail.com>
AuthorDate: Fri Nov 11 00:57:32 2022 +0800
HIVE-26667: Incompatible expression deserialization against latest HMS (Zhihua Deng reviewed by Sai Hemanth)
---
.../hive/ql/exec/SerializationUtilities.java | 2 +-
.../ppr/PartitionExpressionForMetastore.java | 11 ++++--
.../hadoop/hive/metastore/TestMetastoreExpr.java | 40 +++++++++++++---------
3 files changed, 33 insertions(+), 20 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
index 7e7f6138e59..8c95f5d5b2b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java
@@ -897,7 +897,7 @@ public class SerializationUtilities {
return baos.toByteArray();
}
- private static <T extends Serializable> T deserializeObjectFromKryo(byte[] bytes, Class<T> clazz) {
+ public static <T extends Serializable> T deserializeObjectFromKryo(byte[] bytes, Class<T> clazz) {
Input inp = new Input(new ByteArrayInputStream(bytes));
Kryo kryo = borrowKryo();
T func = null;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
index 6547d1d1c7a..ec591c80000 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.slf4j.Logger;
@@ -109,8 +110,14 @@ public class PartitionExpressionForMetastore implements PartitionExpressionProxy
try {
expr = SerializationUtilities.deserializeObjectWithTypeInformation(exprBytes, true);
} catch (Exception ex) {
- LOG.error("Failed to deserialize the expression", ex);
- throw new MetaException(ex.getMessage());
+ LOG.error("Failed to deserialize the expression, fall back to deserializeObjectFromKryo", ex);
+ try {
+ expr = SerializationUtilities.deserializeObjectFromKryo(exprBytes, ExprNodeGenericFuncDesc.class);
+ } catch (Exception e) {
+ LOG.error("Failed to deserialize the expression", e);
+ throw new MetaException("SerializationUtilities#deserializeObjectWithTypeInformation: " + ex.getMessage() +
+ ", SerializationUtilities#deserializeObjectFromKryo: " + e.getMessage());
+ }
}
if (expr == null) {
throw new MetaException("Failed to deserialize expression - ExprNodeDesc not present");
diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java b/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java
index 093f3bc9b68..222c27170a0 100644
--- a/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/TestMetastoreExpr.java
@@ -181,24 +181,30 @@ public class TestMetastoreExpr {
public void checkExpr(int numParts,
String dbName, String tblName, ExprNodeGenericFuncDesc expr, Table t) throws Exception {
- List<Partition> parts = new ArrayList<Partition>();
- client.listPartitionsByExpr(dbName, tblName,
- SerializationUtilities.serializeObjectWithTypeInformation(expr), null, (short)-1, parts);
- assertEquals("Partition check failed: " + expr.getExprString(), numParts, parts.size());
-
- // check with partition spec as well
- PartitionsByExprRequest req = new PartitionsByExprRequest(dbName, tblName,
- ByteBuffer.wrap(SerializationUtilities.serializeObjectWithTypeInformation(expr)));
- req.setMaxParts((short)-1);
- req.setId(t.getId());
-
- List<PartitionSpec> partSpec = new ArrayList<>();
- client.listPartitionsSpecByExpr(req, partSpec);
- int partSpecSize = 0;
- if(!partSpec.isEmpty()) {
- partSpecSize = partSpec.iterator().next().getSharedSDPartitionSpec().getPartitionsSize();
+ List<byte[]> exprBytes = new ArrayList<>();
+ exprBytes.add(SerializationUtilities.serializeObjectWithTypeInformation(expr));
+ // old client
+ exprBytes.add(SerializationUtilities.serializeObjectToKryo(expr));
+ for (int i = 0; i < exprBytes.size(); i++) {
+ List<Partition> parts = new ArrayList<Partition>();
+ client.listPartitionsByExpr(dbName, tblName,
+ exprBytes.get(i), null, (short)-1, parts);
+ assertEquals("Partition check failed: " + expr.getExprString(), numParts, parts.size());
+
+ // check with partition spec as well
+ PartitionsByExprRequest req = new PartitionsByExprRequest(dbName, tblName,
+ ByteBuffer.wrap(exprBytes.get(i)));
+ req.setMaxParts((short)-1);
+ req.setId(t.getId());
+
+ List<PartitionSpec> partSpec = new ArrayList<>();
+ client.listPartitionsSpecByExpr(req, partSpec);
+ int partSpecSize = 0;
+ if(!partSpec.isEmpty()) {
+ partSpecSize = partSpec.iterator().next().getSharedSDPartitionSpec().getPartitionsSize();
+ }
+ assertEquals("Partition Spec check failed: " + expr.getExprString(), numParts, partSpecSize);
}
- assertEquals("Partition Spec check failed: " + expr.getExprString(), numParts, partSpecSize);
}