You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by yc...@apache.org on 2015/11/16 17:43:46 UTC
hive git commit: HIVE-12378: Exception on HBaseSerDe.serialize binary
field (Yongzhi Chen, reviewed by Jimmy Xiang)
Repository: hive
Updated Branches:
refs/heads/master 78b74277a -> 3d1eab512
HIVE-12378: Exception on HBaseSerDe.serialize binary field (Yongzhi Chen, reviewed by Jimmy Xiang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3d1eab51
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3d1eab51
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3d1eab51
Branch: refs/heads/master
Commit: 3d1eab512e31c20d760c8a51c948cab87fe92ced
Parents: 78b7427
Author: Yongzhi Chen <yc...@cloudera.com>
Authored: Thu Nov 12 10:57:07 2015 -0500
Committer: Yongzhi Chen <yc...@apache.org>
Committed: Mon Nov 16 11:42:15 2015 -0500
----------------------------------------------------------------------
.../test/queries/positive/hbase_binary_binary.q | 12 +++++
.../results/positive/hbase_binary_binary.q.out | 54 ++++++++++++++++++++
.../hadoop/hive/serde2/lazy/LazyBinary.java | 2 +-
.../hadoop/hive/serde2/lazy/LazyFactory.java | 3 ++
.../hadoop/hive/serde2/lazy/LazyUtils.java | 6 +++
.../hive/serde2/lazydio/LazyDioBinary.java | 51 ++++++++++++++++++
6 files changed, 127 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3d1eab51/hbase-handler/src/test/queries/positive/hbase_binary_binary.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_binary_binary.q b/hbase-handler/src/test/queries/positive/hbase_binary_binary.q
new file mode 100644
index 0000000..388e5aa
--- /dev/null
+++ b/hbase-handler/src/test/queries/positive/hbase_binary_binary.q
@@ -0,0 +1,12 @@
+drop table if exists testhbaseb;
+CREATE TABLE testhbaseb (key int, val binary)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = ":key,cf:val#b"
+);
+insert into table testhbaseb values(1, 'hello');
+insert into table testhbaseb values(2, 'hi');
+select * from testhbaseb;
+drop table testhbaseb;
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/3d1eab51/hbase-handler/src/test/results/positive/hbase_binary_binary.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_binary_binary.q.out b/hbase-handler/src/test/results/positive/hbase_binary_binary.q.out
new file mode 100644
index 0000000..c828fd1
--- /dev/null
+++ b/hbase-handler/src/test/results/positive/hbase_binary_binary.q.out
@@ -0,0 +1,54 @@
+PREHOOK: query: drop table if exists testhbaseb
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists testhbaseb
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE testhbaseb (key int, val binary)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = ":key,cf:val#b"
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testhbaseb
+POSTHOOK: query: CREATE TABLE testhbaseb (key int, val binary)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = ":key,cf:val#b"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testhbaseb
+PREHOOK: query: insert into table testhbaseb values(1, 'hello')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@testhbaseb
+POSTHOOK: query: insert into table testhbaseb values(1, 'hello')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@testhbaseb
+PREHOOK: query: insert into table testhbaseb values(2, 'hi')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@testhbaseb
+POSTHOOK: query: insert into table testhbaseb values(2, 'hi')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@testhbaseb
+PREHOOK: query: select * from testhbaseb
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testhbaseb
+#### A masked pattern was here ####
+POSTHOOK: query: select * from testhbaseb
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testhbaseb
+#### A masked pattern was here ####
+1 hello
+2 hi
+PREHOOK: query: drop table testhbaseb
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@testhbaseb
+PREHOOK: Output: default@testhbaseb
+POSTHOOK: query: drop table testhbaseb
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@testhbaseb
+POSTHOOK: Output: default@testhbaseb
http://git-wip-us.apache.org/repos/asf/hive/blob/3d1eab51/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java
index 010f1f9..aa19d09 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java
@@ -29,7 +29,7 @@ public class LazyBinary extends LazyPrimitive<LazyBinaryObjectInspector, BytesWr
private static final Logger LOG = LoggerFactory.getLogger(LazyBinary.class);
private static final boolean DEBUG_LOG_ENABLED = LOG.isDebugEnabled();
- LazyBinary(LazyBinaryObjectInspector oi) {
+ public LazyBinary(LazyBinaryObjectInspector oi) {
super(oi);
data = new BytesWritable();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3d1eab51/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
index 7aa0fc7..23dbe6a 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyShortObj
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyTimestampObjectInspector;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyVoidObjectInspector;
+import org.apache.hadoop.hive.serde2.lazydio.LazyDioBinary;
import org.apache.hadoop.hive.serde2.lazydio.LazyDioBoolean;
import org.apache.hadoop.hive.serde2.lazydio.LazyDioByte;
import org.apache.hadoop.hive.serde2.lazydio.LazyDioDouble;
@@ -162,6 +163,8 @@ public final class LazyFactory {
return new LazyDioFloat((LazyFloatObjectInspector) poi);
case DOUBLE:
return new LazyDioDouble((LazyDoubleObjectInspector) poi);
+ case BINARY:
+ return new LazyDioBinary((LazyBinaryObjectInspector) poi);
default:
throw new RuntimeException("Hive Internal Error: no LazyObject for " + poi);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3d1eab51/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
index 29d6ad8..ee39196 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
@@ -368,6 +368,12 @@ public final class LazyUtils {
dos.writeDouble(d);
break;
+ case BINARY: {
+ BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
+ out.write(bw.getBytes(), 0, bw.getLength());
+ break;
+ }
+
default:
throw new RuntimeException("Hive internal error.");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3d1eab51/serde/src/java/org/apache/hadoop/hive/serde2/lazydio/LazyDioBinary.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazydio/LazyDioBinary.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazydio/LazyDioBinary.java
new file mode 100644
index 0000000..a3e4cac
--- /dev/null
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazydio/LazyDioBinary.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazydio;
+
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyBinary;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBinaryObjectInspector;
+
+public class LazyDioBinary extends LazyBinary {
+
+ public LazyDioBinary(LazyBinaryObjectInspector oi) {
+ super(oi);
+ }
+
+ LazyDioBinary(LazyDioBinary copy) {
+ super(copy);
+ }
+
+ /* (non-Javadoc)
+ * This provides a LazyBinary like class which can be initialized from data stored in a
+ * binary format.
+ *
+ * @see org.apache.hadoop.hive.serde2.lazy.LazyObject#init
+ * (org.apache.hadoop.hive.serde2.lazy.ByteArrayRef, int, int)
+ */
+ @Override
+ public void init(ByteArrayRef bytes, int start, int length) {
+ if (bytes == null) {
+ throw new RuntimeException("bytes cannot be null!");
+ }
+ isNull = false;
+ byte[] recv = new byte[length];
+ System.arraycopy(bytes.getData(), start, recv, 0, length);
+ data.set(recv, 0, length);
+ }
+}