You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/08/13 15:18:37 UTC
svn commit: r1513474 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/udf/
test/org/apache/hadoop/hive/ql/udf/ test/queries/clientpositive/
test/results/clientpositive/
Author: hashutosh
Date: Tue Aug 13 13:18:36 2013
New Revision: 1513474
URL: http://svn.apache.org/r1513474
Log:
HIVE-2482 : Convenience UDFs for binary data type (Mark Wagner via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFDecode.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFEncode.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBase64.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnbase64.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDecode.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFEncode.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFBase64.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFHex.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFUnbase64.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFUnhex.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHex.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnhex.java
hive/trunk/ql/src/test/queries/clientpositive/ba_table_udfs.q
hive/trunk/ql/src/test/queries/clientpositive/udf_sentences.q
hive/trunk/ql/src/test/results/clientpositive/ba_table_udfs.q.out
hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
hive/trunk/ql/src/test/results/clientpositive/udf_hex.q.out
hive/trunk/ql/src/test/results/clientpositive/udf_sentences.q.out
hive/trunk/ql/src/test/results/clientpositive/udf_unhex.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1513474&r1=1513473&r2=1513474&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Tue Aug 13 13:18:36 2013
@@ -43,12 +43,15 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.udf.GenericUDFDecode;
+import org.apache.hadoop.hive.ql.udf.GenericUDFEncode;
import org.apache.hadoop.hive.ql.udf.UDAFPercentile;
import org.apache.hadoop.hive.ql.udf.UDFAbs;
import org.apache.hadoop.hive.ql.udf.UDFAcos;
import org.apache.hadoop.hive.ql.udf.UDFAscii;
import org.apache.hadoop.hive.ql.udf.UDFAsin;
import org.apache.hadoop.hive.ql.udf.UDFAtan;
+import org.apache.hadoop.hive.ql.udf.UDFBase64;
import org.apache.hadoop.hive.ql.udf.UDFBin;
import org.apache.hadoop.hive.ql.udf.UDFCeil;
import org.apache.hadoop.hive.ql.udf.UDFConcat;
@@ -122,6 +125,7 @@ import org.apache.hadoop.hive.ql.udf.UDF
import org.apache.hadoop.hive.ql.udf.UDFToString;
import org.apache.hadoop.hive.ql.udf.UDFTrim;
import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.UDFUnbase64;
import org.apache.hadoop.hive.ql.udf.UDFUnhex;
import org.apache.hadoop.hive.ql.udf.UDFUpper;
import org.apache.hadoop.hive.ql.udf.UDFWeekOfYear;
@@ -229,6 +233,11 @@ public final class FunctionRegistry {
registerUDF("bin", UDFBin.class, false);
registerUDF("hex", UDFHex.class, false);
registerUDF("unhex", UDFUnhex.class, false);
+ registerUDF("base64", UDFBase64.class, false);
+ registerUDF("unbase64", UDFUnbase64.class, false);
+
+ registerGenericUDF("encode", GenericUDFEncode.class);
+ registerGenericUDF("decode", GenericUDFDecode.class);
registerUDF("upper", UDFUpper.class, false);
registerUDF("lower", UDFLower.class, false);
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFDecode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFDecode.java?rev=1513474&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFDecode.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFDecode.java Tue Aug 13 13:18:36 2013
@@ -0,0 +1,96 @@
+package org.apache.hadoop.hive.ql.udf;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.io.Text;
+
+@Description(name = "decode",
+ value = "_FUNC_(bin, str) - Decode the first argument using the second argument character set",
+ extended = "Possible options for the character set are 'US_ASCII', 'ISO-8859-1',\n" +
+ "'UTF-8', 'UTF-16BE', 'UTF-16LE', and 'UTF-16'. If either argument\n" +
+ "is null, the result will also be null")
+public class GenericUDFDecode extends GenericUDF {
+ private transient CharsetDecoder decoder = null;
+ private transient BinaryObjectInspector bytesOI = null;
+ private transient StringObjectInspector charsetOI = null;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ if (arguments.length != 2) {
+ throw new UDFArgumentLengthException("Encode() requires exactly two arguments");
+ }
+
+ if (arguments[0].getCategory() != Category.PRIMITIVE ||
+ ((PrimitiveObjectInspector)arguments[0]).getPrimitiveCategory() != PrimitiveCategory.BINARY){
+ throw new UDFArgumentTypeException(0, "The first argument to Encode() must be a binary");
+ }
+
+ bytesOI = (BinaryObjectInspector) arguments[0];
+
+ if (arguments[1].getCategory() != Category.PRIMITIVE ||
+ ((PrimitiveObjectInspector)arguments[1]).getPrimitiveCategory() != PrimitiveCategory.STRING){
+ throw new UDFArgumentTypeException(1, "The second argument to Encode() must be a string");
+ }
+
+ charsetOI = (StringObjectInspector) arguments[1];
+
+ // If the character set for encoding is constant, we can optimize that
+ StringObjectInspector charSetOI = (StringObjectInspector) arguments[1];
+ if (charSetOI instanceof ConstantObjectInspector){
+ String charSetName = ((Text) ((ConstantObjectInspector) charSetOI).getWritableConstantValue()).toString();
+ decoder = Charset.forName(charSetName).newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT);
+ }
+
+ return (ObjectInspector) PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ byte[] value = bytesOI.getPrimitiveJavaObject(arguments[0].get());
+ if (value == null) {
+ return null;
+ }
+
+ ByteBuffer wrappedBytes = ByteBuffer.wrap(value);
+ CharBuffer decoded;
+ if (decoder != null){
+ try {
+ decoded = decoder.decode(wrappedBytes);
+ } catch (CharacterCodingException e) {
+ throw new HiveException(e);
+ }
+ } else {
+ decoded = Charset.forName(charsetOI.getPrimitiveJavaObject(arguments[1].get())).decode(wrappedBytes);
+ }
+ return decoded.toString();
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ assert (children.length == 2);
+ StringBuilder sb = new StringBuilder();
+ sb.append("encode(");
+ sb.append(children[0]).append(",");
+ sb.append(children[1]).append(")");
+ return sb.toString();
+ }
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFEncode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFEncode.java?rev=1513474&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFEncode.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/GenericUDFEncode.java Tue Aug 13 13:18:36 2013
@@ -0,0 +1,100 @@
+package org.apache.hadoop.hive.ql.udf;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+@Description(name = "encode",
+value = "_FUNC_(str, str) - Encode the first argument using the second argument character set",
+extended = "Possible options for the character set are 'US_ASCII', 'ISO-8859-1',\n" +
+ "'UTF-8', 'UTF-16BE', 'UTF-16LE', and 'UTF-16'. If either argument\n" +
+ "is null, the result will also be null")
+public class GenericUDFEncode extends GenericUDF {
+ private transient CharsetEncoder encoder = null;
+ private transient StringObjectInspector stringOI = null;
+ private transient StringObjectInspector charsetOI = null;
+ private transient BytesWritable result = new BytesWritable();
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ if (arguments.length != 2) {
+ throw new UDFArgumentLengthException("Encode() requires exactly two arguments");
+ }
+
+ if (arguments[0].getCategory() != Category.PRIMITIVE ||
+ ((PrimitiveObjectInspector)arguments[0]).getPrimitiveCategory() != PrimitiveCategory.STRING){
+ throw new UDFArgumentTypeException(0, "The first argument to Encode() must be a string");
+ }
+
+ stringOI = (StringObjectInspector) arguments[0];
+
+ if (arguments[1].getCategory() != Category.PRIMITIVE ||
+ ((PrimitiveObjectInspector)arguments[1]).getPrimitiveCategory() != PrimitiveCategory.STRING){
+ throw new UDFArgumentTypeException(1, "The second argument to Encode() must be a string");
+ }
+
+ charsetOI = (StringObjectInspector) arguments[1];
+
+ // If the character set for encoding is constant, we can optimize that
+ StringObjectInspector charSetOI = (StringObjectInspector) arguments[1];
+ if (charSetOI instanceof ConstantObjectInspector){
+ String charSetName = ((Text) ((ConstantObjectInspector) charSetOI).getWritableConstantValue()).toString();
+ encoder = Charset.forName(charSetName).newEncoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT);
+ }
+
+ result = new BytesWritable();
+
+ return (ObjectInspector) PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ String value = stringOI.getPrimitiveJavaObject(arguments[0].get());
+ if (value == null) {
+ return null;
+ }
+
+ ByteBuffer encoded;
+ if (encoder != null){
+ try {
+ encoded = encoder.encode(CharBuffer.wrap(value));
+ } catch (CharacterCodingException e) {
+ throw new HiveException(e);
+ }
+ } else {
+ encoded = Charset.forName(charsetOI.getPrimitiveJavaObject(arguments[1].get())).encode(value);
+ }
+ result.setSize(encoded.limit());
+ encoded.get(result.getBytes(), 0, encoded.limit());
+ return result;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ assert (children.length == 2);
+ StringBuilder sb = new StringBuilder();
+ sb.append("encode(");
+ sb.append(children[0]).append(",");
+ sb.append(children[1]).append(")");
+ return sb.toString();
+ }
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBase64.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBase64.java?rev=1513474&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBase64.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFBase64.java Tue Aug 13 13:18:36 2013
@@ -0,0 +1,23 @@
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+@Description(name = "base64",
+ value = "_FUNC_(bin) - Convert the argument from binary to a base 64 string")
+public class UDFBase64 extends UDF {
+ private final transient Text result = new Text();
+
+ public Text evaluate(BytesWritable b){
+ if (b == null) {
+ return null;
+ }
+ byte[] bytes = new byte[b.getLength()];
+ System.arraycopy(b.getBytes(), 0, bytes, 0, b.getLength());
+ result.set(Base64.encodeBase64(bytes));
+ return result;
+ }
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHex.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHex.java?rev=1513474&r1=1513473&r2=1513474&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHex.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHex.java Tue Aug 13 13:18:36 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -29,10 +30,10 @@ import org.apache.hadoop.io.Text;
*
*/
@Description(name = "hex",
- value = "_FUNC_(n or str) - Convert the argument to hexadecimal ",
+ value = "_FUNC_(n, bin, or str) - Convert the argument to hexadecimal ",
extended = "If the argument is a string, returns two hex digits for each "
+ "character in the string.\n"
- + "If the argument is a number, returns the hexadecimal representation.\n"
+ + "If the argument is a number or binary, returns the hexadecimal representation.\n"
+ "Example:\n"
+ " > SELECT _FUNC_(17) FROM src LIMIT 1;\n"
+ " 'H1'\n"
@@ -44,7 +45,7 @@ public class UDFHex extends UDF {
/**
* Convert num to hex.
- *
+ *
*/
private Text evaluate(long num) {
// Extract the hex digits of num into value[] from right to left
@@ -76,26 +77,41 @@ public class UDFHex extends UDF {
/**
* Convert every character in s to two hex digits.
- *
+ *
*/
public Text evaluate(Text s) {
if (s == null) {
return null;
}
- if (value.length < s.getLength() * 2) {
- value = new byte[s.getLength() * 2];
+ byte[] str = s.getBytes();
+ return evaluate(str, s.getLength());
+ }
+
+ /**
+ * Convert bytes to a hex string
+ */
+ public Text evaluate(BytesWritable b){
+ if (b == null) {
+ return null;
}
- byte[] str = s.getBytes();
- for (int i = 0; i < s.getLength(); i++) {
+ byte[] bytes = b.getBytes();
+ return evaluate(bytes, b.getLength());
+ }
+
+ private Text evaluate(byte[] bytes, int length){
+ if (value.length < length * 2) {
+ value = new byte[length * 2];
+ }
+
+ for (int i = 0; i < length; i++) {
value[i * 2] = (byte) Character.toUpperCase(Character.forDigit(
- (str[i] & 0xF0) >>> 4, 16));
+ (bytes[i] & 0xF0) >>> 4, 16));
value[i * 2 + 1] = (byte) Character.toUpperCase(Character.forDigit(
- str[i] & 0x0F, 16));
+ bytes[i] & 0x0F, 16));
}
-
- result.set(value, 0, s.getLength() * 2);
+ result.set(value, 0, length*2);
return result;
}
}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnbase64.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnbase64.java?rev=1513474&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnbase64.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnbase64.java Tue Aug 13 13:18:36 2013
@@ -0,0 +1,24 @@
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+@Description(name = "unbase64",
+ value = "_FUNC_(str) - Convert the argument from a base 64 string to binary")
+public class UDFUnbase64 extends UDF {
+ private final transient BytesWritable result = new BytesWritable();
+
+ public BytesWritable evaluate(Text value){
+ if (value == null) {
+ return null;
+ }
+ byte[] bytes = new byte[value.getLength()];
+ System.arraycopy(value.getBytes(), 0, bytes, 0, value.getLength());
+ byte[] decoded = Base64.decodeBase64(bytes);
+ result.set(decoded, 0, decoded.length);
+ return result;
+ }
+}
\ No newline at end of file
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnhex.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnhex.java?rev=1513474&r1=1513473&r2=1513474&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnhex.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFUnhex.java Tue Aug 13 13:18:36 2013
@@ -27,18 +27,14 @@ import org.apache.hadoop.io.Text;
*
*/
@Description(name = "unhex",
- value = "_FUNC_(str) - Converts hexadecimal argument to string",
+ value = "_FUNC_(str) - Converts hexadecimal argument to binary",
extended = "Performs the inverse operation of HEX(str). That is, it interprets\n"
+ "each pair of hexadecimal digits in the argument as a number and\n"
- + "converts it to the character represented by the number. The\n"
+ + "converts it to the byte representation of the number. The\n"
+ "resulting characters are returned as a binary string.\n\n"
+ "Example:\n"
- + "> SELECT UNHEX('4D7953514C') from src limit 1;\n"
- + "'MySQL'\n"
- + "> SELECT UNHEX(HEX('string')) from src limit 1;\n"
- + "'string'\n"
- + "> SELECT HEX(UNHEX('1267')) from src limit 1;\n"
- + "'1267'\n\n"
+ + "> SELECT DECODE(UNHEX('4D7953514C'), 'UTF-8') from src limit 1;\n"
+ + "'MySQL'\n\n"
+ "The characters in the argument string must be legal hexadecimal\n"
+ "digits: '0' .. '9', 'A' .. 'F', 'a' .. 'f'. If UNHEX() encounters\n"
+ "any nonhexadecimal digits in the argument, it returns NULL. Also,\n"
@@ -47,9 +43,9 @@ public class UDFUnhex extends UDF {
/**
* Convert every two hex digits in s into.
- *
+ *
*/
- public Text evaluate(Text s) {
+ public byte[] evaluate(Text s) {
if (s == null) {
return null;
}
@@ -72,6 +68,6 @@ public class UDFUnhex extends UDF {
}
}
- return new Text(result);
+ return result;
}
}
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDecode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDecode.java?rev=1513474&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDecode.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFDecode.java Tue Aug 13 13:18:36 2013
@@ -0,0 +1,38 @@
+package org.apache.hadoop.hive.ql.udf;
+
+import java.io.UnsupportedEncodingException;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+public class TestGenericUDFDecode extends TestCase {
+ public void testDecode() throws UnsupportedEncodingException, HiveException {
+ String[] charsetNames = {"US-ASCII", "ISO-8859-1", "UTF-8", "UTF-16BE", "UTF-16LE", "UTF-16"};
+ for (String charsetName : charsetNames){
+ verifyDecode("A sample string", charsetName);
+ }
+ }
+
+ public void verifyDecode(String string, String charsetName) throws UnsupportedEncodingException, HiveException{
+ GenericUDFDecode udf = new GenericUDFDecode();
+ byte[] bytes = string.getBytes(charsetName);
+
+ ObjectInspector valueOI = PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector;
+ ObjectInspector charsetOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector[] initArguments = {valueOI, charsetOI};
+ udf.initialize(initArguments);
+
+ DeferredObject valueObj = new DeferredJavaObject(bytes);
+ DeferredObject charsetObj = new DeferredJavaObject(charsetName);
+ DeferredObject[] arguments = {valueObj, charsetObj};
+ String output = (String) udf.evaluate(arguments);
+
+ assertEquals("Decoding failed for CharSet: " + charsetName, string, output);
+ }
+}
+
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFEncode.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFEncode.java?rev=1513474&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFEncode.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestGenericUDFEncode.java Tue Aug 13 13:18:36 2013
@@ -0,0 +1,42 @@
+package org.apache.hadoop.hive.ql.udf;
+
+import java.io.UnsupportedEncodingException;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.BytesWritable;
+
+public class TestGenericUDFEncode extends TestCase {
+ public void testEncode() throws UnsupportedEncodingException, HiveException{
+ String[] charsetNames = {"US-ASCII", "ISO-8859-1", "UTF-8", "UTF-16BE", "UTF-16LE", "UTF-16"};
+ for (String charsetName : charsetNames){
+ verifyEncode("A sample string", charsetName);
+ }
+ }
+
+ public void verifyEncode(String string, String charsetName) throws UnsupportedEncodingException, HiveException{
+ GenericUDFEncode udf = new GenericUDFEncode();
+ byte[] expected = string.getBytes(charsetName);
+
+ ObjectInspector valueOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector charsetOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+ ObjectInspector[] initArguments = {valueOI, charsetOI};
+ udf.initialize(initArguments);
+
+ DeferredObject valueObj = new DeferredJavaObject(string);
+ DeferredObject charsetObj = new DeferredJavaObject(charsetName);
+ DeferredObject[] arguments = {valueObj, charsetObj};
+ BytesWritable outputWritable = (BytesWritable) udf.evaluate(arguments);
+
+ byte[] output = outputWritable.getBytes();
+ assertTrue("Encoding failed for CharSet: " + charsetName, expected.length == outputWritable.getLength());
+ for (int i = 0; i < expected.length; i++){
+ assertEquals("Encoding failed for CharSet: " + charsetName, expected[i], output[i]);
+ }
+ }
+}
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFBase64.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFBase64.java?rev=1513474&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFBase64.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFBase64.java Tue Aug 13 13:18:36 2013
@@ -0,0 +1,20 @@
+package org.apache.hadoop.hive.ql.udf;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+public class TestUDFBase64 extends TestCase {
+ public void testBase64Conversion(){
+ byte[] bytes = "string".getBytes();
+ // Let's make sure we only read the relevant part of the writable in case of reuse
+ byte[] longBytes = "longer string".getBytes();
+ BytesWritable writable = new BytesWritable(longBytes);
+ writable.set(bytes, 0, bytes.length);
+ UDFBase64 udf = new UDFBase64();
+ Text text = udf.evaluate(writable);
+ String base64String = text.toString();
+ assertEquals("c3RyaW5n", base64String);
+ }
+}
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFHex.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFHex.java?rev=1513474&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFHex.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFHex.java Tue Aug 13 13:18:36 2013
@@ -0,0 +1,20 @@
+package org.apache.hadoop.hive.ql.udf;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+public class TestUDFHex extends TestCase {
+ public void testHexConversion(){
+ byte[] bytes = "string".getBytes();
+ // Let's make sure we only read the relevant part of the writable in case of reuse
+ byte[] longBytes = "longer string".getBytes();
+ BytesWritable writable = new BytesWritable(longBytes);
+ writable.set(bytes, 0, bytes.length);
+ UDFHex udf = new UDFHex();
+ Text text = udf.evaluate(writable);
+ String hexString = text.toString();
+ assertEquals("737472696E67", hexString);
+ }
+}
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFUnbase64.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFUnbase64.java?rev=1513474&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFUnbase64.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFUnbase64.java Tue Aug 13 13:18:36 2013
@@ -0,0 +1,21 @@
+package org.apache.hadoop.hive.ql.udf;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+public class TestUDFUnbase64 extends TestCase {
+ public void testUnbase64Conversion(){
+ Text base64 = new Text();
+ // Let's make sure we only read the relevant part of the writable in case of reuse
+ base64.set("Garbage 64. Should be ignored.");
+ base64.set("c3RyaW5n");
+
+ BytesWritable expected = new BytesWritable("string".getBytes());
+
+ UDFUnbase64 udf = new UDFUnbase64();
+ BytesWritable output = udf.evaluate(base64);
+ assertEquals(expected, output);
+ }
+}
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFUnhex.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFUnhex.java?rev=1513474&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFUnhex.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFUnhex.java Tue Aug 13 13:18:36 2013
@@ -0,0 +1,23 @@
+package org.apache.hadoop.hive.ql.udf;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.io.Text;
+
+public class TestUDFUnhex extends TestCase {
+ public void testUnhexConversion(){
+ Text hex = new Text();
+ // Let's make sure we only read the relevant part of the writable in case of reuse
+ hex.set("57686174207765726520796F7520686F70696E6720666F723F");
+ hex.set("737472696E67");
+
+ byte[] expected = "string".getBytes();
+
+ UDFUnhex udf = new UDFUnhex();
+ byte[] output = udf.evaluate(hex);
+ assertEquals(expected.length,output.length);
+ for (int i = 0; i < expected.length; i++){
+ assertEquals(expected[i], output[i]);
+ }
+ }
+}
Modified: hive/trunk/ql/src/test/queries/clientpositive/ba_table_udfs.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/ba_table_udfs.q?rev=1513474&r1=1513473&r2=1513474&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/ba_table_udfs.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/ba_table_udfs.q Tue Aug 13 13:18:36 2013
@@ -1,27 +1,36 @@
USE default;
CREATE TABLE dest1(bytes1 BINARY,
- bytes2 BINARY);
+ bytes2 BINARY,
+ string STRING);
FROM src INSERT OVERWRITE TABLE dest1
SELECT
CAST(key AS BINARY),
- CAST(value AS BINARY)
+ CAST(value AS BINARY),
+ value
ORDER BY value
LIMIT 100;
--Add in a null row for good measure
-INSERT INTO TABLE dest1 SELECT NULL, NULL FROM dest1 LIMIT 1;
+INSERT INTO TABLE dest1 SELECT NULL, NULL, NULL FROM dest1 LIMIT 1;
-- this query tests all the udfs provided to work with binary types
SELECT
bytes1,
bytes2,
+ string,
LENGTH(bytes1),
CONCAT(bytes1, bytes2),
SUBSTR(bytes2, 1, 4),
SUBSTR(bytes2, 3),
- SUBSTR(bytes2, -4, 3)
+ SUBSTR(bytes2, -4, 3),
+ HEX(bytes1),
+ UNHEX(HEX(bytes1)),
+ BASE64(bytes1),
+ UNBASE64(BASE64(bytes1)),
+ HEX(ENCODE(string, 'US-ASCII')),
+ DECODE(ENCODE(string, 'US-ASCII'), 'US-ASCII')
FROM dest1
ORDER BY bytes2;
Modified: hive/trunk/ql/src/test/queries/clientpositive/udf_sentences.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/udf_sentences.q?rev=1513474&r1=1513473&r2=1513474&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/udf_sentences.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/udf_sentences.q Tue Aug 13 13:18:36 2013
@@ -1,7 +1,7 @@
CREATE TABLE sent_tmp (val array<string>);
CREATE TABLE sent_tmp2 (val string);
INSERT OVERWRITE TABLE sent_tmp
-SELECT explode(sentences(unhex("486976652065737420756E20657863656C6C656E74206F7574696C20706F7572206C65732072657175C3AA74657320646520646F6E6EC3A965732C20657420706575742DC3AA74726520706C757320706F6C7976616C656E7420717565206C612074726164756374696F6E206175746F6D61746971756521206C6120706F6E6374756174696F6E206D756C7469706C65732C206465732070687261736573206D616C20666F726DC3A96573202E2E2E20636F6E667573696F6E202D20657420706F757274616E742063652055444620666F6E6374696F6E6E6520656E636F72652121"), "fr")) AS val FROM src LIMIT 3;
+SELECT explode(sentences(decode(unhex("486976652065737420756E20657863656C6C656E74206F7574696C20706F7572206C65732072657175C3AA74657320646520646F6E6EC3A965732C20657420706575742DC3AA74726520706C757320706F6C7976616C656E7420717565206C612074726164756374696F6E206175746F6D61746971756521206C6120706F6E6374756174696F6E206D756C7469706C65732C206465732070687261736573206D616C20666F726DC3A96573202E2E2E20636F6E667573696F6E202D20657420706F757274616E742063652055444620666F6E6374696F6E6E6520656E636F72652121"), "UTF-8"), "fr")) AS val FROM src LIMIT 3;
INSERT OVERWRITE TABLE sent_tmp2
SELECT explode(val) AS val FROM sent_tmp;
SELECT hex(val) AS value FROM sent_tmp2 ORDER BY value ASC;
@@ -12,7 +12,7 @@ DROP TABLE sent_tmp2;
CREATE TABLE sent_tmp (val array<string>);
CREATE TABLE sent_tmp2 (val string);
INSERT OVERWRITE TABLE sent_tmp
-SELECT explode(sentences(unhex("48697665206973742065696E2061757367657A656963686E65746573205765726B7A6575672066C3BC7220646965204162667261676520766F6E20446174656E2C20756E64207669656C6C6569636874207669656C736569746967657220616C7320646965206D61736368696E656C6C6520C39C6265727365747A756E6721204D756C7469706C652C207363686C6563687420676562696C646574656E2053C3A4747A65202E2E2E205665727765636873656C756E6720496E74657270756E6B74696F6E202D20756E6420646F636820697374206469657365205544462066756E6B74696F6E6965727420696D6D6572206E6F63682121"), "de")) AS val FROM src LIMIT 3;
+SELECT explode(sentences(decode(unhex("48697665206973742065696E2061757367657A656963686E65746573205765726B7A6575672066C3BC7220646965204162667261676520766F6E20446174656E2C20756E64207669656C6C6569636874207669656C736569746967657220616C7320646965206D61736368696E656C6C6520C39C6265727365747A756E6721204D756C7469706C652C207363686C6563687420676562696C646574656E2053C3A4747A65202E2E2E205665727765636873656C756E6720496E74657270756E6B74696F6E202D20756E6420646F636820697374206469657365205544462066756E6B74696F6E6965727420696D6D6572206E6F63682121"), "UTF-8"), "de")) AS val FROM src LIMIT 3;
INSERT OVERWRITE TABLE sent_tmp2
SELECT explode(val) AS val FROM sent_tmp;
SELECT hex(val) AS value FROM sent_tmp2 ORDER BY value ASC;
Modified: hive/trunk/ql/src/test/results/clientpositive/ba_table_udfs.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ba_table_udfs.q.out?rev=1513474&r1=1513473&r2=1513474&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ba_table_udfs.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/ba_table_udfs.q.out Tue Aug 13 13:18:36 2013
@@ -3,16 +3,19 @@ PREHOOK: type: SWITCHDATABASE
POSTHOOK: query: USE default
POSTHOOK: type: SWITCHDATABASE
PREHOOK: query: CREATE TABLE dest1(bytes1 BINARY,
- bytes2 BINARY)
+ bytes2 BINARY,
+ string STRING)
PREHOOK: type: CREATETABLE
POSTHOOK: query: CREATE TABLE dest1(bytes1 BINARY,
- bytes2 BINARY)
+ bytes2 BINARY,
+ string STRING)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@dest1
PREHOOK: query: FROM src INSERT OVERWRITE TABLE dest1
SELECT
CAST(key AS BINARY),
- CAST(value AS BINARY)
+ CAST(value AS BINARY),
+ value
ORDER BY value
LIMIT 100
PREHOOK: type: QUERY
@@ -21,7 +24,8 @@ PREHOOK: Output: default@dest1
POSTHOOK: query: FROM src INSERT OVERWRITE TABLE dest1
SELECT
CAST(key AS BINARY),
- CAST(value AS BINARY)
+ CAST(value AS BINARY),
+ value
ORDER BY value
LIMIT 100
POSTHOOK: type: QUERY
@@ -29,13 +33,14 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.bytes1 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.bytes2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.string SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: --Add in a null row for good measure
-INSERT INTO TABLE dest1 SELECT NULL, NULL FROM dest1 LIMIT 1
+INSERT INTO TABLE dest1 SELECT NULL, NULL, NULL FROM dest1 LIMIT 1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
PREHOOK: Output: default@dest1
POSTHOOK: query: --Add in a null row for good measure
-INSERT INTO TABLE dest1 SELECT NULL, NULL FROM dest1 LIMIT 1
+INSERT INTO TABLE dest1 SELECT NULL, NULL, NULL FROM dest1 LIMIT 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
POSTHOOK: Output: default@dest1
@@ -43,16 +48,25 @@ POSTHOOK: Lineage: dest1.bytes1 EXPRESSI
POSTHOOK: Lineage: dest1.bytes1 EXPRESSION []
POSTHOOK: Lineage: dest1.bytes2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.bytes2 EXPRESSION []
+POSTHOOK: Lineage: dest1.string SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.string SIMPLE []
PREHOOK: query: -- this query tests all the udfs provided to work with binary types
SELECT
bytes1,
bytes2,
+ string,
LENGTH(bytes1),
CONCAT(bytes1, bytes2),
SUBSTR(bytes2, 1, 4),
SUBSTR(bytes2, 3),
- SUBSTR(bytes2, -4, 3)
+ SUBSTR(bytes2, -4, 3),
+ HEX(bytes1),
+ UNHEX(HEX(bytes1)),
+ BASE64(bytes1),
+ UNBASE64(BASE64(bytes1)),
+ HEX(ENCODE(string, 'US-ASCII')),
+ DECODE(ENCODE(string, 'US-ASCII'), 'US-ASCII')
FROM dest1
ORDER BY bytes2
PREHOOK: type: QUERY
@@ -63,11 +77,18 @@ POSTHOOK: query: -- this query tests all
SELECT
bytes1,
bytes2,
+ string,
LENGTH(bytes1),
CONCAT(bytes1, bytes2),
SUBSTR(bytes2, 1, 4),
SUBSTR(bytes2, 3),
- SUBSTR(bytes2, -4, 3)
+ SUBSTR(bytes2, -4, 3),
+ HEX(bytes1),
+ UNHEX(HEX(bytes1)),
+ BASE64(bytes1),
+ UNBASE64(BASE64(bytes1)),
+ HEX(ENCODE(string, 'US-ASCII')),
+ DECODE(ENCODE(string, 'US-ASCII'), 'US-ASCII')
FROM dest1
ORDER BY bytes2
POSTHOOK: type: QUERY
@@ -77,104 +98,106 @@ POSTHOOK: Lineage: dest1.bytes1 EXPRESSI
POSTHOOK: Lineage: dest1.bytes1 EXPRESSION []
POSTHOOK: Lineage: dest1.bytes2 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.bytes2 EXPRESSION []
-NULL NULL NULL NULL NULL NULL NULL
-0 val_0 1 0val_0 val_ l_0 al_
-0 val_0 1 0val_0 val_ l_0 al_
-0 val_0 1 0val_0 val_ l_0 al_
-10 val_10 2 10val_10 val_ l_10 l_1
-100 val_100 3 100val_100 val_ l_100 _10
-100 val_100 3 100val_100 val_ l_100 _10
-103 val_103 3 103val_103 val_ l_103 _10
-103 val_103 3 103val_103 val_ l_103 _10
-104 val_104 3 104val_104 val_ l_104 _10
-104 val_104 3 104val_104 val_ l_104 _10
-105 val_105 3 105val_105 val_ l_105 _10
-11 val_11 2 11val_11 val_ l_11 l_1
-111 val_111 3 111val_111 val_ l_111 _11
-113 val_113 3 113val_113 val_ l_113 _11
-113 val_113 3 113val_113 val_ l_113 _11
-114 val_114 3 114val_114 val_ l_114 _11
-116 val_116 3 116val_116 val_ l_116 _11
-118 val_118 3 118val_118 val_ l_118 _11
-118 val_118 3 118val_118 val_ l_118 _11
-119 val_119 3 119val_119 val_ l_119 _11
-119 val_119 3 119val_119 val_ l_119 _11
-119 val_119 3 119val_119 val_ l_119 _11
-12 val_12 2 12val_12 val_ l_12 l_1
-12 val_12 2 12val_12 val_ l_12 l_1
-120 val_120 3 120val_120 val_ l_120 _12
-120 val_120 3 120val_120 val_ l_120 _12
-125 val_125 3 125val_125 val_ l_125 _12
-125 val_125 3 125val_125 val_ l_125 _12
-126 val_126 3 126val_126 val_ l_126 _12
-128 val_128 3 128val_128 val_ l_128 _12
-128 val_128 3 128val_128 val_ l_128 _12
-128 val_128 3 128val_128 val_ l_128 _12
-129 val_129 3 129val_129 val_ l_129 _12
-129 val_129 3 129val_129 val_ l_129 _12
-131 val_131 3 131val_131 val_ l_131 _13
-133 val_133 3 133val_133 val_ l_133 _13
-134 val_134 3 134val_134 val_ l_134 _13
-134 val_134 3 134val_134 val_ l_134 _13
-136 val_136 3 136val_136 val_ l_136 _13
-137 val_137 3 137val_137 val_ l_137 _13
-137 val_137 3 137val_137 val_ l_137 _13
-138 val_138 3 138val_138 val_ l_138 _13
-138 val_138 3 138val_138 val_ l_138 _13
-138 val_138 3 138val_138 val_ l_138 _13
-138 val_138 3 138val_138 val_ l_138 _13
-143 val_143 3 143val_143 val_ l_143 _14
-145 val_145 3 145val_145 val_ l_145 _14
-146 val_146 3 146val_146 val_ l_146 _14
-146 val_146 3 146val_146 val_ l_146 _14
-149 val_149 3 149val_149 val_ l_149 _14
-149 val_149 3 149val_149 val_ l_149 _14
-15 val_15 2 15val_15 val_ l_15 l_1
-15 val_15 2 15val_15 val_ l_15 l_1
-150 val_150 3 150val_150 val_ l_150 _15
-152 val_152 3 152val_152 val_ l_152 _15
-152 val_152 3 152val_152 val_ l_152 _15
-153 val_153 3 153val_153 val_ l_153 _15
-155 val_155 3 155val_155 val_ l_155 _15
-156 val_156 3 156val_156 val_ l_156 _15
-157 val_157 3 157val_157 val_ l_157 _15
-158 val_158 3 158val_158 val_ l_158 _15
-160 val_160 3 160val_160 val_ l_160 _16
-162 val_162 3 162val_162 val_ l_162 _16
-163 val_163 3 163val_163 val_ l_163 _16
-164 val_164 3 164val_164 val_ l_164 _16
-164 val_164 3 164val_164 val_ l_164 _16
-165 val_165 3 165val_165 val_ l_165 _16
-165 val_165 3 165val_165 val_ l_165 _16
-166 val_166 3 166val_166 val_ l_166 _16
-167 val_167 3 167val_167 val_ l_167 _16
-167 val_167 3 167val_167 val_ l_167 _16
-167 val_167 3 167val_167 val_ l_167 _16
-168 val_168 3 168val_168 val_ l_168 _16
-169 val_169 3 169val_169 val_ l_169 _16
-169 val_169 3 169val_169 val_ l_169 _16
-169 val_169 3 169val_169 val_ l_169 _16
-169 val_169 3 169val_169 val_ l_169 _16
-17 val_17 2 17val_17 val_ l_17 l_1
-170 val_170 3 170val_170 val_ l_170 _17
-172 val_172 3 172val_172 val_ l_172 _17
-172 val_172 3 172val_172 val_ l_172 _17
-174 val_174 3 174val_174 val_ l_174 _17
-174 val_174 3 174val_174 val_ l_174 _17
-175 val_175 3 175val_175 val_ l_175 _17
-175 val_175 3 175val_175 val_ l_175 _17
-176 val_176 3 176val_176 val_ l_176 _17
-176 val_176 3 176val_176 val_ l_176 _17
-177 val_177 3 177val_177 val_ l_177 _17
-178 val_178 3 178val_178 val_ l_178 _17
-179 val_179 3 179val_179 val_ l_179 _17
-179 val_179 3 179val_179 val_ l_179 _17
-18 val_18 2 18val_18 val_ l_18 l_1
-18 val_18 2 18val_18 val_ l_18 l_1
-180 val_180 3 180val_180 val_ l_180 _18
-181 val_181 3 181val_181 val_ l_181 _18
-183 val_183 3 183val_183 val_ l_183 _18
-186 val_186 3 186val_186 val_ l_186 _18
-187 val_187 3 187val_187 val_ l_187 _18
-187 val_187 3 187val_187 val_ l_187 _18
-187 val_187 3 187val_187 val_ l_187 _18
+POSTHOOK: Lineage: dest1.string SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.string SIMPLE []
+NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
+0 val_0 val_0 1 0val_0 val_ l_0 al_ 30 0 MA== 0 76616C5F30 val_0
+0 val_0 val_0 1 0val_0 val_ l_0 al_ 30 0 MA== 0 76616C5F30 val_0
+0 val_0 val_0 1 0val_0 val_ l_0 al_ 30 0 MA== 0 76616C5F30 val_0
+10 val_10 val_10 2 10val_10 val_ l_10 l_1 3130 10 MTA= 10 76616C5F3130 val_10
+100 val_100 val_100 3 100val_100 val_ l_100 _10 313030 100 MTAw 100 76616C5F313030 val_100
+100 val_100 val_100 3 100val_100 val_ l_100 _10 313030 100 MTAw 100 76616C5F313030 val_100
+103 val_103 val_103 3 103val_103 val_ l_103 _10 313033 103 MTAz 103 76616C5F313033 val_103
+103 val_103 val_103 3 103val_103 val_ l_103 _10 313033 103 MTAz 103 76616C5F313033 val_103
+104 val_104 val_104 3 104val_104 val_ l_104 _10 313034 104 MTA0 104 76616C5F313034 val_104
+104 val_104 val_104 3 104val_104 val_ l_104 _10 313034 104 MTA0 104 76616C5F313034 val_104
+105 val_105 val_105 3 105val_105 val_ l_105 _10 313035 105 MTA1 105 76616C5F313035 val_105
+11 val_11 val_11 2 11val_11 val_ l_11 l_1 3131 11 MTE= 11 76616C5F3131 val_11
+111 val_111 val_111 3 111val_111 val_ l_111 _11 313131 111 MTEx 111 76616C5F313131 val_111
+113 val_113 val_113 3 113val_113 val_ l_113 _11 313133 113 MTEz 113 76616C5F313133 val_113
+113 val_113 val_113 3 113val_113 val_ l_113 _11 313133 113 MTEz 113 76616C5F313133 val_113
+114 val_114 val_114 3 114val_114 val_ l_114 _11 313134 114 MTE0 114 76616C5F313134 val_114
+116 val_116 val_116 3 116val_116 val_ l_116 _11 313136 116 MTE2 116 76616C5F313136 val_116
+118 val_118 val_118 3 118val_118 val_ l_118 _11 313138 118 MTE4 118 76616C5F313138 val_118
+118 val_118 val_118 3 118val_118 val_ l_118 _11 313138 118 MTE4 118 76616C5F313138 val_118
+119 val_119 val_119 3 119val_119 val_ l_119 _11 313139 119 MTE5 119 76616C5F313139 val_119
+119 val_119 val_119 3 119val_119 val_ l_119 _11 313139 119 MTE5 119 76616C5F313139 val_119
+119 val_119 val_119 3 119val_119 val_ l_119 _11 313139 119 MTE5 119 76616C5F313139 val_119
+12 val_12 val_12 2 12val_12 val_ l_12 l_1 3132 12 MTI= 12 76616C5F3132 val_12
+12 val_12 val_12 2 12val_12 val_ l_12 l_1 3132 12 MTI= 12 76616C5F3132 val_12
+120 val_120 val_120 3 120val_120 val_ l_120 _12 313230 120 MTIw 120 76616C5F313230 val_120
+120 val_120 val_120 3 120val_120 val_ l_120 _12 313230 120 MTIw 120 76616C5F313230 val_120
+125 val_125 val_125 3 125val_125 val_ l_125 _12 313235 125 MTI1 125 76616C5F313235 val_125
+125 val_125 val_125 3 125val_125 val_ l_125 _12 313235 125 MTI1 125 76616C5F313235 val_125
+126 val_126 val_126 3 126val_126 val_ l_126 _12 313236 126 MTI2 126 76616C5F313236 val_126
+128 val_128 val_128 3 128val_128 val_ l_128 _12 313238 128 MTI4 128 76616C5F313238 val_128
+128 val_128 val_128 3 128val_128 val_ l_128 _12 313238 128 MTI4 128 76616C5F313238 val_128
+128 val_128 val_128 3 128val_128 val_ l_128 _12 313238 128 MTI4 128 76616C5F313238 val_128
+129 val_129 val_129 3 129val_129 val_ l_129 _12 313239 129 MTI5 129 76616C5F313239 val_129
+129 val_129 val_129 3 129val_129 val_ l_129 _12 313239 129 MTI5 129 76616C5F313239 val_129
+131 val_131 val_131 3 131val_131 val_ l_131 _13 313331 131 MTMx 131 76616C5F313331 val_131
+133 val_133 val_133 3 133val_133 val_ l_133 _13 313333 133 MTMz 133 76616C5F313333 val_133
+134 val_134 val_134 3 134val_134 val_ l_134 _13 313334 134 MTM0 134 76616C5F313334 val_134
+134 val_134 val_134 3 134val_134 val_ l_134 _13 313334 134 MTM0 134 76616C5F313334 val_134
+136 val_136 val_136 3 136val_136 val_ l_136 _13 313336 136 MTM2 136 76616C5F313336 val_136
+137 val_137 val_137 3 137val_137 val_ l_137 _13 313337 137 MTM3 137 76616C5F313337 val_137
+137 val_137 val_137 3 137val_137 val_ l_137 _13 313337 137 MTM3 137 76616C5F313337 val_137
+138 val_138 val_138 3 138val_138 val_ l_138 _13 313338 138 MTM4 138 76616C5F313338 val_138
+138 val_138 val_138 3 138val_138 val_ l_138 _13 313338 138 MTM4 138 76616C5F313338 val_138
+138 val_138 val_138 3 138val_138 val_ l_138 _13 313338 138 MTM4 138 76616C5F313338 val_138
+138 val_138 val_138 3 138val_138 val_ l_138 _13 313338 138 MTM4 138 76616C5F313338 val_138
+143 val_143 val_143 3 143val_143 val_ l_143 _14 313433 143 MTQz 143 76616C5F313433 val_143
+145 val_145 val_145 3 145val_145 val_ l_145 _14 313435 145 MTQ1 145 76616C5F313435 val_145
+146 val_146 val_146 3 146val_146 val_ l_146 _14 313436 146 MTQ2 146 76616C5F313436 val_146
+146 val_146 val_146 3 146val_146 val_ l_146 _14 313436 146 MTQ2 146 76616C5F313436 val_146
+149 val_149 val_149 3 149val_149 val_ l_149 _14 313439 149 MTQ5 149 76616C5F313439 val_149
+149 val_149 val_149 3 149val_149 val_ l_149 _14 313439 149 MTQ5 149 76616C5F313439 val_149
+15 val_15 val_15 2 15val_15 val_ l_15 l_1 3135 15 MTU= 15 76616C5F3135 val_15
+15 val_15 val_15 2 15val_15 val_ l_15 l_1 3135 15 MTU= 15 76616C5F3135 val_15
+150 val_150 val_150 3 150val_150 val_ l_150 _15 313530 150 MTUw 150 76616C5F313530 val_150
+152 val_152 val_152 3 152val_152 val_ l_152 _15 313532 152 MTUy 152 76616C5F313532 val_152
+152 val_152 val_152 3 152val_152 val_ l_152 _15 313532 152 MTUy 152 76616C5F313532 val_152
+153 val_153 val_153 3 153val_153 val_ l_153 _15 313533 153 MTUz 153 76616C5F313533 val_153
+155 val_155 val_155 3 155val_155 val_ l_155 _15 313535 155 MTU1 155 76616C5F313535 val_155
+156 val_156 val_156 3 156val_156 val_ l_156 _15 313536 156 MTU2 156 76616C5F313536 val_156
+157 val_157 val_157 3 157val_157 val_ l_157 _15 313537 157 MTU3 157 76616C5F313537 val_157
+158 val_158 val_158 3 158val_158 val_ l_158 _15 313538 158 MTU4 158 76616C5F313538 val_158
+160 val_160 val_160 3 160val_160 val_ l_160 _16 313630 160 MTYw 160 76616C5F313630 val_160
+162 val_162 val_162 3 162val_162 val_ l_162 _16 313632 162 MTYy 162 76616C5F313632 val_162
+163 val_163 val_163 3 163val_163 val_ l_163 _16 313633 163 MTYz 163 76616C5F313633 val_163
+164 val_164 val_164 3 164val_164 val_ l_164 _16 313634 164 MTY0 164 76616C5F313634 val_164
+164 val_164 val_164 3 164val_164 val_ l_164 _16 313634 164 MTY0 164 76616C5F313634 val_164
+165 val_165 val_165 3 165val_165 val_ l_165 _16 313635 165 MTY1 165 76616C5F313635 val_165
+165 val_165 val_165 3 165val_165 val_ l_165 _16 313635 165 MTY1 165 76616C5F313635 val_165
+166 val_166 val_166 3 166val_166 val_ l_166 _16 313636 166 MTY2 166 76616C5F313636 val_166
+167 val_167 val_167 3 167val_167 val_ l_167 _16 313637 167 MTY3 167 76616C5F313637 val_167
+167 val_167 val_167 3 167val_167 val_ l_167 _16 313637 167 MTY3 167 76616C5F313637 val_167
+167 val_167 val_167 3 167val_167 val_ l_167 _16 313637 167 MTY3 167 76616C5F313637 val_167
+168 val_168 val_168 3 168val_168 val_ l_168 _16 313638 168 MTY4 168 76616C5F313638 val_168
+169 val_169 val_169 3 169val_169 val_ l_169 _16 313639 169 MTY5 169 76616C5F313639 val_169
+169 val_169 val_169 3 169val_169 val_ l_169 _16 313639 169 MTY5 169 76616C5F313639 val_169
+169 val_169 val_169 3 169val_169 val_ l_169 _16 313639 169 MTY5 169 76616C5F313639 val_169
+169 val_169 val_169 3 169val_169 val_ l_169 _16 313639 169 MTY5 169 76616C5F313639 val_169
+17 val_17 val_17 2 17val_17 val_ l_17 l_1 3137 17 MTc= 17 76616C5F3137 val_17
+170 val_170 val_170 3 170val_170 val_ l_170 _17 313730 170 MTcw 170 76616C5F313730 val_170
+172 val_172 val_172 3 172val_172 val_ l_172 _17 313732 172 MTcy 172 76616C5F313732 val_172
+172 val_172 val_172 3 172val_172 val_ l_172 _17 313732 172 MTcy 172 76616C5F313732 val_172
+174 val_174 val_174 3 174val_174 val_ l_174 _17 313734 174 MTc0 174 76616C5F313734 val_174
+174 val_174 val_174 3 174val_174 val_ l_174 _17 313734 174 MTc0 174 76616C5F313734 val_174
+175 val_175 val_175 3 175val_175 val_ l_175 _17 313735 175 MTc1 175 76616C5F313735 val_175
+175 val_175 val_175 3 175val_175 val_ l_175 _17 313735 175 MTc1 175 76616C5F313735 val_175
+176 val_176 val_176 3 176val_176 val_ l_176 _17 313736 176 MTc2 176 76616C5F313736 val_176
+176 val_176 val_176 3 176val_176 val_ l_176 _17 313736 176 MTc2 176 76616C5F313736 val_176
+177 val_177 val_177 3 177val_177 val_ l_177 _17 313737 177 MTc3 177 76616C5F313737 val_177
+178 val_178 val_178 3 178val_178 val_ l_178 _17 313738 178 MTc4 178 76616C5F313738 val_178
+179 val_179 val_179 3 179val_179 val_ l_179 _17 313739 179 MTc5 179 76616C5F313739 val_179
+179 val_179 val_179 3 179val_179 val_ l_179 _17 313739 179 MTc5 179 76616C5F313739 val_179
+18 val_18 val_18 2 18val_18 val_ l_18 l_1 3138 18 MTg= 18 76616C5F3138 val_18
+18 val_18 val_18 2 18val_18 val_ l_18 l_1 3138 18 MTg= 18 76616C5F3138 val_18
+180 val_180 val_180 3 180val_180 val_ l_180 _18 313830 180 MTgw 180 76616C5F313830 val_180
+181 val_181 val_181 3 181val_181 val_ l_181 _18 313831 181 MTgx 181 76616C5F313831 val_181
+183 val_183 val_183 3 183val_183 val_ l_183 _18 313833 183 MTgz 183 76616C5F313833 val_183
+186 val_186 val_186 3 186val_186 val_ l_186 _18 313836 186 MTg2 186 76616C5F313836 val_186
+187 val_187 val_187 3 187val_187 val_ l_187 _18 313837 187 MTg3 187 76616C5F313837 val_187
+187 val_187 val_187 3 187val_187 val_ l_187 _18 313837 187 MTg3 187 76616C5F313837 val_187
+187 val_187 val_187 3 187val_187 val_ l_187 _18 313837 187 MTg3 187 76616C5F313837 val_187
Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1513474&r1=1513473&r2=1513474&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Tue Aug 13 13:18:36 2013
@@ -29,6 +29,7 @@ asin
assert_true
atan
avg
+base64
between
bin
case
@@ -53,11 +54,13 @@ date_sub
datediff
day
dayofmonth
+decode
degrees
dense_rank
div
e
elt
+encode
ewah_bitmap
ewah_bitmap_and
ewah_bitmap_empty
@@ -169,6 +172,7 @@ to_utc_timestamp
translate
trim
ucase
+unbase64
unhex
unix_timestamp
upper
@@ -218,7 +222,9 @@ POSTHOOK: type: SHOWFUNCTIONS
assert_true
case
coalesce
+decode
e
+encode
explode
first_value
from_unixtime
Modified: hive/trunk/ql/src/test/results/clientpositive/udf_hex.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udf_hex.q.out?rev=1513474&r1=1513473&r2=1513474&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udf_hex.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udf_hex.q.out Tue Aug 13 13:18:36 2013
@@ -2,14 +2,14 @@ PREHOOK: query: DESCRIBE FUNCTION hex
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION hex
POSTHOOK: type: DESCFUNCTION
-hex(n or str) - Convert the argument to hexadecimal
+hex(n, bin, or str) - Convert the argument to hexadecimal
PREHOOK: query: DESCRIBE FUNCTION EXTENDED hex
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION EXTENDED hex
POSTHOOK: type: DESCFUNCTION
-hex(n or str) - Convert the argument to hexadecimal
+hex(n, bin, or str) - Convert the argument to hexadecimal
If the argument is a string, returns two hex digits for each character in the string.
-If the argument is a number, returns the hexadecimal representation.
+If the argument is a number or binary, returns the hexadecimal representation.
Example:
> SELECT hex(17) FROM src LIMIT 1;
'H1'
Modified: hive/trunk/ql/src/test/results/clientpositive/udf_sentences.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udf_sentences.q.out?rev=1513474&r1=1513473&r2=1513474&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udf_sentences.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udf_sentences.q.out Tue Aug 13 13:18:36 2013
@@ -9,12 +9,12 @@ POSTHOOK: query: CREATE TABLE sent_tmp2
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@sent_tmp2
PREHOOK: query: INSERT OVERWRITE TABLE sent_tmp
-SELECT explode(sentences(unhex("486976652065737420756E20657863656C6C656E74206F7574696C20706F7572206C65732072657175C3AA74657320646520646F6E6EC3A965732C20657420706575742DC3AA74726520706C757320706F6C7976616C656E7420717565206C612074726164756374696F6E206175746F6D61746971756521206C6120706F6E6374756174696F6E206D756C7469706C65732C206465732070687261736573206D616C20666F726DC3A96573202E2E2E20636F6E667573696F6E202D20657420706F757274616E742063652055444620666F6E6374696F6E6E6520656E636F72652121"), "fr")) AS val FROM src LIMIT 3
+SELECT explode(sentences(decode(unhex("486976652065737420756E20657863656C6C656E74206F7574696C20706F7572206C65732072657175C3AA74657320646520646F6E6EC3A965732C20657420706575742DC3AA74726520706C757320706F6C7976616C656E7420717565206C612074726164756374696F6E206175746F6D61746971756521206C6120706F6E6374756174696F6E206D756C7469706C65732C206465732070687261736573206D616C20666F726DC3A96573202E2E2E20636F6E667573696F6E202D20657420706F757274616E742063652055444620666F6E6374696F6E6E6520656E636F72652121"), "UTF-8"), "fr")) AS val FROM src LIMIT 3
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@sent_tmp
POSTHOOK: query: INSERT OVERWRITE TABLE sent_tmp
-SELECT explode(sentences(unhex("486976652065737420756E20657863656C6C656E74206F7574696C20706F7572206C65732072657175C3AA74657320646520646F6E6EC3A965732C20657420706575742DC3AA74726520706C757320706F6C7976616C656E7420717565206C612074726164756374696F6E206175746F6D61746971756521206C6120706F6E6374756174696F6E206D756C7469706C65732C206465732070687261736573206D616C20666F726DC3A96573202E2E2E20636F6E667573696F6E202D20657420706F757274616E742063652055444620666F6E6374696F6E6E6520656E636F72652121"), "fr")) AS val FROM src LIMIT 3
+SELECT explode(sentences(decode(unhex("486976652065737420756E20657863656C6C656E74206F7574696C20706F7572206C65732072657175C3AA74657320646520646F6E6EC3A965732C20657420706575742DC3AA74726520706C757320706F6C7976616C656E7420717565206C612074726164756374696F6E206175746F6D61746971756521206C6120706F6E6374756174696F6E206D756C7469706C65732C206465732070687261736573206D616C20666F726DC3A96573202E2E2E20636F6E667573696F6E202D20657420706F757274616E742063652055444620666F6E6374696F6E6E6520656E636F72652121"), "UTF-8"), "fr")) AS val FROM src LIMIT 3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@sent_tmp
@@ -126,12 +126,12 @@ POSTHOOK: Output: default@sent_tmp2
POSTHOOK: Lineage: sent_tmp.val SCRIPT []
POSTHOOK: Lineage: sent_tmp2.val SCRIPT [(sent_tmp)sent_tmp.FieldSchema(name:val, type:array<string>, comment:null), ]
PREHOOK: query: INSERT OVERWRITE TABLE sent_tmp
-SELECT explode(sentences(unhex("48697665206973742065696E2061757367657A656963686E65746573205765726B7A6575672066C3BC7220646965204162667261676520766F6E20446174656E2C20756E64207669656C6C6569636874207669656C736569746967657220616C7320646965206D61736368696E656C6C6520C39C6265727365747A756E6721204D756C7469706C652C207363686C6563687420676562696C646574656E2053C3A4747A65202E2E2E205665727765636873656C756E6720496E74657270756E6B74696F6E202D20756E6420646F636820697374206469657365205544462066756E6B74696F6E6965727420696D6D6572206E6F63682121"), "de")) AS val FROM src LIMIT 3
+SELECT explode(sentences(decode(unhex("48697665206973742065696E2061757367657A656963686E65746573205765726B7A6575672066C3BC7220646965204162667261676520766F6E20446174656E2C20756E64207669656C6C6569636874207669656C736569746967657220616C7320646965206D61736368696E656C6C6520C39C6265727365747A756E6721204D756C7469706C652C207363686C6563687420676562696C646574656E2053C3A4747A65202E2E2E205665727765636873656C756E6720496E74657270756E6B74696F6E202D20756E6420646F636820697374206469657365205544462066756E6B74696F6E6965727420696D6D6572206E6F63682121"), "UTF-8"), "de")) AS val FROM src LIMIT 3
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@sent_tmp
POSTHOOK: query: INSERT OVERWRITE TABLE sent_tmp
-SELECT explode(sentences(unhex("48697665206973742065696E2061757367657A656963686E65746573205765726B7A6575672066C3BC7220646965204162667261676520766F6E20446174656E2C20756E64207669656C6C6569636874207669656C736569746967657220616C7320646965206D61736368696E656C6C6520C39C6265727365747A756E6721204D756C7469706C652C207363686C6563687420676562696C646574656E2053C3A4747A65202E2E2E205665727765636873656C756E6720496E74657270756E6B74696F6E202D20756E6420646F636820697374206469657365205544462066756E6B74696F6E6965727420696D6D6572206E6F63682121"), "de")) AS val FROM src LIMIT 3
+SELECT explode(sentences(decode(unhex("48697665206973742065696E2061757367657A656963686E65746573205765726B7A6575672066C3BC7220646965204162667261676520766F6E20446174656E2C20756E64207669656C6C6569636874207669656C736569746967657220616C7320646965206D61736368696E656C6C6520C39C6265727365747A756E6721204D756C7469706C652C207363686C6563687420676562696C646574656E2053C3A4747A65202E2E2E205665727765636873656C756E6720496E74657270756E6B74696F6E202D20756E6420646F636820697374206469657365205544462066756E6B74696F6E6965727420696D6D6572206E6F63682121"), "UTF-8"), "de")) AS val FROM src LIMIT 3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@sent_tmp
Modified: hive/trunk/ql/src/test/results/clientpositive/udf_unhex.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udf_unhex.q.out?rev=1513474&r1=1513473&r2=1513474&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udf_unhex.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udf_unhex.q.out Tue Aug 13 13:18:36 2013
@@ -2,24 +2,20 @@ PREHOOK: query: DESCRIBE FUNCTION unhex
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION unhex
POSTHOOK: type: DESCFUNCTION
-unhex(str) - Converts hexadecimal argument to string
+unhex(str) - Converts hexadecimal argument to binary
PREHOOK: query: DESCRIBE FUNCTION EXTENDED unhex
PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESCRIBE FUNCTION EXTENDED unhex
POSTHOOK: type: DESCFUNCTION
-unhex(str) - Converts hexadecimal argument to string
+unhex(str) - Converts hexadecimal argument to binary
Performs the inverse operation of HEX(str). That is, it interprets
each pair of hexadecimal digits in the argument as a number and
-converts it to the character represented by the number. The
+converts it to the byte representation of the number. The
resulting characters are returned as a binary string.
Example:
-> SELECT UNHEX('4D7953514C') from src limit 1;
+> SELECT DECODE(UNHEX('4D7953514C'), 'UTF-8') from src limit 1;
'MySQL'
-> SELECT UNHEX(HEX('string')) from src limit 1;
-'string'
-> SELECT HEX(UNHEX('1267')) from src limit 1;
-'1267'
The characters in the argument string must be legal hexadecimal
digits: '0' .. '9', 'A' .. 'F', 'a' .. 'f'. If UNHEX() encounters