You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2018/09/07 14:41:35 UTC
[flink] branch master updated: [FLINK-10174] [table] Define UTF-8
charset for HEX, TO_BASE64, FROM_BASE64
This is an automated email from the ASF dual-hosted git repository.
twalthr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 99bf3a5 [FLINK-10174] [table] Define UTF-8 charset for HEX, TO_BASE64, FROM_BASE64
99bf3a5 is described below
commit 99bf3a5087ac80ff20c8d22adef82f30e53ba568
Author: xueyu <27...@qq.com>
AuthorDate: Mon Aug 20 15:16:42 2018 +0800
[FLINK-10174] [table] Define UTF-8 charset for HEX, TO_BASE64, FROM_BASE64
This closes #6579.
---
.../table/runtime/functions/ScalarFunctions.scala | 10 +++++++---
.../table/expressions/ScalarFunctionsTest.scala | 21 +++++++++++++++++++++
2 files changed, 28 insertions(+), 3 deletions(-)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/functions/ScalarFunctions.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/functions/ScalarFunctions.scala
index 1aadf31..fa6f020 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/functions/ScalarFunctions.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/functions/ScalarFunctions.scala
@@ -19,6 +19,7 @@ package org.apache.flink.table.runtime.functions
import java.lang.{StringBuilder, Long => JLong}
import java.math.{BigDecimal => JBigDecimal}
+import java.nio.charset.StandardCharsets
import org.apache.commons.codec.binary.{Base64, Hex}
import org.apache.commons.lang3.StringUtils
@@ -207,12 +208,14 @@ object ScalarFunctions {
/**
* Returns the base string decoded with base64.
*/
- def fromBase64(str: String): String = new String(Base64.decodeBase64(str))
+ def fromBase64(str: String): String =
+ new String(Base64.decodeBase64(str), StandardCharsets.UTF_8)
/**
* Returns the base64-encoded result of the input string.
*/
- def toBase64(base: String): String = Base64.encodeBase64String(base.getBytes())
+ def toBase64(base: String): String =
+ Base64.encodeBase64String(base.getBytes(StandardCharsets.UTF_8))
/**
* Returns the hex string of a long argument.
@@ -222,7 +225,8 @@ object ScalarFunctions {
/**
* Returns the hex string of a string argument.
*/
- def hex(x: String): String = Hex.encodeHexString(x.getBytes).toUpperCase()
+ def hex(x: String): String =
+ Hex.encodeHexString(x.getBytes(StandardCharsets.UTF_8)).toUpperCase()
/**
* Returns an UUID string using Java utilities.
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
index 5038254..fbd9b02 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/expressions/ScalarFunctionsTest.scala
@@ -483,6 +483,13 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
"f24.hex()",
"HEX(f24)",
"2A5F546869732069732061207465737420537472696E672E")
+
+ testAllApis(
+ "你好".hex(),
+ "'你好'.hex()",
+ "HEX('你好')",
+ "E4BDA0E5A5BD"
+ )
}
@Test
@@ -563,6 +570,13 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
"f33.fromBase64()",
"FROM_BASE64(f33)",
"null")
+
+ testAllApis(
+ "5L2g5aW9".fromBase64(),
+ "'5L2g5aW9'.fromBase64()",
+ "FROM_BASE64('5L2g5aW9')",
+ "你好"
+ )
}
@Test
@@ -591,6 +605,13 @@ class ScalarFunctionsTest extends ScalarTypesTestBase {
"f33.toBase64()",
"TO_BASE64(f33)",
"null")
+
+ testAllApis(
+ "你好".toBase64(),
+ "'你好'.toBase64()",
+ "TO_BASE64('你好')",
+ "5L2g5aW9"
+ )
}
@Test