You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by ta...@apache.org on 2023/06/26 14:55:12 UTC
[calcite] branch main updated (e969f87aea -> 9ebeea96e3)
This is an automated email from the ASF dual-hosted git repository.
tanner pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
from e969f87aea [CALCITE-5778] Add ARRAY_JOIN, ARRAYS_OVERLAP, ARRAYS_ZIP function (enabled in Spark library)
new b40ce21874 [CALCITE-5709] Add TO_BASE32 and FROM_BASE32 functions (enabled in BigQuery library)
new a60d403486 fix typo
new 0c77afd6af add ReturnTypes.VARCHAR_NULLABLE
new 9ebeea96e3 adjust operand type checker
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../calcite/adapter/enumerable/RexImpTable.java | 4 +++
.../org/apache/calcite/runtime/SqlFunctions.java | 22 +++++++++++++
.../calcite/sql/fun/SqlLibraryOperators.java | 19 +++++++++---
.../org/apache/calcite/sql/type/ReturnTypes.java | 13 ++++++++
.../org/apache/calcite/util/BuiltInMethod.java | 2 ++
site/_docs/reference.md | 2 ++
.../org/apache/calcite/test/SqlOperatorTest.java | 36 ++++++++++++++++++++++
7 files changed, 94 insertions(+), 4 deletions(-)
[calcite] 01/04: [CALCITE-5709] Add TO_BASE32 and FROM_BASE32 functions (enabled in BigQuery library)
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tanner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit b40ce21874c0e632343ea21b9766e5193f15047f
Author: zoudan <zo...@bytedance.com>
AuthorDate: Tue May 23 22:49:59 2023 +0800
[CALCITE-5709] Add TO_BASE32 and FROM_BASE32 functions (enabled in BigQuery library)
---
.../calcite/adapter/enumerable/RexImpTable.java | 4 +++
.../org/apache/calcite/runtime/SqlFunctions.java | 22 +++++++++++++
.../calcite/sql/fun/SqlLibraryOperators.java | 15 +++++++++
.../org/apache/calcite/util/BuiltInMethod.java | 2 ++
site/_docs/reference.md | 2 ++
.../org/apache/calcite/test/SqlOperatorTest.java | 36 ++++++++++++++++++++++
6 files changed, 81 insertions(+)
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java
index 06603af460..6a90116e42 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java
@@ -174,6 +174,7 @@ import static org.apache.calcite.sql.fun.SqlLibraryOperators.FORMAT_DATE;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.FORMAT_DATETIME;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.FORMAT_TIME;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.FORMAT_TIMESTAMP;
+import static org.apache.calcite.sql.fun.SqlLibraryOperators.FROM_BASE32;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.FROM_BASE64;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.ILIKE;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_DEPTH;
@@ -236,6 +237,7 @@ import static org.apache.calcite.sql.fun.SqlLibraryOperators.TIMESTAMP_MILLIS;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.TIMESTAMP_SECONDS;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.TIMESTAMP_TRUNC;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.TIME_TRUNC;
+import static org.apache.calcite.sql.fun.SqlLibraryOperators.TO_BASE32;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.TO_BASE64;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.TO_CHAR;
import static org.apache.calcite.sql.fun.SqlLibraryOperators.TRANSLATE3;
@@ -485,6 +487,8 @@ public class RexImpTable {
defineMethod(INITCAP, BuiltInMethod.INITCAP.method, NullPolicy.STRICT);
defineMethod(TO_BASE64, BuiltInMethod.TO_BASE64.method, NullPolicy.STRICT);
defineMethod(FROM_BASE64, BuiltInMethod.FROM_BASE64.method, NullPolicy.STRICT);
+ defineMethod(TO_BASE32, BuiltInMethod.TO_BASE32.method, NullPolicy.STRICT);
+ defineMethod(FROM_BASE32, BuiltInMethod.FROM_BASE32.method, NullPolicy.STRICT);
defineMethod(MD5, BuiltInMethod.MD5.method, NullPolicy.STRICT);
defineMethod(SHA1, BuiltInMethod.SHA1.method, NullPolicy.STRICT);
defineMethod(SHA256, BuiltInMethod.SHA256.method, NullPolicy.STRICT);
diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
index 486153d423..4f2e0177e1 100644
--- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
+++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java
@@ -45,6 +45,7 @@ import org.apache.calcite.util.Util;
import org.apache.calcite.util.format.FormatElement;
import org.apache.calcite.util.format.FormatModels;
+import org.apache.commons.codec.binary.Base32;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.codec.language.Soundex;
@@ -142,6 +143,8 @@ public class SqlFunctions {
private static final Pattern FROM_BASE64_REGEXP = Pattern.compile("[\\t\\n\\r\\s]");
+ private static final Base32 BASE_32 = new Base32();
+
private static final Function1<List<Object>, Enumerable<Object>> LIST_AS_ENUMERABLE =
a0 -> a0 == null ? Linq4j.emptyEnumerable() : Linq4j.asEnumerable(a0);
@@ -259,6 +262,25 @@ public class SqlFunctions {
}
}
+ /** SQL TO_BASE32(string) function. */
+ public static String toBase32(String string) {
+ return toBase32_(string.getBytes(UTF_8));
+ }
+
+ /** SQL TO_BASE32(string) function for binary string. */
+ public static String toBase32(ByteString string) {
+ return toBase32_(string.getBytes());
+ }
+
+ private static String toBase32_(byte[] bytes) {
+ return BASE_32.encodeToString(bytes);
+ }
+
+ /** SQL FROM_BASE32(string) function. */
+ public static ByteString fromBase32(String base32) {
+ return new ByteString(BASE_32.decode(base32));
+ }
+
/** SQL MD5(string) function. */
public static String md5(String string) {
return DigestUtils.md5Hex(string.getBytes(UTF_8));
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
index 07b150105a..1c81280150 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
@@ -1236,6 +1236,21 @@ public abstract class SqlLibraryOperators {
OperandTypes.STRING.or(OperandTypes.BINARY),
SqlFunctionCategory.STRING);
+ @LibraryOperator(libraries = {BIG_QUERY})
+ public static final SqlFunction FROM_BASE32 =
+ SqlBasicFunction.create("FROM_BASE32",
+ ReturnTypes.explicit(SqlTypeName.VARBINARY)
+ .andThen(SqlTypeTransforms.TO_NULLABLE),
+ OperandTypes.STRING, SqlFunctionCategory.STRING);
+
+ @LibraryOperator(libraries = {BIG_QUERY})
+ public static final SqlFunction TO_BASE32 =
+ SqlBasicFunction.create("TO_BASE32",
+ ReturnTypes.explicit(SqlTypeName.VARCHAR)
+ .andThen(SqlTypeTransforms.TO_NULLABLE),
+ OperandTypes.STRING.or(OperandTypes.BINARY),
+ SqlFunctionCategory.STRING);
+
/** The "TO_CHAR(timestamp, format)" function;
* converts {@code timestamp} to string according to the given {@code format}.
*/
diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
index 969cc0daf8..ca987cc6e5 100644
--- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
+++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
@@ -364,6 +364,8 @@ public enum BuiltInMethod {
RIGHT(SqlFunctions.class, "right", String.class, int.class),
TO_BASE64(SqlFunctions.class, "toBase64", String.class),
FROM_BASE64(SqlFunctions.class, "fromBase64", String.class),
+ TO_BASE32(SqlFunctions.class, "toBase32", String.class),
+ FROM_BASE32(SqlFunctions.class, "fromBase32", String.class),
MD5(SqlFunctions.class, "md5", String.class),
SHA1(SqlFunctions.class, "sha1", String.class),
SHA256(SqlFunctions.class, "sha256", String.class),
diff --git a/site/_docs/reference.md b/site/_docs/reference.md
index bd2e612aa3..64e50c3477 100644
--- a/site/_docs/reference.md
+++ b/site/_docs/reference.md
@@ -2748,6 +2748,8 @@ BigQuery's type system uses confusingly different names for types and functions:
| b | LENGTH(string) | Equivalent to `CHAR_LENGTH(string)`
| b | LOG(numeric1 [, numeric2 ]) | Returns the logarithm of *numeric1* to base *numeric2*, or base e if *numeric2* is not present
| b o | LPAD(string, length [, pattern ]) | Returns a string or bytes value that consists of *string* prepended to *length* with *pattern*
+| b | TO_BASE32(string) | Converts the *string* to base-32 encoded form and returns a encoded string
+| b | FROM_BASE32(string) | Returns the decoded result of a base-32 *string* as a string
| m | TO_BASE64(string) | Converts the *string* to base-64 encoded form and returns a encoded string
| b m | FROM_BASE64(string) | Returns the decoded result of a base-64 *string* as a string
| b o | LTRIM(string) | Returns *string* with all blanks removed from the start
diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
index 424f626f99..e57c2da9dd 100644
--- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
+++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java
@@ -4033,6 +4033,42 @@ public class SqlOperatorTest {
f0.forEachLibrary(list(SqlLibrary.BIG_QUERY, SqlLibrary.MYSQL), consumer);
}
+ @Test void testToBase32() {
+ final SqlOperatorFixture f0 = fixture().setFor(SqlLibraryOperators.TO_BASE32);
+ f0.checkFails("^to_base32('')^",
+ "No match found for function signature TO_BASE32\\(<CHARACTER>\\)",
+ false);
+ final SqlOperatorFixture f = f0.withLibrary(SqlLibrary.BIG_QUERY);
+ f.checkString("to_base32(x'436f6e766572747320612073657175656e6365206f6620425954"
+ + "455320696e746f2061206261736533322d656e636f64656420535452494e472e')",
+ "INXW45TFOJ2HGIDBEBZWK4LVMVXGGZJAN5TCAQSZKRCVGIDJNZ2G6IDBEBRGC43FGMZC2ZLOMNXWIZ"
+ + "LEEBJVIUSJJZDS4===",
+ "VARCHAR NOT NULL");
+ f.checkString("to_base32('Converts a sequence of BYTES into a base32-encoded STRING.')",
+ "INXW45TFOJ2HGIDBEBZWK4LVMVXGGZJAN5TCAQSZKRCVGIDJNZ2G6IDBEBRGC43FGMZC2ZLOMNXWIZ"
+ + "LEEBJVIUSJJZDS4===",
+ "VARCHAR NOT NULL");
+ f.checkNull("to_base32(cast (null as varchar))");
+ f.checkString("to_base32(x'')", "", "VARCHAR NOT NULL");
+ f.checkString("to_base32('')", "", "VARCHAR NOT NULL");
+ }
+
+ @Test void testFromBase32() {
+ final SqlOperatorFixture f0 = fixture().setFor(SqlLibraryOperators.FROM_BASE32);
+ f0.checkFails("^from_base32('')^",
+ "No match found for function signature FROM_BASE32\\(<CHARACTER>\\)",
+ false);
+ final SqlOperatorFixture f = f0.withLibrary(SqlLibrary.BIG_QUERY);
+ f.checkString("from_base32('INXW45TFOJ2HGIDBEBZWK4LVMVXGGZJAN5TCAQSZKRCVGIDJNZ2"
+ + "G6IDBEBRGC43FGMZC2ZLOMNXWIZLEEBJVIUSJJZDS4===')",
+ "436f6e766572747320612073657175656e6365206f6620425954455320696e746f206120626173"
+ + "6533322d656e636f64656420535452494e472e",
+ "VARBINARY NOT NULL");
+
+ f.checkString("from_base32('')", "", "VARBINARY NOT NULL");
+ f.checkNull("from_base32(cast (null as varchar))");
+ }
+
@Test void testMd5() {
final SqlOperatorFixture f0 = fixture().setFor(SqlLibraryOperators.MD5);
f0.checkFails("^md5(x'')^",
[calcite] 02/04: fix typo
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tanner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit a60d403486d6ad599ebc353b23ff4745b5c776ff
Author: zoudan <zo...@bytedance.com>
AuthorDate: Wed Jun 7 10:18:02 2023 +0800
fix typo
---
site/_docs/reference.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/site/_docs/reference.md b/site/_docs/reference.md
index 64e50c3477..852559b7e0 100644
--- a/site/_docs/reference.md
+++ b/site/_docs/reference.md
@@ -2748,7 +2748,7 @@ BigQuery's type system uses confusingly different names for types and functions:
| b | LENGTH(string) | Equivalent to `CHAR_LENGTH(string)`
| b | LOG(numeric1 [, numeric2 ]) | Returns the logarithm of *numeric1* to base *numeric2*, or base e if *numeric2* is not present
| b o | LPAD(string, length [, pattern ]) | Returns a string or bytes value that consists of *string* prepended to *length* with *pattern*
-| b | TO_BASE32(string) | Converts the *string* to base-32 encoded form and returns a encoded string
+| b | TO_BASE32(string) | Converts the *string* to base-32 encoded form and returns an encoded string
| b | FROM_BASE32(string) | Returns the decoded result of a base-32 *string* as a string
| m | TO_BASE64(string) | Converts the *string* to base-64 encoded form and returns a encoded string
| b m | FROM_BASE64(string) | Returns the decoded result of a base-64 *string* as a string
[calcite] 04/04: adjust operand type checker
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tanner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 9ebeea96e3eb0c5f76c3fe06934e8cc544559c0a
Author: zoudan <zo...@bytedance.com>
AuthorDate: Wed Jun 14 10:45:35 2023 +0800
adjust operand type checker
---
.../src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
index 38f3e4110a..6f7485f5e0 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
@@ -1238,13 +1238,13 @@ public abstract class SqlLibraryOperators {
public static final SqlFunction FROM_BASE32 =
SqlBasicFunction.create("FROM_BASE32",
ReturnTypes.VARBINARY_NULLABLE,
- OperandTypes.STRING, SqlFunctionCategory.STRING);
+ OperandTypes.CHARACTER, SqlFunctionCategory.STRING);
@LibraryOperator(libraries = {BIG_QUERY})
public static final SqlFunction TO_BASE32 =
SqlBasicFunction.create("TO_BASE32",
ReturnTypes.VARCHAR_NULLABLE,
- OperandTypes.STRING.or(OperandTypes.BINARY),
+ OperandTypes.STRING,
SqlFunctionCategory.STRING);
/** The "TO_CHAR(timestamp, format)" function;
[calcite] 03/04: add ReturnTypes.VARCHAR_NULLABLE
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tanner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 0c77afd6af698c4c2170a9a19dd7a143a52de4fb
Author: zoudan <zo...@bytedance.com>
AuthorDate: Wed Jun 14 10:28:34 2023 +0800
add ReturnTypes.VARCHAR_NULLABLE
---
.../org/apache/calcite/sql/fun/SqlLibraryOperators.java | 12 ++++--------
.../main/java/org/apache/calcite/sql/type/ReturnTypes.java | 13 +++++++++++++
2 files changed, 17 insertions(+), 8 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
index 1c81280150..38f3e4110a 100644
--- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
+++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java
@@ -469,8 +469,7 @@ public abstract class SqlLibraryOperators {
@LibraryOperator(libraries = {MYSQL})
public static final SqlFunction COMPRESS =
SqlBasicFunction.create("COMPRESS",
- ReturnTypes.explicit(SqlTypeName.VARBINARY)
- .andThen(SqlTypeTransforms.TO_NULLABLE),
+ ReturnTypes.VARBINARY_NULLABLE,
OperandTypes.STRING, SqlFunctionCategory.STRING);
@LibraryOperator(libraries = {MYSQL})
@@ -1225,8 +1224,7 @@ public abstract class SqlLibraryOperators {
@LibraryOperator(libraries = {BIG_QUERY, MYSQL})
public static final SqlFunction FROM_BASE64 =
SqlBasicFunction.create("FROM_BASE64",
- ReturnTypes.explicit(SqlTypeName.VARBINARY)
- .andThen(SqlTypeTransforms.TO_NULLABLE),
+ ReturnTypes.VARBINARY_NULLABLE,
OperandTypes.STRING, SqlFunctionCategory.STRING);
@LibraryOperator(libraries = {MYSQL})
@@ -1239,15 +1237,13 @@ public abstract class SqlLibraryOperators {
@LibraryOperator(libraries = {BIG_QUERY})
public static final SqlFunction FROM_BASE32 =
SqlBasicFunction.create("FROM_BASE32",
- ReturnTypes.explicit(SqlTypeName.VARBINARY)
- .andThen(SqlTypeTransforms.TO_NULLABLE),
+ ReturnTypes.VARBINARY_NULLABLE,
OperandTypes.STRING, SqlFunctionCategory.STRING);
@LibraryOperator(libraries = {BIG_QUERY})
public static final SqlFunction TO_BASE32 =
SqlBasicFunction.create("TO_BASE32",
- ReturnTypes.explicit(SqlTypeName.VARCHAR)
- .andThen(SqlTypeTransforms.TO_NULLABLE),
+ ReturnTypes.VARCHAR_NULLABLE,
OperandTypes.STRING.or(OperandTypes.BINARY),
SqlFunctionCategory.STRING);
diff --git a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java
index 5b9d7aa73c..e10107f28b 100644
--- a/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java
+++ b/core/src/main/java/org/apache/calcite/sql/type/ReturnTypes.java
@@ -481,6 +481,19 @@ public abstract class ReturnTypes {
public static final SqlReturnTypeInference VARCHAR_NULLABLE =
VARCHAR.andThen(SqlTypeTransforms.TO_NULLABLE);
+ /**
+ * Type-inference strategy that always returns "VARBINARY".
+ */
+ public static final SqlReturnTypeInference VARBINARY =
+ ReturnTypes.explicit(SqlTypeName.VARBINARY);
+
+ /**
+ * Type-inference strategy that always returns "VARBINARY" with nulls
+ * allowed if any of the operands allow nulls.
+ */
+ public static final SqlReturnTypeInference VARBINARY_NULLABLE =
+ VARBINARY.andThen(SqlTypeTransforms.TO_NULLABLE);
+
/**
* Type-inference strategy for Histogram agg support.
*/