You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by pa...@apache.org on 2020/07/17 02:24:46 UTC
[shardingsphere] branch master updated: add emoji support (#6369)
This is an automated email from the ASF dual-hosted git repository.
panjuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new d99d7e8 add emoji support (#6369)
d99d7e8 is described below
commit d99d7e898da41e7ca8e5a5ad33d4adc0cc14e95b
Author: xbkaishui <xb...@126.com>
AuthorDate: Fri Jul 17 10:24:20 2020 +0800
add emoji support (#6369)
* add emoji support test case
* add space for comment
* add rewrite test case for emoji literals
---
.../src/test/resources/mix/insert.xml | 5 +++++
.../src/test/resources/sharding/insert.xml | 5 +++++
.../sql/parser/core/parser/SQLParserFactory.java | 9 +++++++--
.../src/test/resources/case/dml/insert.xml | 16 ++++++++++++++++
.../src/test/resources/sql/supported/dml/insert.xml | 1 +
5 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-rewrite/src/test/resources/mix/insert.xml b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-rewrite/src/test/resources/mix/insert.xml
index 3425f2b..5802a1d 100644
--- a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-rewrite/src/test/resources/mix/insert.xml
+++ b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-rewrite/src/test/resources/mix/insert.xml
@@ -60,6 +60,11 @@
<input sql="INSERT INTO t_account_bak(password, amount, status) VALUES ('aaa', 1000, 'OK'), ('bbb', 2000, 'OK'), ('ccc', 3000, 'OK'), ('ddd', 4000, 'OK')" />
<output sql="INSERT INTO t_account_bak_1(cipher_password, assisted_query_password, plain_password, cipher_amount, plain_amount, status, account_id) VALUES ('encrypt_aaa', 'assisted_query_aaa', 'aaa', 'encrypt_1000', 1000, 'OK', 1), ('encrypt_bbb', 'assisted_query_bbb', 'bbb', 'encrypt_2000', 2000, 'OK', 1), ('encrypt_ccc', 'assisted_query_ccc', 'ccc', 'encrypt_3000', 3000, 'OK', 1), ('encrypt_ddd', 'assisted_query_ddd', 'ddd', 'encrypt_4000', 4000, 'OK', 1)" />
</rewrite-assertion>
+
+ <rewrite-assertion id="insert_values_with_columns_with_plain_without_id_for_emojis">
+ <input sql="INSERT INTO t_account_bak(password, amount, status) VALUES ('aaa', 1000, 'test๐๐'), ('bbb', 2000, 'test๐๐'), ('ccc', 3000, 'test๐๐'), ('ddd', 4000, 'test๐๐')" />
+ <output sql="INSERT INTO t_account_bak_1(cipher_password, assisted_query_password, plain_password, cipher_amount, plain_amount, status, account_id) VALUES ('encrypt_aaa', 'assisted_query_aaa', 'aaa', 'encrypt_1000', 1000, 'test๐๐', 1), ('encrypt_bbb', 'assisted_query_bbb', 'bbb', 'encrypt_2000', 2000, 'test๐๐', 1), ('encrypt_ccc', 'assisted_query_ccc', 'ccc', 'encrypt_3000', 3000, 'test๐๐', 1), ('encrypt_ddd', 'assisted_query_ddd', 'ddd', 'encrypt_4000', 4000, 'test๐๐', 1)" />
+ </rewrite-assertion>
<rewrite-assertion id="insert_values_without_columns_with_id_for_parameters">
<input sql="INSERT INTO t_account(account_id, password, amount, status) VALUES (?, ?, ?, ?), (2, 'bbb', 2000, 'OK'), (?, ?, ?, ?), (4, 'ddd', 4000, 'OK')" parameters="1, aaa, 1000, OK, 3, ccc, 3000, OK" />
diff --git a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-rewrite/src/test/resources/sharding/insert.xml b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-rewrite/src/test/resources/sharding/insert.xml
index 731727e..948712f 100644
--- a/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-rewrite/src/test/resources/sharding/insert.xml
+++ b/shardingsphere-features/shardingsphere-sharding/shardingsphere-sharding-rewrite/src/test/resources/sharding/insert.xml
@@ -56,6 +56,11 @@
<input sql="INSERT INTO t_account VALUES (1000, 'OK')" />
<output sql="INSERT INTO t_account_1(amount, status, account_id) VALUES (1000, 'OK', 1)" />
</rewrite-assertion>
+
+ <rewrite-assertion id="insert_values_without_columns_without_id_for_emojis" db-type="MySQL">
+ <input sql="INSERT INTO t_account VALUES (1000, 'test๐๐')" />
+ <output sql="INSERT INTO t_account_1(amount, status, account_id) VALUES (1000, 'test๐๐', 1)" />
+ </rewrite-assertion>
<rewrite-assertion id="insert_multiple_values_with_columns_with_id_for_parameters">
<input sql="INSERT INTO t_account (account_id, amount, status) VALUES (100, 0, 'OK'), (?, ?, ?), (102, 2000, 'OK'), (?, ?, ?), (?, ?, ?), (105, 5000, 'OK')" parameters="101, 1000, OK, 103, 3000, OK, 104, 4000, OK" />
diff --git a/shardingsphere-sql-parser/shardingsphere-sql-parser-engine/src/main/java/org/apache/shardingsphere/sql/parser/core/parser/SQLParserFactory.java b/shardingsphere-sql-parser/shardingsphere-sql-parser-engine/src/main/java/org/apache/shardingsphere/sql/parser/core/parser/SQLParserFactory.java
index dbcfaff..0b1eebf 100644
--- a/shardingsphere-sql-parser/shardingsphere-sql-parser-engine/src/main/java/org/apache/shardingsphere/sql/parser/core/parser/SQLParserFactory.java
+++ b/shardingsphere-sql-parser/shardingsphere-sql-parser-engine/src/main/java/org/apache/shardingsphere/sql/parser/core/parser/SQLParserFactory.java
@@ -21,7 +21,8 @@ import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.SneakyThrows;
import org.antlr.v4.runtime.CharStream;
-import org.antlr.v4.runtime.CharStreams;
+import org.antlr.v4.runtime.CodePointBuffer;
+import org.antlr.v4.runtime.CodePointCharStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.TokenStream;
@@ -29,6 +30,8 @@ import org.apache.shardingsphere.sql.parser.api.parser.SQLParser;
import org.apache.shardingsphere.sql.parser.core.SQLParserConfigurationRegistry;
import org.apache.shardingsphere.sql.parser.spi.SQLParserConfiguration;
+import java.nio.CharBuffer;
+
/**
* SQL parser factory.
*/
@@ -48,7 +51,9 @@ public final class SQLParserFactory {
@SneakyThrows(ReflectiveOperationException.class)
private static SQLParser createSQLParser(final String sql, final SQLParserConfiguration configuration) {
- Lexer lexer = (Lexer) configuration.getLexerClass().getConstructor(CharStream.class).newInstance(CharStreams.fromString(sql));
+ CodePointBuffer buffer = CodePointBuffer.withChars(CharBuffer.wrap(sql.toCharArray()));
+ CodePointCharStream codePointCharStream = CodePointCharStream.fromBuffer(buffer);
+ Lexer lexer = (Lexer) configuration.getLexerClass().getConstructor(CharStream.class).newInstance(codePointCharStream);
return configuration.getParserClass().getConstructor(TokenStream.class).newInstance(new CommonTokenStream(lexer));
}
}
diff --git a/shardingsphere-sql-parser/shardingsphere-sql-parser-test/src/test/resources/case/dml/insert.xml b/shardingsphere-sql-parser/shardingsphere-sql-parser-test/src/test/resources/case/dml/insert.xml
index 2e88c73..01a0e4d 100644
--- a/shardingsphere-sql-parser/shardingsphere-sql-parser-test/src/test/resources/case/dml/insert.xml
+++ b/shardingsphere-sql-parser/shardingsphere-sql-parser-test/src/test/resources/case/dml/insert.xml
@@ -1288,4 +1288,20 @@
</assignment>
</on-duplicate-key-columns>
</insert>
+
+ <insert sql-case-id="insert_with_emoji_value" >
+ <table name="t_emoji_test" start-index="12" stop-index="23" />
+ <columns start-index="24" stop-index="29">
+ <column name="col1" start-index="25" stop-index="28" />
+ </columns>
+ <values>
+ <value>
+ <assignment-value>
+ <parameter-marker-expression value="0" start-index="99" stop-index="99" />
+ <literal-expression value="test๐" start-index="38" stop-index="45" />
+ </assignment-value>
+ </value>
+ </values>
+ </insert>
+
</sql-parser-test-cases>
diff --git a/shardingsphere-sql-parser/shardingsphere-sql-parser-test/src/test/resources/sql/supported/dml/insert.xml b/shardingsphere-sql-parser/shardingsphere-sql-parser-test/src/test/resources/sql/supported/dml/insert.xml
index 5ddfec8..9c76c18 100644
--- a/shardingsphere-sql-parser/shardingsphere-sql-parser-test/src/test/resources/sql/supported/dml/insert.xml
+++ b/shardingsphere-sql-parser/shardingsphere-sql-parser-test/src/test/resources/sql/supported/dml/insert.xml
@@ -60,4 +60,5 @@
<sql-case id="insert_select_with_generate_key_column" value="INSERT INTO t_order_item(item_id, order_id, user_id, status, creation_date) SELECT item_id, order_id, user_id, 'insert', '2017-08-08' FROM t_order_item WHERE item_id = ?" db-types="MySQL" />
<sql-case id="insert_select_without_generate_key_column" value="INSERT INTO t_order_item(order_id, user_id, status, creation_date) SELECT order_id, user_id, 'insert', '2017-08-08' FROM t_order_item WHERE order_id = ?" db-types="MySQL" />
<sql-case id="insert_select_on_duplicate_key_update" value="INSERT INTO t_order(order_id, user_id, status) SELECT order_id, user_id, status FROM t_order WHERE order_id = ? ON DUPLICATE KEY UPDATE status = VALUES(status)" db-types="MySQL" />
+ <sql-case id="insert_with_emoji_value" value="INSERT INTO t_emoji_test(col1) VALUES('test๐')" db-types="MySQL" />
</sql-cases>