You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/07/03 07:44:26 UTC
[15/26] kylin git commit: minor, make calcite work with unicode
minor, make calcite work with unicode
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f52eb0c5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f52eb0c5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f52eb0c5
Branch: refs/heads/kylin-2.1.x
Commit: f52eb0c5c667b655d9a20f6226af2a3f5cbe0f4e
Parents: 2546e00
Author: Li Yang <li...@apache.org>
Authored: Fri Jun 30 13:42:21 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 30 13:46:14 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 21 ++++++++++++++++++++
.../apache/kylin/query/ITCombinationTest.java | 3 +--
.../org/apache/kylin/query/KylinTestBase.java | 4 ++--
.../kylin/query/schema/OLAPSchemaFactory.java | 19 ------------------
4 files changed, 24 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 1298807..cc08056 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -28,6 +28,7 @@ import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringReader;
import java.net.URL;
+import java.nio.ByteOrder;
import java.nio.charset.Charset;
import java.util.Enumeration;
import java.util.Map;
@@ -58,6 +59,26 @@ public class KylinConfig extends KylinConfigBase {
// thread-local instances, will override SYS_ENV_INSTANCE
private static transient ThreadLocal<KylinConfig> THREAD_ENV_INSTANCE = new ThreadLocal<>();
+
+ static {
+ /*
+ * Make Calcite to work with Unicode.
+ *
+ * Sets default char set for string literals in SQL and row types of
+ * RelNode. This is more a label used to compare row type equality. For
+ * both SQL string and row record, they are passed to Calcite in String
+ * object and does not require additional codec.
+ *
+ * Ref SaffronProperties.defaultCharset
+ * Ref SqlUtil.translateCharacterSetName()
+ * Ref NlsString constructor()
+ */
+ // copied from org.apache.calcite.util.ConversionUtil.NATIVE_UTF16_CHARSET_NAME
+ String NATIVE_UTF16_CHARSET_NAME = (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) ? "UTF-16BE" : "UTF-16LE";
+ System.setProperty("saffron.default.charset", NATIVE_UTF16_CHARSET_NAME);
+ System.setProperty("saffron.default.nationalcharset", NATIVE_UTF16_CHARSET_NAME);
+ System.setProperty("saffron.default.collation.name", NATIVE_UTF16_CHARSET_NAME + "$en_US");
+ }
public static KylinConfig getInstanceFromEnv() {
synchronized (KylinConfig.class) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 0bed5ed..5051db1 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -18,7 +18,6 @@
package org.apache.kylin.query;
-import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
@@ -43,7 +42,7 @@ public class ITCombinationTest extends ITKylinQueryTest {
private static final Logger logger = LoggerFactory.getLogger(ITCombinationTest.class);
@BeforeClass
- public static void setUp() throws SQLException {
+ public static void setUp() {
Map<RealizationType, Integer> priorities = Maps.newHashMap();
priorities.put(RealizationType.HYBRID, 0);
priorities.put(RealizationType.CUBE, 0);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index a05d0c3..af05237 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -21,9 +21,9 @@ package org.apache.kylin.query;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -166,7 +166,7 @@ public class KylinTestBase {
}
public static String getTextFromFile(File file) throws IOException {
- BufferedReader reader = new BufferedReader(new FileReader(file));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
String line = null;
StringBuilder stringBuilder = new StringBuilder();
String ls = System.getProperty("line.separator");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f52eb0c5/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
index bfe49cf..c098b23 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
@@ -31,7 +31,6 @@ import java.util.Map.Entry;
import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.SchemaFactory;
import org.apache.calcite.schema.SchemaPlus;
-import org.apache.calcite.util.ConversionUtil;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -50,24 +49,6 @@ import com.google.common.collect.Maps;
public class OLAPSchemaFactory implements SchemaFactory {
public static final Logger logger = LoggerFactory.getLogger(OLAPSchemaFactory.class);
- static {
- /*
- * Tricks Optiq to work with Unicode.
- *
- * Sets default char set for string literals in SQL and row types of
- * RelNode. This is more a label used to compare row type equality. For
- * both SQL string and row record, they are passed to Optiq in String
- * object and does not require additional codec.
- *
- * Ref SaffronProperties.defaultCharset
- * Ref SqlUtil.translateCharacterSetName()
- * Ref NlsString constructor()
- */
- System.setProperty("saffron.default.charset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
- System.setProperty("saffron.default.nationalcharset", ConversionUtil.NATIVE_UTF16_CHARSET_NAME);
- System.setProperty("saffron.default.collation.name", ConversionUtil.NATIVE_UTF16_CHARSET_NAME + "$en_US");
- }
-
private final static String SCHEMA_PROJECT = "project";
@Override