You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/11/24 03:24:01 UTC
[kylin] branch master updated (911dd23 -> 3e12b6d)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 911dd23 KYLIN-4807 Add liveness and readiness probe to k8s deployment
new f04d1d8 FIX KYLIN-4810
new 3e12b6d FIX KYLIN-4810, Add some tips and test case
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/kylin/dict/TrieDictionaryBuilder.java | 26 +++++++++++++---------
.../org/apache/kylin/dict/TrieDictionaryTest.java | 20 +++++++++++++++++
2 files changed, 36 insertions(+), 10 deletions(-)
[kylin] 02/02: FIX KYLIN-4810, Add some tips and test case
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3e12b6d621fe8e5c747a5783f64bc535618c8035
Author: zhengshengjun <zh...@youzan.com>
AuthorDate: Wed Nov 11 16:52:03 2020 +0800
FIX KYLIN-4810, Add some tips and test case
---
.../org/apache/kylin/dict/TrieDictionaryTest.java | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
index 62f4de4..a39b302 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
@@ -170,6 +170,26 @@ public class TrieDictionaryTest {
}
@Test
+ public void utf8PartOverflowTest() {
+ //construct values more than 255 bytes
+ StringBuilder sb = new StringBuilder();
+ for (int i=0; i<255; i++){
+ sb.append('a');
+ }
+ // append utf-8 words, each word is more than 1 byte
+ sb.append("你好");
+ TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
+ b.addValue(sb.toString());
+ TrieDictionary<String> dict = b.build(0);
+ int totalValues = 0;
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ totalValues ++;
+ }
+ assertEquals(1, totalValues);
+ assertEquals(sb.toString(), dict.getValueFromId(0));
+ }
+
+ @Test
public void emptyValueTest() {
ArrayList<String> str = new ArrayList<String>();
str.add("");
[kylin] 01/02: FIX KYLIN-4810
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f04d1d83273f0000ec2d7e0a6d33444205555e9d
Author: zhengshengjun <zh...@youzan.com>
AuthorDate: Wed Nov 11 16:52:03 2020 +0800
FIX KYLIN-4810
---
.../apache/kylin/dict/TrieDictionaryBuilder.java | 26 +++++++++++++---------
1 file changed, 16 insertions(+), 10 deletions(-)
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index b3440a1..803125e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -89,10 +89,14 @@ public class TrieDictionaryBuilder<T> {
// add a converted value (given in byte[] format), use with care, for internal only
void addValue(byte[] value) {
- addValueR(root, value, 0);
+ addValueR(root, value, 0, false);
}
- private void addValueR(Node node, byte[] value, int start) {
+ void addValue(byte[] value, boolean isSplitValue) {
+ addValueR(root, value, 0, isSplitValue);
+ }
+
+ private void addValueR(Node node, byte[] value, int start, boolean isSplitValue) {
hasValue = true;
// match the value part of current node
int i = 0, j = start;
@@ -108,11 +112,13 @@ public class TrieDictionaryBuilder<T> {
// if value fully matched within the current node
if (i == n) {
// if equals to current node, just mark end of value
- node.isEndOfValue = true;
+ if (!isSplitValue) {
+ node.isEndOfValue = true;
+ }
} else {
// otherwise, split the current node into two
Node c = new Node(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children);
- node.reset(BytesUtil.subarray(node.part, 0, i), true);
+ node.reset(BytesUtil.subarray(node.part, 0, i), isSplitValue? false : true);
node.children.add(c);
}
return;
@@ -121,7 +127,7 @@ public class TrieDictionaryBuilder<T> {
// if partially matched the current, split the current node, add the new value, make a 3-way
if (i < n) {
Node c1 = new Node(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children);
- Node c2 = new Node(BytesUtil.subarray(value, j, nn), true);
+ Node c2 = new Node(BytesUtil.subarray(value, j, nn), isSplitValue? false : true);
node.reset(BytesUtil.subarray(node.part, 0, i), false);
if (comp < 0) {
node.children.add(c1);
@@ -152,10 +158,10 @@ public class TrieDictionaryBuilder<T> {
}
if (found) {
// found a child node matching the first byte, continue in that child
- addValueR(node.children.get(mid), value, j);
+ addValueR(node.children.get(mid), value, j, isSplitValue);
} else {
// otherwise, make the value a new child
- Node c = new Node(BytesUtil.subarray(value, j, nn), true);
+ Node c = new Node(BytesUtil.subarray(value, j, nn), isSplitValue ? false : true);
node.children.add(comp <= 0 ? mid : mid + 1, c);
}
}
@@ -389,7 +395,7 @@ public class TrieDictionaryBuilder<T> {
completeParts.append(node.part);
completeParts.append(first255);
byte[] visited = completeParts.retrieve();
- this.addValue(visited);
+ this.addValue(visited, true);
completeParts.withdraw(255);
completeParts.withdraw(node.part.length);
}