You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/10/23 04:56:44 UTC
incubator-kylin git commit: KYLIN-1068 fix concurrent bug in
DoubleDeltaSerializer and cleanup debug
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-1068 bf17d9d20 -> a5b271452
KYLIN-1068 fix concurrent bug in DoubleDeltaSerializer and cleanup debug
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a5b27145
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a5b27145
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a5b27145
Branch: refs/heads/KYLIN-1068
Commit: a5b2714520b4ed5a2c16951ad9cc33ca597294c9
Parents: bf17d9d
Author: shaofengshi <sh...@apache.org>
Authored: Fri Oct 23 10:56:25 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Oct 23 10:56:25 2015 +0800
----------------------------------------------------------------------
.../kylin/job/BuildCubeWithEngineTest.java | 14 +++------
.../common/topn/DoubleDeltaSerializer.java | 19 +-----------
.../kylin/common/topn/DoublyLinkedList.java | 11 ++-----
.../apache/kylin/common/topn/TopNCounter.java | 32 --------------------
.../kylin/common/topn/TopNCounterBasicTest.java | 1 -
.../measure/serializer/DataTypeSerializer.java | 2 ++
6 files changed, 9 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a5b27145/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 26ad960..af3dc43 100644
--- a/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/assembly/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -196,8 +196,8 @@ public class BuildCubeWithEngineTest {
long date2 = f.parse("2013-01-01").getTime();
long date3 = f.parse("2022-01-01").getTime();
List<String> result = Lists.newArrayList();
- result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date3));
-// result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date1, date2));
+ result.add(buildSegment("test_kylin_cube_with_slr_empty", date2, date3));
return result;
}
@@ -230,7 +230,6 @@ public class BuildCubeWithEngineTest {
// this cube's start date is 0, end date is 20120601000000
long dateStart = cubeManager.getCube(cubeName).getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- /*
long dateEnd = f.parse("2012-06-01").getTime();
clearSegment(cubeName);
@@ -238,20 +237,15 @@ public class BuildCubeWithEngineTest {
// then submit an append job, start date is 20120601000000, end
// date is 20220101000000
- dateStart = f.parse("2012-06-01").getTime();
+ dateStart = dateEnd;
dateEnd = f.parse("2022-01-01").getTime();
result.add(buildSegment(cubeName, dateStart, dateEnd));
// build an empty segment which doesn't have data
- dateStart = f.parse("2022-01-01").getTime();
+ dateStart = dateEnd;
dateEnd = f.parse("2023-01-01").getTime();
result.add(buildSegment(cubeName, dateStart, dateEnd));
-*/
- long dateEnd = f.parse("2023-01-01").getTime();
-
- clearSegment(cubeName);
- result.add(buildSegment(cubeName, dateStart, dateEnd));
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a5b27145/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java b/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
index f28aeea..2dfb5b0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/DoubleDeltaSerializer.java
@@ -41,8 +41,6 @@ public class DoubleDeltaSerializer {
final private int precision;
final private int multiplier;
- transient long[] reuseDeltas;
-
public DoubleDeltaSerializer() {
this(2);
}
@@ -58,11 +56,6 @@ public class DoubleDeltaSerializer {
}
public void serialize(double values[], ByteBuffer buf) {
- double total = 0.0;
- for (double value: values) {
- total += value;
- }
- buf.putDouble(total);
long[] deltas = calculateDeltas(values);
int deltaSize = maxDeltaSize(deltas, values.length - 1);
@@ -115,9 +108,7 @@ public class DoubleDeltaSerializer {
private long[] calculateDeltas(double[] values) {
int len = values.length - 1;
len = Math.max(0, len);
- if (reuseDeltas == null || reuseDeltas.length < len) {
- reuseDeltas = new long[len];
- }
+ long[] reuseDeltas = new long[len];
if (len == 0)
return reuseDeltas;
@@ -142,20 +133,12 @@ public class DoubleDeltaSerializer {
}
public double[] deserialize(ByteBuffer buf) {
- double expectedTotal = buf.getDouble();
int meta = buf.getInt();
int len = meta & ((1 << LENGTH_BITS) - 1);
double[] result = new double[len];
deserialize(buf, meta, result);
- double actualTotal = 0.0;
- for (double value : result) {
- actualTotal += value;
- }
- if (Math.abs(actualTotal - expectedTotal) > 1.0) {
- System.err.println("DoubleDeltaSerializer expected: " + expectedTotal + ", actual: " + actualTotal + ", array size:" + result.length + ", delta is: " + (expectedTotal - actualTotal));
- }
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a5b27145/core-common/src/main/java/org/apache/kylin/common/topn/DoublyLinkedList.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/DoublyLinkedList.java b/core-common/src/main/java/org/apache/kylin/common/topn/DoublyLinkedList.java
index d268a30..1520ce1 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/DoublyLinkedList.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/DoublyLinkedList.java
@@ -28,7 +28,7 @@ import java.util.Iterator;
*/
public class DoublyLinkedList<T> {
- private int size;
+ private int size = 0;
private ListNode2<T> tail;
private ListNode2<T> head;
@@ -37,14 +37,7 @@ public class DoublyLinkedList<T> {
*/
public ListNode2<T> add(T value) {
ListNode2<T> node = new ListNode2<T>(value);
- if (size++ == 0) {
- tail = node;
- } else {
- node.prev = head;
- head.next = node;
- }
-
- head = node;
+ add(node);
return node;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a5b27145/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
index e1da6b9..e6c5c90 100644
--- a/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
+++ b/core-common/src/main/java/org/apache/kylin/common/topn/TopNCounter.java
@@ -104,8 +104,6 @@ public class TopNCounter<T> implements ITopK<T>, Iterable<Counter<T>> {
counterMap.remove(droppedItem);
counter.item = item;
counter.count = 0.0;
- //FIXME: throw for testing
- throw new IllegalStateException();
}
counterMap.put(item, counterNode);
}
@@ -188,7 +186,6 @@ public class TopNCounter<T> implements ITopK<T>, Iterable<Counter<T>> {
* @return number of items stored
*/
public int size() {
- assert counterMap.size() == counterList.size();
return counterMap.size();
}
@@ -220,12 +217,6 @@ public class TopNCounter<T> implements ITopK<T>, Iterable<Counter<T>> {
}
/**
- * For de-serialization
- */
- public TopNCounter() {
- }
-
- /**
* Merge another counter into this counter;
* @param another
* @return
@@ -233,12 +224,10 @@ public class TopNCounter<T> implements ITopK<T>, Iterable<Counter<T>> {
public TopNCounter<T> merge(TopNCounter<T> another) {
double m1 = 0.0, m2 = 0.0;
if (this.size() >= this.capacity) {
- assert (this.capacity > 0);
m1 = this.counterList.tail().getValue().count;
}
if (another.size() >= another.capacity) {
- assert (another.capacity > 0);
m2 = another.counterList.tail().getValue().count;
}
@@ -301,37 +290,16 @@ public class TopNCounter<T> implements ITopK<T>, Iterable<Counter<T>> {
double[] counters = new double[size()];
int index = 0;
- double last = Double.MIN_VALUE;
for (ListNode2<Counter<T>> bNode = counterList.tail(); bNode != null; bNode = bNode.getNext()) {
Counter<T> b = bNode.getValue();
counters[index] = b.count;
index++;
- if (b.count < last) {
- throw new IllegalStateException("Counters should be in ascending state: " + b.count + " should >= " + last);
- }
- last = b.count;
}
assert index == size();
return counters;
}
- /**
- * Get the item list order by counter values in ascending order
- * @return
- */
- public List<T> getItems() {
- List<T> items = Lists.newArrayList();
- for (ListNode2<Counter<T>> bNode = counterList.tail(); bNode != null; bNode = bNode.getNext()) {
- Counter<T> b = bNode.getValue();
- items.add(b.item);
- }
-
- assert items.size() == this.size();
- return items;
-
- }
-
@Override
public Iterator<Counter<T>> iterator() {
return new TopNCounterIterator();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a5b27145/core-common/src/test/java/org/apache/kylin/common/topn/TopNCounterBasicTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/topn/TopNCounterBasicTest.java b/core-common/src/test/java/org/apache/kylin/common/topn/TopNCounterBasicTest.java
index d67751e..c3e941b 100644
--- a/core-common/src/test/java/org/apache/kylin/common/topn/TopNCounterBasicTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/topn/TopNCounterBasicTest.java
@@ -25,7 +25,6 @@ import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-@Ignore("disable test temporarily")
public class TopNCounterBasicTest {
@Test
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a5b27145/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
index 4fadbb0..f4fd594 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/measure/serializer/DataTypeSerializer.java
@@ -32,6 +32,8 @@ import com.google.common.collect.Maps;
/**
* @author yangli9
*
+ * Note: the implementations MUST be thread-safe.
+ *
*/
abstract public class DataTypeSerializer<T> implements BytesSerializer<T> {