You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/07 04:32:00 UTC
[01/50] [abbrv] kylin git commit: KYLIN-2213 minor code review
[Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/master-cdh5.7 7c742023b -> db1b826ac (forced update)
KYLIN-2213 minor code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2ecf18d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2ecf18d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2ecf18d
Branch: refs/heads/master-cdh5.7
Commit: a2ecf18de8ceb73f6b2748cd3e1d7b43ac1ca8ac
Parents: be7b5d1
Author: Li Yang <li...@apache.org>
Authored: Mon Nov 28 16:28:56 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Nov 28 16:28:56 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/BuiltInFunctionTransformer.java | 14 ++------------
1 file changed, 2 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2ecf18d/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
index 0f8e895..f451c2c 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -89,12 +89,7 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
if (dict == null)
return null;
- CompareTupleFilter translated;
- if (builtInFunctionTupleFilter.isReversed()) {
- translated = new CompareTupleFilter(FilterOperatorEnum.NOTIN);
- } else {
- translated = new CompareTupleFilter(FilterOperatorEnum.IN);
- }
+ CompareTupleFilter translated = new CompareTupleFilter(builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
translated.addChild(new ColumnTupleFilter(columnRef));
try {
@@ -126,12 +121,7 @@ public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
if (dict == null)
return null;
- CompareTupleFilter translated;
- if (builtInFunctionTupleFilter.isReversed()) {
- translated = new CompareTupleFilter(FilterOperatorEnum.NOTIN);
- } else {
- translated = new CompareTupleFilter(FilterOperatorEnum.IN);
- }
+ CompareTupleFilter translated = new CompareTupleFilter(builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
translated.addChild(new ColumnTupleFilter(columnRef));
try {
[03/50] [abbrv] kylin git commit: minor, fix typo
Posted by li...@apache.org.
minor, fix typo
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c3d62e9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c3d62e9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c3d62e9c
Branch: refs/heads/master-cdh5.7
Commit: c3d62e9c5e6c39771bec1bbe2660130ff2033ef3
Parents: b1448e5
Author: lidongsjtu <li...@apache.org>
Authored: Mon Nov 28 23:48:46 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Nov 28 23:48:46 2016 +0800
----------------------------------------------------------------------
build/smoke-test/testDiag.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c3d62e9c/build/smoke-test/testDiag.py
----------------------------------------------------------------------
diff --git a/build/smoke-test/testDiag.py b/build/smoke-test/testDiag.py
index cc932da..588864b 100644
--- a/build/smoke-test/testDiag.py
+++ b/build/smoke-test/testDiag.py
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-# This is python unittest used in smoke-test.sh, aim to testing query via rest APIs.
+# This is python unittest used in smoke-test.sh, aim to testing diagnosis via rest APIs.
import unittest
import requests
@@ -40,5 +40,5 @@ class testDiag(unittest.TestCase):
if __name__ == '__main__':
- print 'Test Diagnogis for Kylin sample.'
+ print 'Test Diagnosis for Kylin sample.'
unittest.main()
[39/50] [abbrv] kylin git commit: KYLIN-2245 further minor refactor
Posted by li...@apache.org.
KYLIN-2245 further minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9968c225
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9968c225
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9968c225
Branch: refs/heads/master-cdh5.7
Commit: 9968c2257c2a8605637a93830d1d95db94185dce
Parents: fc1e11a
Author: Li Yang <li...@apache.org>
Authored: Tue Dec 6 11:10:26 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Dec 6 11:10:26 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 2 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 3 ++-
.../apache/kylin/metadata/model/ISegment.java | 3 +++
.../apache/kylin/metadata/model/Segments.java | 15 +++------------
.../test_case_data/sandbox/kylin_hive_conf.xml | 20 +-------------------
5 files changed, 10 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index a3665f5..1d60575 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -368,7 +368,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public Segments calculateToBeSegments(CubeSegment newSegment) {
- return segments.calculateToBeSegments(newSegment, getModel().getPartitionDesc().isPartitioned());
+ return segments.calculateToBeSegments(newSegment);
}
public CubeSegment getLastSegment() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 5339d81..e155f86 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -361,7 +361,8 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
return Segments.sourceOffsetContains(this, seg);
}
- public void validate() {
+ @Override
+ public void validate() throws IllegalStateException {
if (cubeInstance.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
if (!isSourceOffsetsOn() && dateRangeStart >= dateRangeEnd)
throw new IllegalStateException("Invalid segment, dateRangeStart(" + dateRangeStart + ") must be smaller than dateRangeEnd(" + dateRangeEnd + ") in segment " + this);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index f006613..d46ea96 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -37,4 +37,7 @@ public interface ISegment {
public SegmentStatusEnum getStatus();
public long getLastBuildTime();
+
+ public void validate() throws IllegalStateException;
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 5198dc7..9371f76 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -229,7 +229,7 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
* - Favors new segments over the old
* - Favors big segments over the small
*/
- public Segments calculateToBeSegments(ISegment newSegment, boolean isPartitioned) {
+ public Segments calculateToBeSegments(ISegment newSegment) {
Segments tobe = (Segments) this.clone();
if (newSegment != null && !tobe.contains(newSegment)) {
@@ -242,12 +242,12 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
Collections.sort(tobe);
ISegment firstSeg = tobe.getFirst();
- validate(firstSeg, isPartitioned);
+ firstSeg.validate();
for (int i = 0, j = 1; j < tobe.size();) {
ISegment is = (ISegment) tobe.get(i);
ISegment js = (ISegment) tobe.get(j);
- validate(js, isPartitioned);
+ js.validate();
// check i is either ready or new
if (!isNew(is) && !isReady(is)) {
@@ -306,15 +306,6 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
return tobe;
}
- private void validate(ISegment seg, boolean isPartitioned) {
- if (isPartitioned) {
- if (!seg.isSourceOffsetsOn() && seg.getDateRangeStart() >= seg.getDateRangeEnd())
- throw new IllegalStateException("Invalid segment, dateRangeStart(" + seg.getDateRangeStart() + ") must be smaller than dateRangeEnd(" + seg.getDateRangeEnd() + ") in segment " + seg);
- if (seg.isSourceOffsetsOn() && seg.getSourceOffsetStart() >= seg.getSourceOffsetEnd())
- throw new IllegalStateException("Invalid segment, sourceOffsetStart(" + seg.getSourceOffsetStart() + ") must be smaller than sourceOffsetEnd(" + seg.getSourceOffsetEnd() + ") in segment " + seg);
- }
- }
-
private boolean isReady(ISegment seg) {
return seg.getStatus() == SegmentStatusEnum.READY;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9968c225/examples/test_case_data/sandbox/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_hive_conf.xml b/examples/test_case_data/sandbox/kylin_hive_conf.xml
index c85731d..9a60bdd 100644
--- a/examples/test_case_data/sandbox/kylin_hive_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_hive_conf.xml
@@ -29,29 +29,11 @@
</property>
<property>
- <name>hive.auto.convert.join.noconditionaltask</name>
- <value>true</value>
- <description>enable map-side join</description>
- </property>
-
- <property>
- <name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>300000000</value>
- <description>enable map-side join</description>
- </property>
-
- <property>
<name>mapreduce.map.output.compress.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
<description></description>
</property>
- <property>
- <name>hive.execution.engine</name>
- <value>mr</value>
- <description></description>
- </property>
-
<!--
<property>
<name>mapreduce.output.fileoutputformat.compress.codec</name>
@@ -64,4 +46,4 @@
<description>Size for the merged file</description>
</property>
-->
-</configuration>
\ No newline at end of file
+</configuration>
[12/50] [abbrv] kylin git commit: minor, massin udf reverse
Posted by li...@apache.org.
minor, massin udf reverse
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1b90adb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1b90adb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1b90adb
Branch: refs/heads/master-cdh5.7
Commit: b1b90adbe896e808776caf49ad7172979cf860df
Parents: 875a593
Author: Roger Shi <ro...@hotmail.com>
Authored: Wed Nov 30 13:35:12 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Nov 30 13:49:16 2016 +0800
----------------------------------------------------------------------
.../metadata/filter/UDF/MassInTupleFilter.java | 28 +++++++++++++++++++-
1 file changed, 27 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b1b90adb/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 3f0546c..29c5550 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -50,6 +50,7 @@ public class MassInTupleFilter extends FunctionTupleFilter {
private String filterTableName;//key in MetadataManager.extFilterMap
private String filterTableResourceIdentifier;//HDFS path, or hbase table name depending on FilterTableType
private Functions.FilterTableType filterTableType;
+ private boolean reverse = false;
public MassInTupleFilter() {
super(Lists.<TupleFilter> newArrayList(), TupleFilter.FilterOperatorEnum.MASSIN);
@@ -66,7 +67,18 @@ public class MassInTupleFilter extends FunctionTupleFilter {
valueProvider = VALUE_PROVIDER_FACTORY.getProvider(filterTableType, filterTableResourceIdentifier, column);
}
boolean ret = valueProvider.getMassInValues().contains(colValue);
- return ret;
+ return reverse ? !ret : ret;
+ }
+
+ @Override
+ public TupleFilter reverse() {
+ try {
+ MassInTupleFilter result = (MassInTupleFilter) this.clone();
+ result.setReverse(!this.isReverse());
+ return result;
+ } catch (CloneNotSupportedException e) {
+ throw new UnsupportedOperationException(e);
+ }
}
@Override
@@ -144,4 +156,18 @@ public class MassInTupleFilter extends FunctionTupleFilter {
return false;
}
+ public boolean isReverse() {
+ return reverse;
+ }
+
+ public void setReverse(boolean reverse) {
+ this.reverse = reverse;
+ }
+
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ MassInTupleFilter result = new MassInTupleFilter();
+ result.setReverse(this.isReverse());
+ return result;
+ }
}
[36/50] [abbrv] kylin git commit: KYLIN-2248 TopN merge further
optimization after KYLIN-1917
Posted by li...@apache.org.
KYLIN-2248 TopN merge further optimization after KYLIN-1917
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59a30f66
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59a30f66
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59a30f66
Branch: refs/heads/master-cdh5.7
Commit: 59a30f66d47cc1838e6852405699fd7957bfac29
Parents: af429e5
Author: shaofengshi <sh...@apache.org>
Authored: Sun Dec 4 09:39:45 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 5 17:42:34 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/measure/topn/TopNCounter.java | 47 ++++++--------------
1 file changed, 13 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/59a30f66/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
index 968e694..caf7961 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounter.java
@@ -26,11 +26,9 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import com.google.common.collect.Maps;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
/**
* Modified from the StreamSummary.java in https://github.com/addthis/stream-lib
@@ -157,41 +155,22 @@ public class TopNCounter<T> implements Iterable<Counter<T>> {
* @return
*/
public TopNCounter<T> merge(TopNCounter<T> another) {
- double m1 = 0.0, m2 = 0.0;
- if (this.size() >= this.capacity) {
- m1 = this.counterList.getLast().count;
- }
-
- if (another.size() >= another.capacity) {
- m2 = another.counterList.getLast().count;
- }
-
- Set<T> duplicateItems = Sets.newHashSet();
- List<T> notDuplicateItems = Lists.newArrayList();
-
- for (Map.Entry<T, Counter<T>> entry : this.counterMap.entrySet()) {
- T item = entry.getKey();
- Counter<T> existing = another.counterMap.get(item);
- if (existing != null) {
- duplicateItems.add(item);
- } else {
- notDuplicateItems.add(item);
+ boolean thisFull = this.size() >= this.capacity;
+ boolean anotherFull = another.size() >= another.capacity;
+ double m1 = thisFull ? this.counterList.getLast().count : 0.0;
+ double m2 = anotherFull ? another.counterList.getLast().count : 0.0;
+
+ if (anotherFull == true) {
+ for (Counter<T> entry : this.counterMap.values()) {
+ entry.count += m2;
}
}
- for (T item : duplicateItems) {
- this.offer(item, another.counterMap.get(item).count);
- }
-
- for (T item : notDuplicateItems) {
- this.offer(item, m2);
- }
-
for (Map.Entry<T, Counter<T>> entry : another.counterMap.entrySet()) {
- T item = entry.getKey();
- if (duplicateItems.contains(item) == false) {
- double counter = entry.getValue().count;
- this.offer(item, counter + m1);
+ if (this.counterMap.containsKey(entry.getKey())) {
+ this.offer(entry.getValue().getItem(), (entry.getValue().count - m2));
+ } else {
+ this.offer(entry.getValue().getItem(), entry.getValue().count + m1);
}
}
[27/50] [abbrv] kylin git commit: KYLIN-2245 slim ISegment a bit
Posted by li...@apache.org.
KYLIN-2245 slim ISegment a bit
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0eebd7d8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0eebd7d8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0eebd7d8
Branch: refs/heads/master-cdh5.7
Commit: 0eebd7d8ab10b6b2b4823d6a23eaa774ac36989e
Parents: 3ca5395
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 2 17:23:16 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 2 17:26:13 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeSegment.java | 20 +++-----------------
.../apache/kylin/metadata/model/ISegment.java | 1 -
.../apache/kylin/metadata/model/Segments.java | 12 +++++++++++-
3 files changed, 14 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 1fc28eb..5339d81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -38,6 +38,7 @@ import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
@@ -350,29 +351,14 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
this.sourceOffsetEnd = sourceOffsetEnd;
}
- public boolean dateRangeOverlaps(CubeSegment seg) {
- return dateRangeStart < seg.dateRangeEnd && seg.dateRangeStart < dateRangeEnd;
- }
-
- public boolean dateRangeContains(CubeSegment seg) {
- return dateRangeStart <= seg.dateRangeStart && seg.dateRangeEnd <= dateRangeEnd;
- }
-
// date range is used in place of source offsets when offsets are missing
public boolean sourceOffsetOverlaps(CubeSegment seg) {
- if (isSourceOffsetsOn())
- return sourceOffsetStart < seg.sourceOffsetEnd && seg.sourceOffsetStart < sourceOffsetEnd;
- else
- return dateRangeOverlaps(seg);
+ return Segments.sourceOffsetOverlaps(this, seg);
}
// date range is used in place of source offsets when offsets are missing
- @Override
public boolean sourceOffsetContains(ISegment seg) {
- if (isSourceOffsetsOn())
- return sourceOffsetStart <= ((CubeSegment) seg).sourceOffsetStart && ((CubeSegment) seg).sourceOffsetEnd <= sourceOffsetEnd;
- else
- return dateRangeContains(((CubeSegment) seg));
+ return Segments.sourceOffsetContains(this, seg);
}
public void validate() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index e97f4f4..e3fcdcb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -36,5 +36,4 @@ public interface ISegment{
public long getLastBuildTime();
- public boolean sourceOffsetContains(ISegment seg);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0eebd7d8/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 104c2af..f0a58cb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -21,6 +21,16 @@ package org.apache.kylin.metadata.model;
import java.util.ArrayList;
public class Segments<T extends ISegment> extends ArrayList<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static boolean sourceOffsetContains(ISegment a, ISegment b) {
+ return a.getSourceOffsetStart() <= b.getSourceOffsetStart() && b.getSourceOffsetEnd() <= a.getSourceOffsetEnd();
+ }
+
+ public static boolean sourceOffsetOverlaps(ISegment a, ISegment b) {
+ return a.getSourceOffsetStart() < b.getSourceOffsetEnd() && b.getSourceOffsetStart() < a.getSourceOffsetEnd();
+ }
public T getFirstSegment() {
if (this == null || this.size() == 0) {
@@ -121,7 +131,7 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
if (seg == mergedSegment)
continue;
- if (mergedSegment.sourceOffsetContains(seg)) {
+ if (sourceOffsetContains(mergedSegment, seg)) {
// make sure no holes
if (result.size() > 0 && result.getLast().getSourceOffsetEnd() != seg.getSourceOffsetStart())
throw new IllegalStateException("Merging segments must not have holes between " + result.getLast() + " and " + seg);
[37/50] [abbrv] kylin git commit: KYLIN-2245 slim Segments in
CubeMananger
Posted by li...@apache.org.
KYLIN-2245 slim Segments in CubeMananger
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0a441c3f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0a441c3f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0a441c3f
Branch: refs/heads/master-cdh5.7
Commit: 0a441c3fa30aee0a06cfc6301f7fbfa412103179
Parents: 59a30f6
Author: Cheng Wang <ch...@kyligence.io>
Authored: Mon Dec 5 16:17:31 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Dec 5 20:20:30 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 14 ++
.../java/org/apache/kylin/cube/CubeManager.java | 165 +--------------
.../apache/kylin/metadata/model/ISegment.java | 5 +-
.../apache/kylin/metadata/model/Segments.java | 206 ++++++++++++++++++-
4 files changed, 221 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0a441c3f/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 8b12c2e..ecbb437 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -18,6 +18,7 @@
package org.apache.kylin.cube;
+import java.io.IOException;
import java.util.List;
import java.util.Set;
@@ -25,6 +26,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -361,6 +363,18 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return this.getDescriptor().getAutoMergeTimeRanges() != null && this.getDescriptor().getAutoMergeTimeRanges().length > 0;
}
+ public Pair<Long, Long> autoMergeCubeSegments() throws IOException {
+ return segments.autoMergeCubeSegments(needAutoMerge(), getName(), getDescriptor().getAutoMergeTimeRanges());
+ }
+
+ public Segments calculateToBeSegments(CubeSegment newSegment) {
+ return segments.calculateToBeSegments(newSegment, getModel().getPartitionDesc().isPartitioned());
+ }
+
+ public Pair<CubeSegment, CubeSegment> findMergeOffsetsByDateRange(Segments<CubeSegment> segs, long startDate, long endDate, long skipSegDateRangeCap) {
+ return this.segments.findMergeOffsetsByDateRange(segs, startDate, endDate, skipSegDateRangeCap);
+ }
+
public CubeSegment getLastSegment() {
List<CubeSegment> existing = getSegments();
if (existing.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0a441c3f/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 4ba29af..296a4e7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -27,7 +27,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;
@@ -520,7 +519,7 @@ public class CubeManager implements IRealizationProvider {
if (isOffsetsOn) {
// offset cube, merge by date range?
if (startOffset == endOffset) {
- Pair<CubeSegment, CubeSegment> pair = findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), startDate, endDate, Long.MAX_VALUE);
+ Pair<CubeSegment, CubeSegment> pair = cube.findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), startDate, endDate, Long.MAX_VALUE);
if (pair == null)
throw new IllegalArgumentException("Find no segments to merge by date range " + startDate + "-" + endDate + " for cube " + cube);
startOffset = pair.getFirst().getSourceOffsetStart();
@@ -580,32 +579,6 @@ public class CubeManager implements IRealizationProvider {
return newSegment;
}
- private Pair<CubeSegment, CubeSegment> findMergeOffsetsByDateRange(List<CubeSegment> segments, long startDate, long endDate, long skipSegDateRangeCap) {
- // must be offset cube
- LinkedList<CubeSegment> result = Lists.newLinkedList();
- for (CubeSegment seg : segments) {
-
- // include if date range overlaps
- if (startDate < seg.getDateRangeEnd() && seg.getDateRangeStart() < endDate) {
-
- // reject too big segment
- if (seg.getDateRangeEnd() - seg.getDateRangeStart() > skipSegDateRangeCap)
- break;
-
- // reject holes
- if (result.size() > 0 && result.getLast().getSourceOffsetEnd() != seg.getSourceOffsetStart())
- break;
-
- result.add(seg);
- }
- }
-
- if (result.size() <= 1)
- return null;
- else
- return Pair.newPair(result.getFirst(), result.getLast());
- }
-
public static long minDateRangeStart(List<CubeSegment> mergingSegments) {
long min = Long.MAX_VALUE;
for (CubeSegment seg : mergingSegments)
@@ -708,50 +681,7 @@ public class CubeManager implements IRealizationProvider {
}
public Pair<Long, Long> autoMergeCubeSegments(CubeInstance cube) throws IOException {
- if (!cube.needAutoMerge()) {
- logger.debug("Cube " + cube.getName() + " doesn't need auto merge");
- return null;
- }
-
- List<CubeSegment> buildingSegs = cube.getBuildingSegments();
- if (buildingSegs.size() > 0) {
- logger.debug("Cube " + cube.getName() + " has " + buildingSegs.size() + " building segments");
- }
-
- List<CubeSegment> readySegs = cube.getSegments(SegmentStatusEnum.READY);
-
- List<CubeSegment> mergingSegs = Lists.newArrayList();
- if (buildingSegs.size() > 0) {
-
- for (CubeSegment building : buildingSegs) {
- // exclude those under-merging segs
- for (CubeSegment ready : readySegs) {
- if (ready.getSourceOffsetStart() >= building.getSourceOffsetStart() && ready.getSourceOffsetEnd() <= building.getSourceOffsetEnd()) {
- mergingSegs.add(ready);
- }
- }
- }
- }
-
- // exclude those already under merging segments
- readySegs.removeAll(mergingSegs);
-
- long[] timeRanges = cube.getDescriptor().getAutoMergeTimeRanges();
- Arrays.sort(timeRanges);
-
- for (int i = timeRanges.length - 1; i >= 0; i--) {
- long toMergeRange = timeRanges[i];
-
- for (int s = 0; s < readySegs.size(); s++) {
- CubeSegment seg = readySegs.get(s);
- Pair<CubeSegment, CubeSegment> p = findMergeOffsetsByDateRange(readySegs.subList(s, readySegs.size()), //
- seg.getDateRangeStart(), seg.getDateRangeStart() + toMergeRange, toMergeRange);
- if (p != null && p.getSecond().getDateRangeEnd() - p.getFirst().getDateRangeStart() >= toMergeRange)
- return Pair.newPair(p.getFirst().getSourceOffsetStart(), p.getSecond().getSourceOffsetEnd());
- }
- }
-
- return null;
+ return cube.autoMergeCubeSegments();
}
public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment newSegment) throws IOException {
@@ -765,7 +695,7 @@ public class CubeManager implements IRealizationProvider {
logger.warn("For cube " + cube + ", segment " + newSegment + " state should be NEW but is READY");
}
- List<CubeSegment> tobe = calculateToBeSegments(cube, newSegment);
+ List<CubeSegment> tobe = cube.calculateToBeSegments(newSegment);
if (tobe.contains(newSegment) == false)
throw new IllegalStateException("For cube " + cube + ", segment " + newSegment + " is expected but not in the tobe " + tobe);
@@ -786,104 +716,17 @@ public class CubeManager implements IRealizationProvider {
}
public void validateNewSegments(CubeInstance cube, CubeSegment newSegments) {
- List<CubeSegment> tobe = calculateToBeSegments(cube, newSegments);
+ List<CubeSegment> tobe = cube.calculateToBeSegments(newSegments);
List<CubeSegment> newList = Arrays.asList(newSegments);
if (tobe.containsAll(newList) == false) {
throw new IllegalStateException("For cube " + cube + ", the new segments " + newList + " do not fit in its current " + cube.getSegments() + "; the resulted tobe is " + tobe);
}
}
- /**
- * Smartly figure out the TOBE segments once all new segments are built.
- * - Ensures no gap, no overlap
- * - Favors new segments over the old
- * - Favors big segments over the small
- */
- private List<CubeSegment> calculateToBeSegments(CubeInstance cube, CubeSegment newSegments) {
-
- List<CubeSegment> tobe = Lists.newArrayList(cube.getSegments());
- if (newSegments != null && !tobe.contains(newSegments)) {
- tobe.add(newSegments);
- }
- if (tobe.size() == 0)
- return tobe;
-
- // sort by source offset
- Collections.sort(tobe);
-
- CubeSegment firstSeg = tobe.get(0);
- firstSeg.validate();
-
- for (int i = 0, j = 1; j < tobe.size();) {
- CubeSegment is = tobe.get(i);
- CubeSegment js = tobe.get(j);
- js.validate();
-
- // check i is either ready or new
- if (!isNew(is) && !isReady(is)) {
- tobe.remove(i);
- continue;
- }
-
- // check j is either ready or new
- if (!isNew(js) && !isReady(js)) {
- tobe.remove(j);
- continue;
- }
-
- if (is.getSourceOffsetStart() == js.getSourceOffsetStart()) {
- // if i, j competes
- if (isReady(is) && isReady(js) || isNew(is) && isNew(js)) {
- // if both new or ready, favor the bigger segment
- if (is.getSourceOffsetEnd() <= js.getSourceOffsetEnd()) {
- tobe.remove(i);
- } else {
- tobe.remove(j);
- }
- continue;
- } else {
- // otherwise, favor the new segment
- if (isNew(is) && is.equals(newSegments)) {
- tobe.remove(j);
- continue;
- } else if (js.equals(newSegments)) {
- tobe.remove(i);
- continue;
- }
- }
- }
-
- // if i, j in sequence
- if (is.getSourceOffsetEnd() <= js.getSourceOffsetStart()) {
- i++;
- j++;
- continue;
- }
-
- // js can be covered by is
- if (is.equals(newSegments)) {
- // seems j not fitting
- tobe.remove(j);
- continue;
- } else {
- i++;
- j++;
- continue;
- }
-
- }
-
- return tobe;
- }
-
private boolean isReady(CubeSegment seg) {
return seg.getStatus() == SegmentStatusEnum.READY;
}
- private boolean isNew(CubeSegment seg) {
- return seg.getStatus() == SegmentStatusEnum.NEW || seg.getStatus() == SegmentStatusEnum.READY_PENDING;
- }
-
private void loadAllCubeInstance() throws IOException {
ResourceStore store = getStore();
List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_RESOURCE_ROOT, ".json");
http://git-wip-us.apache.org/repos/asf/kylin/blob/0a441c3f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index e3fcdcb..9d26927 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -18,7 +18,7 @@
package org.apache.kylin.metadata.model;
-public interface ISegment{
+public interface ISegment {
public String getName();
@@ -29,11 +29,12 @@ public interface ISegment{
public long getSourceOffsetStart();
public long getSourceOffsetEnd();
-
+
public DataModelDesc getModel();
public SegmentStatusEnum getStatus();
public long getLastBuildTime();
+ public boolean isSourceOffsetsOn();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0a441c3f/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index f0a58cb..bc115cc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -18,16 +18,25 @@
package org.apache.kylin.metadata.model;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.apache.kylin.common.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class Segments<T extends ISegment> extends ArrayList<T> {
-
+
private static final long serialVersionUID = 1L;
+ private static final Logger logger = LoggerFactory.getLogger(Segments.class);
+
public static boolean sourceOffsetContains(ISegment a, ISegment b) {
return a.getSourceOffsetStart() <= b.getSourceOffsetStart() && b.getSourceOffsetEnd() <= a.getSourceOffsetEnd();
}
-
+
public static boolean sourceOffsetOverlaps(ISegment a, ISegment b) {
return a.getSourceOffsetStart() < b.getSourceOffsetEnd() && b.getSourceOffsetStart() < a.getSourceOffsetEnd();
}
@@ -44,7 +53,7 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
Segments<T> readySegs = getSegments(SegmentStatusEnum.READY);
long startTime = Long.MAX_VALUE;
- for (T seg : readySegs) {
+ for (ISegment seg : readySegs) {
startTime = Math.min(startTime, seg.getDateRangeStart());
}
@@ -55,7 +64,7 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
Segments<T> readySegs = getSegments(SegmentStatusEnum.READY);
long endTime = Long.MIN_VALUE;
- for (T seg : readySegs) {
+ for (ISegment seg : readySegs) {
endTime = Math.max(endTime, seg.getDateRangeEnd());
}
@@ -87,7 +96,7 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
return latest;
}
- public Segments getSegments(SegmentStatusEnum status) {
+ public Segments<T> getSegments(SegmentStatusEnum status) {
Segments<T> result = new Segments<>();
for (T segment : this) {
@@ -107,7 +116,7 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
return null;
}
- public Segments getBuildingSegments() {
+ public Segments<T> getBuildingSegments() {
Segments<T> buildingSegments = new Segments();
if (null != this) {
for (T segment : this) {
@@ -142,9 +151,194 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
return result;
}
+ public Pair<Long, Long> autoMergeCubeSegments(boolean needAutoMerge, String cubeName, long[] timeRanges) throws IOException {
+ if (!needAutoMerge) {
+ logger.debug("Cube " + cubeName + " doesn't need auto merge");
+ return null;
+ }
+
+ int buildingSize = getBuildingSegments().size();
+ if (buildingSize > 0) {
+ logger.debug("Cube " + cubeName + " has " + buildingSize + " building segments");
+ }
+
+ Segments<T> readySegs = getSegments(SegmentStatusEnum.READY);
+
+ Segments mergingSegs = new Segments();
+ if (buildingSize > 0) {
+
+ for (ISegment building : getBuildingSegments()) {
+ // exclude those under-merging segs
+ for (ISegment ready : readySegs) {
+ if (ready.getSourceOffsetStart() >= building.getSourceOffsetStart() && ready.getSourceOffsetEnd() <= building.getSourceOffsetEnd()) {
+ mergingSegs.add(ready);
+ }
+ }
+ }
+ }
+
+ // exclude those already under merging segments
+ readySegs.removeAll(mergingSegs);
+
+ Arrays.sort(timeRanges);
+
+ for (int i = timeRanges.length - 1; i >= 0; i--) {
+ long toMergeRange = timeRanges[i];
+
+ for (int s = 0; s < readySegs.size(); s++) {
+ ISegment seg = readySegs.get(s);
+ Pair<T, T> p = findMergeOffsetsByDateRange(readySegs.getSubList(s, readySegs.size()), //
+ seg.getDateRangeStart(), seg.getDateRangeStart() + toMergeRange, toMergeRange);
+ if (p != null && p.getSecond().getDateRangeEnd() - p.getFirst().getDateRangeStart() >= toMergeRange)
+ return Pair.newPair(p.getFirst().getSourceOffsetStart(), p.getSecond().getSourceOffsetEnd());
+ }
+ }
+
+ return null;
+ }
+
+ public Pair<T, T> findMergeOffsetsByDateRange(Segments<T> segments, long startDate, long endDate, long skipSegDateRangeCap) {
+ // must be offset cube
+ Segments result = new Segments();
+ for (ISegment seg : segments) {
+
+ // include if date range overlaps
+ if (startDate < seg.getDateRangeEnd() && seg.getDateRangeStart() < endDate) {
+
+ // reject too big segment
+ if (seg.getDateRangeEnd() - seg.getDateRangeStart() > skipSegDateRangeCap)
+ break;
+
+ // reject holes
+ if (result.size() > 0 && result.getLast().getSourceOffsetEnd() != seg.getSourceOffsetStart())
+ break;
+
+ result.add(seg);
+ }
+ }
+
+ if (result.size() <= 1)
+ return null;
+ else
+ return (Pair<T, T>) Pair.newPair(result.getFirst(), result.getLast());
+ }
+
+ /**
+ * Smartly figure out the TOBE segments once all new segments are built.
+ * - Ensures no gap, no overlap
+ * - Favors new segments over the old
+ * - Favors big segments over the small
+ */
+ public Segments calculateToBeSegments(ISegment newSegment, boolean isPartitioned) {
+
+ Segments tobe = (Segments) this.clone();
+ if (newSegment != null && !tobe.contains(newSegment)) {
+ tobe.add(newSegment);
+ }
+ if (tobe.size() == 0)
+ return tobe;
+
+ // sort by source offset
+ Collections.sort(tobe);
+
+ ISegment firstSeg = tobe.getFirst();
+ validate(firstSeg, isPartitioned);
+
+ for (int i = 0, j = 1; j < tobe.size();) {
+ ISegment is = (ISegment) tobe.get(i);
+ ISegment js = (ISegment) tobe.get(j);
+ validate(js, isPartitioned);
+
+ // check i is either ready or new
+ if (!isNew(is) && !isReady(is)) {
+ tobe.remove(i);
+ continue;
+ }
+
+ // check j is either ready or new
+ if (!isNew(js) && !isReady(js)) {
+ tobe.remove(j);
+ continue;
+ }
+
+ if (is.getSourceOffsetStart() == js.getSourceOffsetStart()) {
+ // if i, j competes
+ if (isReady(is) && isReady(js) || isNew(is) && isNew(js)) {
+ // if both new or ready, favor the bigger segment
+ if (is.getSourceOffsetEnd() <= js.getSourceOffsetEnd()) {
+ tobe.remove(i);
+ } else {
+ tobe.remove(j);
+ }
+ continue;
+ } else {
+ // otherwise, favor the new segment
+ if (isNew(is) && is.equals(newSegment)) {
+ tobe.remove(j);
+ continue;
+ } else if (js.equals(newSegment)) {
+ tobe.remove(i);
+ continue;
+ }
+ }
+ }
+
+ // if i, j in sequence
+ if (is.getSourceOffsetEnd() <= js.getSourceOffsetStart()) {
+ i++;
+ j++;
+ continue;
+ }
+
+ // js can be covered by is
+ if (is.equals(newSegment)) {
+ // seems j not fitting
+ tobe.remove(j);
+ continue;
+ } else {
+ i++;
+ j++;
+ continue;
+ }
+
+ }
+
+ return tobe;
+ }
+
+ private void validate(ISegment seg, boolean isPartitioned) {
+ if (isPartitioned) {
+ if (!seg.isSourceOffsetsOn() && seg.getDateRangeStart() >= seg.getDateRangeEnd())
+ throw new IllegalStateException("Invalid segment, dateRangeStart(" + seg.getDateRangeStart() + ") must be smaller than dateRangeEnd(" + seg.getDateRangeEnd() + ") in segment " + seg);
+ if (seg.isSourceOffsetsOn() && seg.getSourceOffsetStart() >= seg.getSourceOffsetEnd())
+ throw new IllegalStateException("Invalid segment, sourceOffsetStart(" + seg.getSourceOffsetStart() + ") must be smaller than sourceOffsetEnd(" + seg.getSourceOffsetEnd() + ") in segment " + seg);
+ }
+ }
+
+ private boolean isReady(ISegment seg) {
+ return seg.getStatus() == SegmentStatusEnum.READY;
+ }
+
+ private boolean isNew(ISegment seg) {
+ return seg.getStatus() == SegmentStatusEnum.NEW || seg.getStatus() == SegmentStatusEnum.READY_PENDING;
+ }
+
private T getLast() {
assert this.size() != 0;
return this.get(this.size() - 1);
}
+ private T getFirst() {
+ assert this.size() != 0;
+ return this.get(0);
+ }
+
+ private Segments<T> getSubList(int from, int to) {
+ Segments<T> result = new Segments<>();
+ for (T seg : this.subList(from, to)) {
+ result.add(seg);
+ }
+ return result;
+ }
+
}
\ No newline at end of file
[31/50] [abbrv] kylin git commit: minor, refine MassInTupleFilter
Posted by li...@apache.org.
minor, refine MassInTupleFilter
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a03eec88
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a03eec88
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a03eec88
Branch: refs/heads/master-cdh5.7
Commit: a03eec88f3b4e60d1f1d6bd2fd5922a97aec23f5
Parents: fe9efa8
Author: Roger Shi <ro...@hotmail.com>
Authored: Sun Dec 4 16:28:47 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Dec 4 21:28:17 2016 +0800
----------------------------------------------------------------------
.../metadata/filter/UDF/MassInTupleFilter.java | 34 +++++++++++++++++---
1 file changed, 30 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a03eec88/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 29c5550..e4e311e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -19,6 +19,7 @@
package org.apache.kylin.metadata.filter.UDF;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Collection;
import org.apache.kylin.common.KylinConfig;
@@ -40,7 +41,6 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
public class MassInTupleFilter extends FunctionTupleFilter {
-
public static final Logger logger = LoggerFactory.getLogger(MassInTupleFilter.class);
public static MassInValueProviderFactory VALUE_PROVIDER_FACTORY = null;
@@ -56,6 +56,16 @@ public class MassInTupleFilter extends FunctionTupleFilter {
super(Lists.<TupleFilter> newArrayList(), TupleFilter.FilterOperatorEnum.MASSIN);
}
+ public MassInTupleFilter(MassInTupleFilter filter) {
+ super(new ArrayList<TupleFilter>(filter.children), filter.operator);
+ this.valueProvider = filter.getValueProvider();
+ this.column = filter.getColumn();
+ this.filterTableName = filter.getFilterTableName();
+ this.filterTableResourceIdentifier = filter.getFilterTableResourceIdentifier();
+ this.filterTableType = filter.getFilterTableType();
+ this.reverse = filter.isReverse();
+ }
+
@Override
public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
Preconditions.checkNotNull(tuple);
@@ -132,6 +142,7 @@ public class MassInTupleFilter extends FunctionTupleFilter {
BytesUtil.writeUTFString(filterTableName, buffer);
BytesUtil.writeUTFString(filterTableResourceIdentifier, buffer);
BytesUtil.writeUTFString(filterTableType.toString(), buffer);
+ BytesUtil.writeUTFString(String.valueOf(reverse), buffer);
}
@Override
@@ -139,6 +150,7 @@ public class MassInTupleFilter extends FunctionTupleFilter {
filterTableName = BytesUtil.readUTFString(buffer);
filterTableResourceIdentifier = BytesUtil.readUTFString(buffer);
filterTableType = Functions.FilterTableType.valueOf(BytesUtil.readUTFString(buffer));
+ reverse = Boolean.valueOf(BytesUtil.readUTFString(buffer));
}
public static boolean containsMassInTupleFilter(TupleFilter filter) {
@@ -166,8 +178,22 @@ public class MassInTupleFilter extends FunctionTupleFilter {
@Override
protected Object clone() throws CloneNotSupportedException {
- MassInTupleFilter result = new MassInTupleFilter();
- result.setReverse(this.isReverse());
- return result;
+ return new MassInTupleFilter(this);
+ }
+
+ public MassInValueProvider getValueProvider() {
+ return valueProvider;
+ }
+
+ public String getFilterTableName() {
+ return filterTableName;
+ }
+
+ public String getFilterTableResourceIdentifier() {
+ return filterTableResourceIdentifier;
+ }
+
+ public Functions.FilterTableType getFilterTableType() {
+ return filterTableType;
}
}
[09/50] [abbrv] kylin git commit: KYLIN-2195 could run
get-properties.sh in source code dir
Posted by li...@apache.org.
KYLIN-2195 could run get-properties.sh in source code dir
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8506e4d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8506e4d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8506e4d4
Branch: refs/heads/master-cdh5.7
Commit: 8506e4d4f75fbbda518997cac97e3333229fa5a0
Parents: bda8787
Author: lidongsjtu <li...@apache.org>
Authored: Tue Nov 29 18:24:07 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Nov 29 18:24:07 2016 +0800
----------------------------------------------------------------------
build/bin/get-properties.sh | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8506e4d4/build/bin/get-properties.sh
----------------------------------------------------------------------
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 170442d..0fa4f6f 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -19,13 +19,21 @@
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
if [ $# != 1 ]
then
echo 'invalid input'
exit -1
fi
-job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
-tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
+if [[ $CI_MODE == 'true' ]]; then
+ cd $dir
+ job_jar=$(ls ../../assembly/target/kylin-*-job.jar)
+ tool_jar=$(ls ../../tool/target/kylin-tool-*.jar|grep -v assembly)
+else
+ job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
+ tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
+fi
+
result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
echo "$result"
[06/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample
data, and rename 'kylin_metadata' to 'kylin_default_instance'
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
----------------------------------------------------------------------
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv b/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
new file mode 100644
index 0000000..253afbf
--- /dev/null
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv
@@ -0,0 +1,10000 @@
+10000000,1,4,FR,N/A
+10000001,0,1,DE,N/A
+10000002,0,1,IT,N/A
+10000003,4,2,DE,N/A
+10000004,2,3,JP,N/A
+10000005,0,1,CN,N/A
+10000006,2,3,JP,N/A
+10000007,0,2,GB,N/A
+10000008,5,3,US,N/A
+10000009,1,3,US,N/A
+10000010,1,1,IT,N/A
+10000011,2,4,FR,N/A
+10000012,3,1,CN,N/A
+10000013,1,1,FR,N/A
+10000014,5,3,CN,N/A
+10000015,3,5,IT,N/A
+10000016,4,5,JP,N/A
+10000017,3,4,US,N/A
+10000018,0,5,CN,N/A
+10000019,4,4,GB,N/A
+10000020,4,4,IT,N/A
+10000021,1,2,FR,N/A
+10000022,2,2,DE,N/A
+10000023,0,1,GB,N/A
+10000024,3,4,IT,N/A
+10000025,4,4,JP,N/A
+10000026,5,4,IT,N/A
+10000027,2,4,FR,N/A
+10000028,2,2,GB,N/A
+10000029,1,1,GB,N/A
+10000030,5,5,CN,N/A
+10000031,4,2,CN,N/A
+10000032,1,3,CN,N/A
+10000033,2,1,JP,N/A
+10000034,2,1,JP,N/A
+10000035,4,2,US,N/A
+10000036,5,4,DE,N/A
+10000037,2,4,DE,N/A
+10000038,1,3,US,N/A
+10000039,3,5,US,N/A
+10000040,2,3,GB,N/A
+10000041,1,5,IT,N/A
+10000042,5,5,CN,N/A
+10000043,5,1,IT,N/A
+10000044,3,4,IT,N/A
+10000045,1,3,US,N/A
+10000046,3,3,JP,N/A
+10000047,2,2,FR,N/A
+10000048,5,4,JP,N/A
+10000049,3,1,JP,N/A
+10000050,3,5,JP,N/A
+10000051,5,1,US,N/A
+10000052,3,5,DE,N/A
+10000053,1,3,JP,N/A
+10000054,5,1,GB,N/A
+10000055,2,1,GB,N/A
+10000056,2,3,CN,N/A
+10000057,1,5,US,N/A
+10000058,2,2,CN,N/A
+10000059,2,4,FR,N/A
+10000060,0,3,FR,N/A
+10000061,0,5,FR,N/A
+10000062,1,1,US,N/A
+10000063,4,5,DE,N/A
+10000064,2,5,GB,N/A
+10000065,0,3,GB,N/A
+10000066,1,1,IT,N/A
+10000067,0,2,GB,N/A
+10000068,4,1,IT,N/A
+10000069,0,4,FR,N/A
+10000070,3,4,CN,N/A
+10000071,3,5,GB,N/A
+10000072,4,3,CN,N/A
+10000073,0,4,IT,N/A
+10000074,5,3,GB,N/A
+10000075,0,4,JP,N/A
+10000076,2,5,JP,N/A
+10000077,0,3,CN,N/A
+10000078,4,3,GB,N/A
+10000079,4,5,US,N/A
+10000080,4,2,JP,N/A
+10000081,5,2,CN,N/A
+10000082,4,1,JP,N/A
+10000083,0,4,GB,N/A
+10000084,1,4,DE,N/A
+10000085,4,4,FR,N/A
+10000086,2,5,JP,N/A
+10000087,0,1,IT,N/A
+10000088,0,2,FR,N/A
+10000089,3,2,DE,N/A
+10000090,5,2,IT,N/A
+10000091,0,3,GB,N/A
+10000092,1,4,DE,N/A
+10000093,2,1,IT,N/A
+10000094,4,3,JP,N/A
+10000095,1,1,DE,N/A
+10000096,0,5,JP,N/A
+10000097,3,3,CN,N/A
+10000098,5,3,JP,N/A
+10000099,3,1,GB,N/A
+10000100,5,1,CN,N/A
+10000101,2,3,JP,N/A
+10000102,5,1,IT,N/A
+10000103,5,2,GB,N/A
+10000104,3,5,GB,N/A
+10000105,3,1,DE,N/A
+10000106,3,5,GB,N/A
+10000107,5,3,DE,N/A
+10000108,4,1,CN,N/A
+10000109,4,3,GB,N/A
+10000110,4,1,FR,N/A
+10000111,3,5,FR,N/A
+10000112,1,1,GB,N/A
+10000113,5,4,FR,N/A
+10000114,2,2,DE,N/A
+10000115,4,3,GB,N/A
+10000116,0,3,FR,N/A
+10000117,1,4,CN,N/A
+10000118,4,4,IT,N/A
+10000119,0,2,CN,N/A
+10000120,5,2,DE,N/A
+10000121,0,2,GB,N/A
+10000122,4,3,CN,N/A
+10000123,1,4,FR,N/A
+10000124,0,2,CN,N/A
+10000125,4,5,US,N/A
+10000126,0,4,GB,N/A
+10000127,4,3,CN,N/A
+10000128,4,3,GB,N/A
+10000129,1,4,JP,N/A
+10000130,2,2,JP,N/A
+10000131,0,3,IT,N/A
+10000132,4,1,GB,N/A
+10000133,1,5,US,N/A
+10000134,3,3,DE,N/A
+10000135,3,2,IT,N/A
+10000136,2,2,JP,N/A
+10000137,0,3,CN,N/A
+10000138,3,1,FR,N/A
+10000139,2,3,FR,N/A
+10000140,3,1,IT,N/A
+10000141,0,3,US,N/A
+10000142,3,4,FR,N/A
+10000143,4,5,DE,N/A
+10000144,3,3,JP,N/A
+10000145,5,5,DE,N/A
+10000146,2,5,FR,N/A
+10000147,0,5,IT,N/A
+10000148,2,5,US,N/A
+10000149,0,3,CN,N/A
+10000150,2,2,GB,N/A
+10000151,5,5,CN,N/A
+10000152,2,2,IT,N/A
+10000153,3,2,FR,N/A
+10000154,5,4,FR,N/A
+10000155,3,5,FR,N/A
+10000156,5,2,GB,N/A
+10000157,1,2,IT,N/A
+10000158,3,3,JP,N/A
+10000159,5,1,IT,N/A
+10000160,3,4,CN,N/A
+10000161,2,5,IT,N/A
+10000162,4,5,CN,N/A
+10000163,3,5,FR,N/A
+10000164,1,2,FR,N/A
+10000165,0,3,GB,N/A
+10000166,5,4,CN,N/A
+10000167,4,5,IT,N/A
+10000168,3,2,IT,N/A
+10000169,3,2,DE,N/A
+10000170,2,2,DE,N/A
+10000171,3,1,IT,N/A
+10000172,3,1,IT,N/A
+10000173,3,5,CN,N/A
+10000174,5,2,US,N/A
+10000175,2,3,FR,N/A
+10000176,5,5,JP,N/A
+10000177,1,4,DE,N/A
+10000178,4,5,GB,N/A
+10000179,5,3,FR,N/A
+10000180,2,3,GB,N/A
+10000181,0,3,IT,N/A
+10000182,3,2,JP,N/A
+10000183,5,5,IT,N/A
+10000184,4,1,IT,N/A
+10000185,3,2,IT,N/A
+10000186,3,2,CN,N/A
+10000187,2,5,IT,N/A
+10000188,3,2,JP,N/A
+10000189,4,4,IT,N/A
+10000190,4,3,GB,N/A
+10000191,3,4,JP,N/A
+10000192,2,2,FR,N/A
+10000193,5,1,FR,N/A
+10000194,5,2,US,N/A
+10000195,2,5,FR,N/A
+10000196,1,2,FR,N/A
+10000197,5,3,FR,N/A
+10000198,1,4,GB,N/A
+10000199,1,3,FR,N/A
+10000200,4,5,IT,N/A
+10000201,3,1,GB,N/A
+10000202,0,3,GB,N/A
+10000203,4,3,DE,N/A
+10000204,4,2,GB,N/A
+10000205,4,2,DE,N/A
+10000206,5,2,GB,N/A
+10000207,1,5,DE,N/A
+10000208,4,5,GB,N/A
+10000209,0,3,FR,N/A
+10000210,1,5,FR,N/A
+10000211,3,2,DE,N/A
+10000212,5,1,JP,N/A
+10000213,0,4,CN,N/A
+10000214,3,1,JP,N/A
+10000215,5,1,CN,N/A
+10000216,4,2,FR,N/A
+10000217,4,5,DE,N/A
+10000218,5,3,DE,N/A
+10000219,3,1,CN,N/A
+10000220,4,5,IT,N/A
+10000221,3,1,GB,N/A
+10000222,2,3,CN,N/A
+10000223,3,3,GB,N/A
+10000224,2,4,JP,N/A
+10000225,2,5,CN,N/A
+10000226,4,4,IT,N/A
+10000227,4,1,GB,N/A
+10000228,2,2,IT,N/A
+10000229,5,3,DE,N/A
+10000230,2,1,GB,N/A
+10000231,4,4,DE,N/A
+10000232,2,5,JP,N/A
+10000233,4,4,DE,N/A
+10000234,0,1,GB,N/A
+10000235,5,4,FR,N/A
+10000236,4,4,DE,N/A
+10000237,0,3,DE,N/A
+10000238,4,2,CN,N/A
+10000239,2,4,CN,N/A
+10000240,0,2,GB,N/A
+10000241,2,4,DE,N/A
+10000242,2,1,US,N/A
+10000243,1,1,US,N/A
+10000244,5,4,DE,N/A
+10000245,5,3,GB,N/A
+10000246,2,1,FR,N/A
+10000247,1,1,JP,N/A
+10000248,2,5,JP,N/A
+10000249,2,1,GB,N/A
+10000250,4,4,FR,N/A
+10000251,3,2,IT,N/A
+10000252,3,2,DE,N/A
+10000253,0,3,US,N/A
+10000254,3,4,JP,N/A
+10000255,0,4,CN,N/A
+10000256,4,3,FR,N/A
+10000257,1,2,IT,N/A
+10000258,2,2,JP,N/A
+10000259,5,2,US,N/A
+10000260,3,3,CN,N/A
+10000261,5,1,GB,N/A
+10000262,3,2,DE,N/A
+10000263,0,2,JP,N/A
+10000264,4,2,CN,N/A
+10000265,2,3,CN,N/A
+10000266,3,4,US,N/A
+10000267,2,2,IT,N/A
+10000268,5,5,FR,N/A
+10000269,5,3,DE,N/A
+10000270,0,5,FR,N/A
+10000271,5,3,DE,N/A
+10000272,2,2,GB,N/A
+10000273,5,5,DE,N/A
+10000274,3,5,FR,N/A
+10000275,1,5,IT,N/A
+10000276,5,1,CN,N/A
+10000277,5,1,IT,N/A
+10000278,4,1,JP,N/A
+10000279,1,3,US,N/A
+10000280,1,3,IT,N/A
+10000281,3,5,US,N/A
+10000282,0,5,IT,N/A
+10000283,4,4,CN,N/A
+10000284,0,1,US,N/A
+10000285,1,5,DE,N/A
+10000286,3,3,US,N/A
+10000287,5,1,JP,N/A
+10000288,5,1,GB,N/A
+10000289,2,4,FR,N/A
+10000290,2,5,GB,N/A
+10000291,5,3,IT,N/A
+10000292,4,4,US,N/A
+10000293,3,3,GB,N/A
+10000294,2,2,GB,N/A
+10000295,4,5,DE,N/A
+10000296,0,5,DE,N/A
+10000297,1,2,JP,N/A
+10000298,1,5,GB,N/A
+10000299,4,1,JP,N/A
+10000300,1,2,DE,N/A
+10000301,4,4,IT,N/A
+10000302,5,5,DE,N/A
+10000303,5,3,JP,N/A
+10000304,5,5,GB,N/A
+10000305,0,1,GB,N/A
+10000306,4,1,CN,N/A
+10000307,4,5,US,N/A
+10000308,5,5,DE,N/A
+10000309,0,5,IT,N/A
+10000310,5,1,GB,N/A
+10000311,4,5,FR,N/A
+10000312,2,1,CN,N/A
+10000313,1,4,FR,N/A
+10000314,1,2,DE,N/A
+10000315,5,3,CN,N/A
+10000316,0,3,CN,N/A
+10000317,3,2,DE,N/A
+10000318,0,1,CN,N/A
+10000319,1,5,JP,N/A
+10000320,2,1,FR,N/A
+10000321,0,5,IT,N/A
+10000322,0,4,DE,N/A
+10000323,4,1,DE,N/A
+10000324,5,5,GB,N/A
+10000325,0,4,GB,N/A
+10000326,5,2,IT,N/A
+10000327,3,5,IT,N/A
+10000328,2,1,JP,N/A
+10000329,5,3,CN,N/A
+10000330,5,2,CN,N/A
+10000331,3,1,CN,N/A
+10000332,0,3,GB,N/A
+10000333,4,5,CN,N/A
+10000334,0,3,IT,N/A
+10000335,2,1,GB,N/A
+10000336,4,3,DE,N/A
+10000337,1,2,FR,N/A
+10000338,2,4,IT,N/A
+10000339,5,4,FR,N/A
+10000340,0,2,GB,N/A
+10000341,2,3,IT,N/A
+10000342,0,3,DE,N/A
+10000343,4,1,US,N/A
+10000344,4,1,CN,N/A
+10000345,5,1,FR,N/A
+10000346,2,4,DE,N/A
+10000347,4,1,DE,N/A
+10000348,5,5,DE,N/A
+10000349,2,2,US,N/A
+10000350,1,5,DE,N/A
+10000351,2,4,US,N/A
+10000352,0,4,US,N/A
+10000353,2,5,GB,N/A
+10000354,0,3,DE,N/A
+10000355,2,1,FR,N/A
+10000356,1,5,JP,N/A
+10000357,3,2,GB,N/A
+10000358,3,4,FR,N/A
+10000359,0,4,IT,N/A
+10000360,3,1,CN,N/A
+10000361,5,4,GB,N/A
+10000362,5,3,IT,N/A
+10000363,1,1,IT,N/A
+10000364,2,1,DE,N/A
+10000365,1,1,FR,N/A
+10000366,2,2,IT,N/A
+10000367,2,5,FR,N/A
+10000368,5,5,CN,N/A
+10000369,4,3,DE,N/A
+10000370,0,2,US,N/A
+10000371,2,2,DE,N/A
+10000372,0,5,JP,N/A
+10000373,0,5,DE,N/A
+10000374,3,2,CN,N/A
+10000375,4,2,IT,N/A
+10000376,3,5,GB,N/A
+10000377,5,4,GB,N/A
+10000378,3,4,US,N/A
+10000379,3,4,FR,N/A
+10000380,3,1,IT,N/A
+10000381,0,2,FR,N/A
+10000382,1,1,JP,N/A
+10000383,1,5,CN,N/A
+10000384,3,4,US,N/A
+10000385,3,2,DE,N/A
+10000386,3,1,JP,N/A
+10000387,2,2,US,N/A
+10000388,4,2,US,N/A
+10000389,0,2,IT,N/A
+10000390,5,5,CN,N/A
+10000391,2,3,JP,N/A
+10000392,3,3,IT,N/A
+10000393,0,1,DE,N/A
+10000394,1,2,IT,N/A
+10000395,4,4,FR,N/A
+10000396,3,1,CN,N/A
+10000397,0,5,CN,N/A
+10000398,4,5,IT,N/A
+10000399,2,5,DE,N/A
+10000400,2,5,JP,N/A
+10000401,0,1,FR,N/A
+10000402,4,5,FR,N/A
+10000403,4,3,JP,N/A
+10000404,4,1,JP,N/A
+10000405,2,5,US,N/A
+10000406,0,2,US,N/A
+10000407,0,3,GB,N/A
+10000408,5,3,FR,N/A
+10000409,5,1,US,N/A
+10000410,1,2,CN,N/A
+10000411,3,3,CN,N/A
+10000412,2,2,JP,N/A
+10000413,4,4,DE,N/A
+10000414,0,5,GB,N/A
+10000415,3,1,GB,N/A
+10000416,2,5,CN,N/A
+10000417,5,1,GB,N/A
+10000418,4,3,IT,N/A
+10000419,2,3,US,N/A
+10000420,1,2,DE,N/A
+10000421,0,5,GB,N/A
+10000422,1,3,US,N/A
+10000423,0,3,DE,N/A
+10000424,4,3,JP,N/A
+10000425,4,4,DE,N/A
+10000426,3,2,CN,N/A
+10000427,4,2,FR,N/A
+10000428,2,2,JP,N/A
+10000429,0,2,IT,N/A
+10000430,3,4,DE,N/A
+10000431,0,3,CN,N/A
+10000432,2,5,FR,N/A
+10000433,5,2,IT,N/A
+10000434,1,3,JP,N/A
+10000435,0,5,JP,N/A
+10000436,2,4,FR,N/A
+10000437,0,5,IT,N/A
+10000438,0,3,JP,N/A
+10000439,1,1,CN,N/A
+10000440,5,5,GB,N/A
+10000441,4,3,JP,N/A
+10000442,5,3,US,N/A
+10000443,4,1,DE,N/A
+10000444,1,3,GB,N/A
+10000445,1,2,IT,N/A
+10000446,2,2,JP,N/A
+10000447,1,5,FR,N/A
+10000448,3,3,IT,N/A
+10000449,1,1,US,N/A
+10000450,3,3,JP,N/A
+10000451,5,5,CN,N/A
+10000452,5,4,CN,N/A
+10000453,2,4,US,N/A
+10000454,3,5,JP,N/A
+10000455,0,3,GB,N/A
+10000456,3,4,CN,N/A
+10000457,5,2,IT,N/A
+10000458,1,1,DE,N/A
+10000459,4,5,IT,N/A
+10000460,3,3,US,N/A
+10000461,1,2,US,N/A
+10000462,2,5,US,N/A
+10000463,1,4,GB,N/A
+10000464,3,1,FR,N/A
+10000465,5,3,GB,N/A
+10000466,3,1,IT,N/A
+10000467,0,1,FR,N/A
+10000468,4,5,GB,N/A
+10000469,3,2,US,N/A
+10000470,4,3,CN,N/A
+10000471,4,2,JP,N/A
+10000472,4,5,DE,N/A
+10000473,4,3,FR,N/A
+10000474,1,1,GB,N/A
+10000475,1,4,CN,N/A
+10000476,2,3,US,N/A
+10000477,0,1,DE,N/A
+10000478,3,2,IT,N/A
+10000479,4,4,GB,N/A
+10000480,5,4,GB,N/A
+10000481,4,3,US,N/A
+10000482,0,5,CN,N/A
+10000483,0,1,CN,N/A
+10000484,4,3,FR,N/A
+10000485,5,2,FR,N/A
+10000486,5,3,CN,N/A
+10000487,5,3,GB,N/A
+10000488,4,3,JP,N/A
+10000489,1,4,FR,N/A
+10000490,1,1,US,N/A
+10000491,2,3,DE,N/A
+10000492,1,2,GB,N/A
+10000493,4,5,GB,N/A
+10000494,5,3,CN,N/A
+10000495,1,2,CN,N/A
+10000496,3,3,US,N/A
+10000497,3,1,IT,N/A
+10000498,4,4,US,N/A
+10000499,4,5,JP,N/A
+10000500,4,3,DE,N/A
+10000501,4,3,FR,N/A
+10000502,2,1,CN,N/A
+10000503,5,5,CN,N/A
+10000504,3,4,US,N/A
+10000505,0,2,GB,N/A
+10000506,2,4,FR,N/A
+10000507,1,2,IT,N/A
+10000508,4,2,IT,N/A
+10000509,2,5,IT,N/A
+10000510,0,1,DE,N/A
+10000511,5,5,IT,N/A
+10000512,2,5,FR,N/A
+10000513,4,2,IT,N/A
+10000514,4,2,JP,N/A
+10000515,4,5,IT,N/A
+10000516,2,4,JP,N/A
+10000517,4,2,CN,N/A
+10000518,2,3,JP,N/A
+10000519,2,3,GB,N/A
+10000520,4,2,JP,N/A
+10000521,0,4,DE,N/A
+10000522,2,2,US,N/A
+10000523,0,2,FR,N/A
+10000524,1,2,GB,N/A
+10000525,4,5,IT,N/A
+10000526,0,3,US,N/A
+10000527,4,1,FR,N/A
+10000528,0,5,DE,N/A
+10000529,3,5,IT,N/A
+10000530,4,1,IT,N/A
+10000531,0,1,GB,N/A
+10000532,0,2,DE,N/A
+10000533,2,4,GB,N/A
+10000534,4,4,FR,N/A
+10000535,3,2,JP,N/A
+10000536,3,2,GB,N/A
+10000537,2,4,US,N/A
+10000538,0,3,DE,N/A
+10000539,5,2,IT,N/A
+10000540,2,1,IT,N/A
+10000541,3,2,US,N/A
+10000542,2,3,JP,N/A
+10000543,4,2,DE,N/A
+10000544,1,3,JP,N/A
+10000545,5,3,FR,N/A
+10000546,4,2,IT,N/A
+10000547,4,1,CN,N/A
+10000548,3,2,US,N/A
+10000549,5,2,DE,N/A
+10000550,1,4,US,N/A
+10000551,2,1,FR,N/A
+10000552,3,5,GB,N/A
+10000553,5,3,FR,N/A
+10000554,3,1,JP,N/A
+10000555,2,4,JP,N/A
+10000556,3,4,FR,N/A
+10000557,3,2,CN,N/A
+10000558,4,1,US,N/A
+10000559,4,2,JP,N/A
+10000560,0,1,GB,N/A
+10000561,5,3,CN,N/A
+10000562,5,2,IT,N/A
+10000563,2,3,DE,N/A
+10000564,4,5,US,N/A
+10000565,1,3,US,N/A
+10000566,1,3,IT,N/A
+10000567,3,5,CN,N/A
+10000568,3,1,FR,N/A
+10000569,0,2,DE,N/A
+10000570,0,2,JP,N/A
+10000571,4,5,DE,N/A
+10000572,0,3,IT,N/A
+10000573,5,4,JP,N/A
+10000574,5,3,GB,N/A
+10000575,2,4,GB,N/A
+10000576,4,4,JP,N/A
+10000577,4,3,GB,N/A
+10000578,2,2,FR,N/A
+10000579,0,4,DE,N/A
+10000580,1,5,JP,N/A
+10000581,2,3,JP,N/A
+10000582,2,3,JP,N/A
+10000583,4,4,US,N/A
+10000584,2,3,JP,N/A
+10000585,0,2,JP,N/A
+10000586,1,4,CN,N/A
+10000587,1,3,DE,N/A
+10000588,2,1,JP,N/A
+10000589,2,1,DE,N/A
+10000590,0,1,US,N/A
+10000591,2,4,JP,N/A
+10000592,4,5,GB,N/A
+10000593,0,4,FR,N/A
+10000594,1,4,US,N/A
+10000595,5,1,FR,N/A
+10000596,0,1,JP,N/A
+10000597,0,3,GB,N/A
+10000598,5,3,IT,N/A
+10000599,2,5,JP,N/A
+10000600,4,1,JP,N/A
+10000601,2,3,GB,N/A
+10000602,3,3,IT,N/A
+10000603,4,4,CN,N/A
+10000604,3,1,FR,N/A
+10000605,5,2,GB,N/A
+10000606,4,1,DE,N/A
+10000607,4,4,US,N/A
+10000608,4,2,GB,N/A
+10000609,3,4,IT,N/A
+10000610,5,1,US,N/A
+10000611,5,5,GB,N/A
+10000612,0,2,CN,N/A
+10000613,0,1,GB,N/A
+10000614,5,1,IT,N/A
+10000615,0,1,FR,N/A
+10000616,3,3,DE,N/A
+10000617,3,3,FR,N/A
+10000618,2,4,IT,N/A
+10000619,3,5,FR,N/A
+10000620,1,5,FR,N/A
+10000621,2,2,DE,N/A
+10000622,5,3,CN,N/A
+10000623,0,5,DE,N/A
+10000624,0,3,IT,N/A
+10000625,0,3,IT,N/A
+10000626,2,1,IT,N/A
+10000627,1,1,CN,N/A
+10000628,2,1,IT,N/A
+10000629,2,5,GB,N/A
+10000630,2,2,GB,N/A
+10000631,5,1,IT,N/A
+10000632,0,1,FR,N/A
+10000633,2,3,US,N/A
+10000634,5,4,GB,N/A
+10000635,4,5,US,N/A
+10000636,0,4,GB,N/A
+10000637,4,1,US,N/A
+10000638,5,4,JP,N/A
+10000639,4,3,IT,N/A
+10000640,3,3,US,N/A
+10000641,2,4,IT,N/A
+10000642,4,1,FR,N/A
+10000643,5,3,FR,N/A
+10000644,4,5,IT,N/A
+10000645,4,1,IT,N/A
+10000646,3,2,US,N/A
+10000647,1,5,JP,N/A
+10000648,5,2,DE,N/A
+10000649,3,2,FR,N/A
+10000650,4,1,IT,N/A
+10000651,1,2,JP,N/A
+10000652,0,3,DE,N/A
+10000653,3,5,IT,N/A
+10000654,2,2,DE,N/A
+10000655,3,3,US,N/A
+10000656,2,4,GB,N/A
+10000657,1,5,FR,N/A
+10000658,2,4,JP,N/A
+10000659,4,1,JP,N/A
+10000660,4,3,CN,N/A
+10000661,3,4,IT,N/A
+10000662,1,2,DE,N/A
+10000663,4,3,US,N/A
+10000664,3,5,GB,N/A
+10000665,1,1,CN,N/A
+10000666,2,4,DE,N/A
+10000667,2,2,IT,N/A
+10000668,3,2,JP,N/A
+10000669,5,5,CN,N/A
+10000670,0,5,IT,N/A
+10000671,4,1,GB,N/A
+10000672,2,5,FR,N/A
+10000673,1,2,DE,N/A
+10000674,1,4,JP,N/A
+10000675,0,3,FR,N/A
+10000676,1,1,FR,N/A
+10000677,5,5,IT,N/A
+10000678,1,1,CN,N/A
+10000679,2,2,IT,N/A
+10000680,3,4,DE,N/A
+10000681,4,1,US,N/A
+10000682,1,4,GB,N/A
+10000683,4,4,CN,N/A
+10000684,3,1,GB,N/A
+10000685,2,1,DE,N/A
+10000686,0,3,IT,N/A
+10000687,4,1,IT,N/A
+10000688,5,1,US,N/A
+10000689,2,3,IT,N/A
+10000690,5,2,GB,N/A
+10000691,2,3,JP,N/A
+10000692,4,2,JP,N/A
+10000693,0,3,GB,N/A
+10000694,1,4,IT,N/A
+10000695,2,2,JP,N/A
+10000696,2,1,US,N/A
+10000697,2,5,FR,N/A
+10000698,5,4,IT,N/A
+10000699,0,3,FR,N/A
+10000700,2,4,CN,N/A
+10000701,3,1,DE,N/A
+10000702,0,5,GB,N/A
+10000703,3,1,CN,N/A
+10000704,4,5,JP,N/A
+10000705,2,4,JP,N/A
+10000706,4,4,JP,N/A
+10000707,1,5,DE,N/A
+10000708,1,3,FR,N/A
+10000709,2,3,US,N/A
+10000710,5,1,GB,N/A
+10000711,3,5,IT,N/A
+10000712,0,1,IT,N/A
+10000713,1,4,DE,N/A
+10000714,0,2,GB,N/A
+10000715,3,4,US,N/A
+10000716,0,3,FR,N/A
+10000717,0,5,IT,N/A
+10000718,5,2,GB,N/A
+10000719,2,1,FR,N/A
+10000720,1,4,GB,N/A
+10000721,0,2,US,N/A
+10000722,4,1,US,N/A
+10000723,4,1,DE,N/A
+10000724,4,4,US,N/A
+10000725,4,4,US,N/A
+10000726,2,3,US,N/A
+10000727,3,2,GB,N/A
+10000728,1,5,GB,N/A
+10000729,1,2,FR,N/A
+10000730,1,5,IT,N/A
+10000731,2,2,US,N/A
+10000732,3,1,IT,N/A
+10000733,4,4,US,N/A
+10000734,5,3,US,N/A
+10000735,3,3,US,N/A
+10000736,0,1,IT,N/A
+10000737,2,3,JP,N/A
+10000738,2,1,CN,N/A
+10000739,4,5,FR,N/A
+10000740,5,5,US,N/A
+10000741,3,3,DE,N/A
+10000742,1,1,IT,N/A
+10000743,5,2,JP,N/A
+10000744,1,2,GB,N/A
+10000745,2,3,DE,N/A
+10000746,0,4,CN,N/A
+10000747,1,2,US,N/A
+10000748,3,5,JP,N/A
+10000749,5,4,FR,N/A
+10000750,3,1,CN,N/A
+10000751,1,2,GB,N/A
+10000752,3,3,JP,N/A
+10000753,0,4,JP,N/A
+10000754,1,1,JP,N/A
+10000755,4,3,US,N/A
+10000756,4,5,IT,N/A
+10000757,4,1,US,N/A
+10000758,2,3,CN,N/A
+10000759,3,5,GB,N/A
+10000760,5,2,GB,N/A
+10000761,2,3,US,N/A
+10000762,2,5,JP,N/A
+10000763,0,3,US,N/A
+10000764,2,2,IT,N/A
+10000765,1,2,CN,N/A
+10000766,0,1,IT,N/A
+10000767,1,5,FR,N/A
+10000768,3,3,GB,N/A
+10000769,3,3,US,N/A
+10000770,0,5,US,N/A
+10000771,2,1,GB,N/A
+10000772,0,1,US,N/A
+10000773,4,3,CN,N/A
+10000774,3,3,DE,N/A
+10000775,4,5,CN,N/A
+10000776,1,2,IT,N/A
+10000777,3,3,DE,N/A
+10000778,0,1,IT,N/A
+10000779,0,2,CN,N/A
+10000780,0,2,GB,N/A
+10000781,4,2,DE,N/A
+10000782,1,4,FR,N/A
+10000783,4,4,JP,N/A
+10000784,5,3,CN,N/A
+10000785,3,5,FR,N/A
+10000786,1,1,CN,N/A
+10000787,0,5,GB,N/A
+10000788,2,4,IT,N/A
+10000789,0,5,CN,N/A
+10000790,0,3,GB,N/A
+10000791,1,2,JP,N/A
+10000792,0,2,GB,N/A
+10000793,5,3,US,N/A
+10000794,1,5,FR,N/A
+10000795,3,4,CN,N/A
+10000796,1,2,JP,N/A
+10000797,3,1,JP,N/A
+10000798,1,4,US,N/A
+10000799,2,3,DE,N/A
+10000800,2,5,JP,N/A
+10000801,5,4,CN,N/A
+10000802,0,4,JP,N/A
+10000803,0,1,CN,N/A
+10000804,4,4,FR,N/A
+10000805,0,3,CN,N/A
+10000806,0,1,US,N/A
+10000807,5,5,US,N/A
+10000808,3,5,DE,N/A
+10000809,1,3,US,N/A
+10000810,5,4,IT,N/A
+10000811,3,2,DE,N/A
+10000812,5,5,GB,N/A
+10000813,2,5,CN,N/A
+10000814,4,5,JP,N/A
+10000815,5,4,IT,N/A
+10000816,4,2,CN,N/A
+10000817,0,4,DE,N/A
+10000818,1,1,JP,N/A
+10000819,0,1,CN,N/A
+10000820,5,2,JP,N/A
+10000821,4,5,JP,N/A
+10000822,0,2,JP,N/A
+10000823,0,2,GB,N/A
+10000824,3,3,US,N/A
+10000825,4,1,CN,N/A
+10000826,5,4,JP,N/A
+10000827,3,2,US,N/A
+10000828,4,4,DE,N/A
+10000829,2,5,DE,N/A
+10000830,4,3,JP,N/A
+10000831,2,2,DE,N/A
+10000832,2,2,US,N/A
+10000833,5,2,FR,N/A
+10000834,3,4,FR,N/A
+10000835,5,2,JP,N/A
+10000836,4,4,IT,N/A
+10000837,5,1,IT,N/A
+10000838,1,3,IT,N/A
+10000839,2,3,GB,N/A
+10000840,4,2,GB,N/A
+10000841,5,5,JP,N/A
+10000842,5,3,GB,N/A
+10000843,0,2,GB,N/A
+10000844,3,2,JP,N/A
+10000845,2,4,GB,N/A
+10000846,2,2,DE,N/A
+10000847,0,1,CN,N/A
+10000848,2,1,US,N/A
+10000849,0,1,DE,N/A
+10000850,5,5,US,N/A
+10000851,4,1,US,N/A
+10000852,1,2,DE,N/A
+10000853,5,4,FR,N/A
+10000854,5,4,US,N/A
+10000855,2,2,IT,N/A
+10000856,2,3,JP,N/A
+10000857,2,5,DE,N/A
+10000858,0,4,IT,N/A
+10000859,2,5,CN,N/A
+10000860,0,5,DE,N/A
+10000861,5,5,US,N/A
+10000862,5,5,CN,N/A
+10000863,4,4,GB,N/A
+10000864,1,3,FR,N/A
+10000865,4,2,US,N/A
+10000866,5,1,US,N/A
+10000867,0,1,DE,N/A
+10000868,4,2,CN,N/A
+10000869,5,5,JP,N/A
+10000870,4,4,DE,N/A
+10000871,2,5,FR,N/A
+10000872,5,5,CN,N/A
+10000873,0,1,IT,N/A
+10000874,2,1,CN,N/A
+10000875,3,4,CN,N/A
+10000876,3,1,IT,N/A
+10000877,5,3,GB,N/A
+10000878,5,4,CN,N/A
+10000879,1,4,JP,N/A
+10000880,0,5,FR,N/A
+10000881,2,3,JP,N/A
+10000882,5,3,GB,N/A
+10000883,5,4,CN,N/A
+10000884,3,3,CN,N/A
+10000885,5,1,DE,N/A
+10000886,5,3,FR,N/A
+10000887,4,5,CN,N/A
+10000888,4,3,JP,N/A
+10000889,5,3,IT,N/A
+10000890,2,4,JP,N/A
+10000891,4,5,DE,N/A
+10000892,0,4,JP,N/A
+10000893,1,2,US,N/A
+10000894,2,1,FR,N/A
+10000895,1,5,DE,N/A
+10000896,5,1,US,N/A
+10000897,2,5,FR,N/A
+10000898,4,1,IT,N/A
+10000899,3,2,JP,N/A
+10000900,5,5,DE,N/A
+10000901,0,5,CN,N/A
+10000902,1,4,DE,N/A
+10000903,4,5,IT,N/A
+10000904,4,3,IT,N/A
+10000905,1,1,GB,N/A
+10000906,0,5,GB,N/A
+10000907,0,4,GB,N/A
+10000908,1,4,GB,N/A
+10000909,5,1,IT,N/A
+10000910,5,5,IT,N/A
+10000911,0,2,CN,N/A
+10000912,5,1,CN,N/A
+10000913,0,2,IT,N/A
+10000914,2,1,CN,N/A
+10000915,4,4,GB,N/A
+10000916,0,4,IT,N/A
+10000917,4,2,FR,N/A
+10000918,0,1,US,N/A
+10000919,1,1,GB,N/A
+10000920,0,4,IT,N/A
+10000921,0,3,CN,N/A
+10000922,4,2,DE,N/A
+10000923,1,4,IT,N/A
+10000924,2,1,GB,N/A
+10000925,0,4,US,N/A
+10000926,4,5,CN,N/A
+10000927,2,5,FR,N/A
+10000928,5,5,CN,N/A
+10000929,4,1,FR,N/A
+10000930,0,2,JP,N/A
+10000931,1,2,IT,N/A
+10000932,3,4,JP,N/A
+10000933,1,2,JP,N/A
+10000934,2,5,IT,N/A
+10000935,4,4,CN,N/A
+10000936,2,5,US,N/A
+10000937,3,3,IT,N/A
+10000938,2,4,GB,N/A
+10000939,4,5,IT,N/A
+10000940,4,3,GB,N/A
+10000941,4,2,FR,N/A
+10000942,1,1,JP,N/A
+10000943,0,3,FR,N/A
+10000944,0,3,DE,N/A
+10000945,0,1,DE,N/A
+10000946,3,2,GB,N/A
+10000947,5,5,US,N/A
+10000948,4,2,CN,N/A
+10000949,0,4,DE,N/A
+10000950,3,2,FR,N/A
+10000951,4,3,JP,N/A
+10000952,5,2,GB,N/A
+10000953,3,1,JP,N/A
+10000954,5,3,DE,N/A
+10000955,2,3,DE,N/A
+10000956,1,2,JP,N/A
+10000957,2,1,JP,N/A
+10000958,2,1,CN,N/A
+10000959,4,1,US,N/A
+10000960,5,3,US,N/A
+10000961,3,1,FR,N/A
+10000962,5,5,CN,N/A
+10000963,2,2,GB,N/A
+10000964,3,3,IT,N/A
+10000965,5,1,JP,N/A
+10000966,2,3,US,N/A
+10000967,5,1,US,N/A
+10000968,1,1,IT,N/A
+10000969,5,5,US,N/A
+10000970,5,3,IT,N/A
+10000971,5,3,FR,N/A
+10000972,1,4,US,N/A
+10000973,5,3,CN,N/A
+10000974,4,5,GB,N/A
+10000975,5,4,JP,N/A
+10000976,0,2,GB,N/A
+10000977,0,1,FR,N/A
+10000978,2,3,GB,N/A
+10000979,3,5,JP,N/A
+10000980,2,2,GB,N/A
+10000981,0,3,FR,N/A
+10000982,1,3,US,N/A
+10000983,0,1,FR,N/A
+10000984,2,1,CN,N/A
+10000985,0,4,CN,N/A
+10000986,5,4,FR,N/A
+10000987,5,2,IT,N/A
+10000988,5,1,DE,N/A
+10000989,2,4,IT,N/A
+10000990,5,1,CN,N/A
+10000991,2,2,DE,N/A
+10000992,5,2,IT,N/A
+10000993,0,3,DE,N/A
+10000994,1,4,US,N/A
+10000995,0,4,IT,N/A
+10000996,1,3,US,N/A
+10000997,3,2,FR,N/A
+10000998,2,5,CN,N/A
+10000999,4,1,US,N/A
+10001000,2,0,GB,N/A
+10001001,4,0,DE,N/A
+10001002,3,0,IT,N/A
+10001003,2,0,US,N/A
+10001004,1,0,DE,N/A
+10001005,3,0,CN,N/A
+10001006,2,0,CN,N/A
+10001007,2,0,DE,N/A
+10001008,1,0,DE,N/A
+10001009,2,0,DE,N/A
+10001010,5,0,IT,N/A
+10001011,4,0,GB,N/A
+10001012,0,0,JP,N/A
+10001013,0,0,US,N/A
+10001014,5,0,JP,N/A
+10001015,0,0,CN,N/A
+10001016,2,0,DE,N/A
+10001017,1,0,CN,N/A
+10001018,3,0,IT,N/A
+10001019,5,0,GB,N/A
+10001020,1,0,GB,N/A
+10001021,0,0,IT,N/A
+10001022,3,0,CN,N/A
+10001023,1,0,CN,N/A
+10001024,2,0,JP,N/A
+10001025,3,0,CN,N/A
+10001026,0,0,IT,N/A
+10001027,3,0,GB,N/A
+10001028,5,0,CN,N/A
+10001029,5,0,CN,N/A
+10001030,2,0,FR,N/A
+10001031,3,0,FR,N/A
+10001032,2,0,FR,N/A
+10001033,3,0,JP,N/A
+10001034,5,0,CN,N/A
+10001035,4,0,DE,N/A
+10001036,2,0,IT,N/A
+10001037,0,0,FR,N/A
+10001038,4,0,DE,N/A
+10001039,0,0,DE,N/A
+10001040,3,0,US,N/A
+10001041,0,0,CN,N/A
+10001042,3,0,GB,N/A
+10001043,4,0,DE,N/A
+10001044,5,0,DE,N/A
+10001045,4,0,GB,N/A
+10001046,1,0,FR,N/A
+10001047,1,0,US,N/A
+10001048,3,0,CN,N/A
+10001049,1,0,IT,N/A
+10001050,2,0,GB,N/A
+10001051,1,0,GB,N/A
+10001052,1,0,CN,N/A
+10001053,0,0,FR,N/A
+10001054,4,0,JP,N/A
+10001055,4,0,CN,N/A
+10001056,1,0,CN,N/A
+10001057,3,0,US,N/A
+10001058,2,0,DE,N/A
+10001059,1,0,FR,N/A
+10001060,3,0,CN,N/A
+10001061,0,0,CN,N/A
+10001062,1,0,GB,N/A
+10001063,4,0,GB,N/A
+10001064,1,0,FR,N/A
+10001065,4,0,JP,N/A
+10001066,3,0,JP,N/A
+10001067,3,0,GB,N/A
+10001068,5,0,DE,N/A
+10001069,2,0,FR,N/A
+10001070,3,0,IT,N/A
+10001071,5,0,GB,N/A
+10001072,0,0,JP,N/A
+10001073,1,0,IT,N/A
+10001074,3,0,DE,N/A
+10001075,5,0,IT,N/A
+10001076,1,0,IT,N/A
+10001077,4,0,IT,N/A
+10001078,5,0,GB,N/A
+10001079,5,0,US,N/A
+10001080,4,0,CN,N/A
+10001081,3,0,FR,N/A
+10001082,0,0,US,N/A
+10001083,4,0,CN,N/A
+10001084,2,0,GB,N/A
+10001085,1,0,FR,N/A
+10001086,3,0,DE,N/A
+10001087,4,0,IT,N/A
+10001088,0,0,IT,N/A
+10001089,4,0,GB,N/A
+10001090,5,0,IT,N/A
+10001091,2,0,JP,N/A
+10001092,1,0,FR,N/A
+10001093,1,0,US,N/A
+10001094,3,0,IT,N/A
+10001095,1,0,JP,N/A
+10001096,1,0,FR,N/A
+10001097,1,0,FR,N/A
+10001098,3,0,US,N/A
+10001099,0,0,JP,N/A
+10001100,5,0,DE,N/A
+10001101,4,0,JP,N/A
+10001102,5,0,CN,N/A
+10001103,5,0,GB,N/A
+10001104,0,0,DE,N/A
+10001105,4,0,IT,N/A
+10001106,3,0,DE,N/A
+10001107,0,0,DE,N/A
+10001108,4,0,CN,N/A
+10001109,4,0,DE,N/A
+10001110,3,0,GB,N/A
+10001111,4,0,DE,N/A
+10001112,1,0,US,N/A
+10001113,4,0,US,N/A
+10001114,0,0,FR,N/A
+10001115,5,0,CN,N/A
+10001116,0,0,CN,N/A
+10001117,4,0,IT,N/A
+10001118,2,0,GB,N/A
+10001119,4,0,US,N/A
+10001120,3,0,GB,N/A
+10001121,2,0,DE,N/A
+10001122,3,0,US,N/A
+10001123,3,0,US,N/A
+10001124,1,0,JP,N/A
+10001125,5,0,IT,N/A
+10001126,2,0,JP,N/A
+10001127,1,0,JP,N/A
+10001128,2,0,JP,N/A
+10001129,2,0,IT,N/A
+10001130,4,0,GB,N/A
+10001131,3,0,FR,N/A
+10001132,4,0,US,N/A
+10001133,4,0,JP,N/A
+10001134,4,0,GB,N/A
+10001135,1,0,FR,N/A
+10001136,4,0,DE,N/A
+10001137,1,0,CN,N/A
+10001138,1,0,US,N/A
+10001139,2,0,JP,N/A
+10001140,2,0,US,N/A
+10001141,1,0,CN,N/A
+10001142,0,0,DE,N/A
+10001143,5,0,US,N/A
+10001144,4,0,GB,N/A
+10001145,0,0,US,N/A
+10001146,0,0,DE,N/A
+10001147,2,0,IT,N/A
+10001148,2,0,IT,N/A
+10001149,3,0,DE,N/A
+10001150,3,0,GB,N/A
+10001151,4,0,JP,N/A
+10001152,5,0,JP,N/A
+10001153,4,0,JP,N/A
+10001154,1,0,FR,N/A
+10001155,3,0,IT,N/A
+10001156,2,0,US,N/A
+10001157,0,0,GB,N/A
+10001158,1,0,JP,N/A
+10001159,0,0,DE,N/A
+10001160,0,0,DE,N/A
+10001161,0,0,CN,N/A
+10001162,0,0,IT,N/A
+10001163,1,0,DE,N/A
+10001164,0,0,US,N/A
+10001165,4,0,DE,N/A
+10001166,4,0,IT,N/A
+10001167,5,0,JP,N/A
+10001168,4,0,US,N/A
+10001169,4,0,JP,N/A
+10001170,0,0,FR,N/A
+10001171,3,0,IT,N/A
+10001172,4,0,CN,N/A
+10001173,3,0,FR,N/A
+10001174,0,0,US,N/A
+10001175,2,0,JP,N/A
+10001176,3,0,US,N/A
+10001177,3,0,CN,N/A
+10001178,0,0,FR,N/A
+10001179,4,0,CN,N/A
+10001180,5,0,GB,N/A
+10001181,0,0,GB,N/A
+10001182,1,0,FR,N/A
+10001183,2,0,CN,N/A
+10001184,2,0,IT,N/A
+10001185,0,0,GB,N/A
+10001186,3,0,JP,N/A
+10001187,2,0,GB,N/A
+10001188,0,0,US,N/A
+10001189,2,0,US,N/A
+10001190,2,0,CN,N/A
+10001191,2,0,IT,N/A
+10001192,4,0,CN,N/A
+10001193,2,0,GB,N/A
+10001194,0,0,IT,N/A
+10001195,2,0,FR,N/A
+10001196,4,0,FR,N/A
+10001197,0,0,US,N/A
+10001198,2,0,US,N/A
+10001199,1,0,DE,N/A
+10001200,0,0,CN,N/A
+10001201,5,0,CN,N/A
+10001202,2,0,DE,N/A
+10001203,0,0,CN,N/A
+10001204,0,0,DE,N/A
+10001205,0,0,GB,N/A
+10001206,1,0,IT,N/A
+10001207,3,0,DE,N/A
+10001208,1,0,JP,N/A
+10001209,3,0,GB,N/A
+10001210,5,0,GB,N/A
+10001211,5,0,JP,N/A
+10001212,1,0,US,N/A
+10001213,5,0,DE,N/A
+10001214,2,0,JP,N/A
+10001215,0,0,CN,N/A
+10001216,4,0,CN,N/A
+10001217,3,0,US,N/A
+10001218,3,0,CN,N/A
+10001219,1,0,CN,N/A
+10001220,1,0,CN,N/A
+10001221,0,0,DE,N/A
+10001222,3,0,DE,N/A
+10001223,1,0,GB,N/A
+10001224,4,0,DE,N/A
+10001225,0,0,IT,N/A
+10001226,4,0,GB,N/A
+10001227,4,0,IT,N/A
+10001228,0,0,JP,N/A
+10001229,2,0,FR,N/A
+10001230,4,0,CN,N/A
+10001231,5,0,GB,N/A
+10001232,3,0,DE,N/A
+10001233,1,0,FR,N/A
+10001234,3,0,JP,N/A
+10001235,2,0,US,N/A
+10001236,0,0,DE,N/A
+10001237,1,0,DE,N/A
+10001238,2,0,IT,N/A
+10001239,4,0,US,N/A
+10001240,0,0,US,N/A
+10001241,0,0,CN,N/A
+10001242,3,0,JP,N/A
+10001243,3,0,JP,N/A
+10001244,4,0,DE,N/A
+10001245,3,0,CN,N/A
+10001246,4,0,GB,N/A
+10001247,5,0,US,N/A
+10001248,2,0,DE,N/A
+10001249,1,0,CN,N/A
+10001250,3,0,CN,N/A
+10001251,1,0,FR,N/A
+10001252,3,0,IT,N/A
+10001253,0,0,CN,N/A
+10001254,3,0,GB,N/A
+10001255,0,0,JP,N/A
+10001256,0,0,GB,N/A
+10001257,3,0,FR,N/A
+10001258,3,0,IT,N/A
+10001259,5,0,DE,N/A
+10001260,5,0,US,N/A
+10001261,3,0,DE,N/A
+10001262,4,0,FR,N/A
+10001263,2,0,IT,N/A
+10001264,2,0,US,N/A
+10001265,0,0,GB,N/A
+10001266,1,0,FR,N/A
+10001267,5,0,JP,N/A
+10001268,3,0,GB,N/A
+10001269,4,0,IT,N/A
+10001270,5,0,JP,N/A
+10001271,0,0,IT,N/A
+10001272,0,0,GB,N/A
+10001273,1,0,FR,N/A
+10001274,0,0,FR,N/A
+10001275,4,0,GB,N/A
+10001276,4,0,DE,N/A
+10001277,4,0,FR,N/A
+10001278,1,0,FR,N/A
+10001279,5,0,DE,N/A
+10001280,0,0,IT,N/A
+10001281,2,0,IT,N/A
+10001282,1,0,FR,N/A
+10001283,1,0,JP,N/A
+10001284,1,0,CN,N/A
+10001285,0,0,CN,N/A
+10001286,2,0,DE,N/A
+10001287,2,0,CN,N/A
+10001288,4,0,FR,N/A
+10001289,5,0,IT,N/A
+10001290,2,0,US,N/A
+10001291,3,0,JP,N/A
+10001292,0,0,GB,N/A
+10001293,5,0,US,N/A
+10001294,3,0,FR,N/A
+10001295,2,0,CN,N/A
+10001296,5,0,JP,N/A
+10001297,2,0,CN,N/A
+10001298,5,0,DE,N/A
+10001299,1,0,DE,N/A
+10001300,3,0,US,N/A
+10001301,5,0,DE,N/A
+10001302,4,0,CN,N/A
+10001303,5,0,DE,N/A
+10001304,2,0,JP,N/A
+10001305,2,0,CN,N/A
+10001306,4,0,FR,N/A
+10001307,0,0,CN,N/A
+10001308,5,0,JP,N/A
+10001309,4,0,DE,N/A
+10001310,5,0,FR,N/A
+10001311,0,0,CN,N/A
+10001312,2,0,CN,N/A
+10001313,4,0,DE,N/A
+10001314,0,0,US,N/A
+10001315,1,0,FR,N/A
+10001316,3,0,FR,N/A
+10001317,0,0,DE,N/A
+10001318,2,0,JP,N/A
+10001319,5,0,CN,N/A
+10001320,4,0,US,N/A
+10001321,0,0,DE,N/A
+10001322,4,0,JP,N/A
+10001323,3,0,DE,N/A
+10001324,0,0,GB,N/A
+10001325,2,0,CN,N/A
+10001326,3,0,IT,N/A
+10001327,3,0,CN,N/A
+10001328,4,0,GB,N/A
+10001329,5,0,FR,N/A
+10001330,5,0,GB,N/A
+10001331,3,0,FR,N/A
+10001332,4,0,US,N/A
+10001333,0,0,DE,N/A
+10001334,5,0,US,N/A
+10001335,3,0,GB,N/A
+10001336,0,0,IT,N/A
+10001337,4,0,DE,N/A
+10001338,3,0,GB,N/A
+10001339,2,0,FR,N/A
+10001340,1,0,IT,N/A
+10001341,2,0,US,N/A
+10001342,1,0,JP,N/A
+10001343,0,0,US,N/A
+10001344,5,0,US,N/A
+10001345,3,0,FR,N/A
+10001346,4,0,JP,N/A
+10001347,3,0,JP,N/A
+10001348,1,0,IT,N/A
+10001349,2,0,CN,N/A
+10001350,5,0,GB,N/A
+10001351,1,0,DE,N/A
+10001352,3,0,US,N/A
+10001353,4,0,US,N/A
+10001354,5,0,US,N/A
+10001355,5,0,DE,N/A
+10001356,5,0,IT,N/A
+10001357,5,0,DE,N/A
+10001358,2,0,FR,N/A
+10001359,4,0,CN,N/A
+10001360,5,0,JP,N/A
+10001361,3,0,US,N/A
+10001362,5,0,IT,N/A
+10001363,3,0,DE,N/A
+10001364,4,0,DE,N/A
+10001365,2,0,FR,N/A
+10001366,0,0,DE,N/A
+10001367,3,0,CN,N/A
+10001368,4,0,US,N/A
+10001369,1,0,IT,N/A
+10001370,0,0,US,N/A
+10001371,2,0,IT,N/A
+10001372,3,0,GB,N/A
+10001373,5,0,IT,N/A
+10001374,5,0,GB,N/A
+10001375,1,0,IT,N/A
+10001376,4,0,CN,N/A
+10001377,2,0,FR,N/A
+10001378,2,0,IT,N/A
+10001379,5,0,CN,N/A
+10001380,1,0,DE,N/A
+10001381,3,0,US,N/A
+10001382,0,0,CN,N/A
+10001383,5,0,IT,N/A
+10001384,1,0,FR,N/A
+10001385,2,0,US,N/A
+10001386,1,0,IT,N/A
+10001387,3,0,GB,N/A
+10001388,1,0,JP,N/A
+10001389,4,0,IT,N/A
+10001390,4,0,US,N/A
+10001391,1,0,GB,N/A
+10001392,1,0,FR,N/A
+10001393,1,0,CN,N/A
+10001394,2,0,GB,N/A
+10001395,3,0,US,N/A
+10001396,1,0,US,N/A
+10001397,1,0,IT,N/A
+10001398,0,0,DE,N/A
+10001399,1,0,GB,N/A
+10001400,4,0,US,N/A
+10001401,4,0,FR,N/A
+10001402,1,0,CN,N/A
+10001403,2,0,CN,N/A
+10001404,5,0,IT,N/A
+10001405,5,0,CN,N/A
+10001406,5,0,US,N/A
+10001407,2,0,JP,N/A
+10001408,3,0,IT,N/A
+10001409,5,0,FR,N/A
+10001410,4,0,GB,N/A
+10001411,1,0,CN,N/A
+10001412,5,0,CN,N/A
+10001413,3,0,JP,N/A
+10001414,3,0,JP,N/A
+10001415,5,0,IT,N/A
+10001416,5,0,US,N/A
+10001417,3,0,CN,N/A
+10001418,2,0,FR,N/A
+10001419,5,0,GB,N/A
+10001420,2,0,FR,N/A
+10001421,3,0,DE,N/A
+10001422,3,0,DE,N/A
+10001423,4,0,GB,N/A
+10001424,5,0,DE,N/A
+10001425,5,0,IT,N/A
+10001426,0,0,FR,N/A
+10001427,4,0,JP,N/A
+10001428,2,0,JP,N/A
+10001429,1,0,JP,N/A
+10001430,1,0,JP,N/A
+10001431,1,0,JP,N/A
+10001432,2,0,GB,N/A
+10001433,2,0,FR,N/A
+10001434,4,0,FR,N/A
+10001435,1,0,JP,N/A
+10001436,3,0,GB,N/A
+10001437,0,0,JP,N/A
+10001438,4,0,GB,N/A
+10001439,3,0,JP,N/A
+10001440,4,0,DE,N/A
+10001441,2,0,IT,N/A
+10001442,1,0,CN,N/A
+10001443,3,0,FR,N/A
+10001444,0,0,FR,N/A
+10001445,0,0,FR,N/A
+10001446,0,0,US,N/A
+10001447,5,0,IT,N/A
+10001448,3,0,FR,N/A
+10001449,2,0,US,N/A
+10001450,4,0,DE,N/A
+10001451,3,0,IT,N/A
+10001452,2,0,IT,N/A
+10001453,0,0,IT,N/A
+10001454,3,0,US,N/A
+10001455,1,0,DE,N/A
+10001456,0,0,GB,N/A
+10001457,5,0,FR,N/A
+10001458,1,0,JP,N/A
+10001459,4,0,GB,N/A
+10001460,1,0,GB,N/A
+10001461,0,0,GB,N/A
+10001462,3,0,CN,N/A
+10001463,1,0,GB,N/A
+10001464,3,0,CN,N/A
+10001465,0,0,DE,N/A
+10001466,2,0,FR,N/A
+10001467,3,0,JP,N/A
+10001468,0,0,FR,N/A
+10001469,0,0,JP,N/A
+10001470,4,0,GB,N/A
+10001471,4,0,IT,N/A
+10001472,3,0,JP,N/A
+10001473,1,0,JP,N/A
+10001474,0,0,FR,N/A
+10001475,0,0,FR,N/A
+10001476,2,0,IT,N/A
+10001477,5,0,DE,N/A
+10001478,4,0,GB,N/A
+10001479,3,0,FR,N/A
+10001480,5,0,US,N/A
+10001481,4,0,JP,N/A
+10001482,4,0,FR,N/A
+10001483,0,0,IT,N/A
+10001484,0,0,DE,N/A
+10001485,1,0,IT,N/A
+10001486,1,0,US,N/A
+10001487,4,0,FR,N/A
+10001488,1,0,GB,N/A
+10001489,2,0,CN,N/A
+10001490,1,0,FR,N/A
+10001491,4,0,IT,N/A
+10001492,0,0,IT,N/A
+10001493,0,0,DE,N/A
+10001494,1,0,DE,N/A
+10001495,1,0,DE,N/A
+10001496,5,0,IT,N/A
+10001497,4,0,GB,N/A
+10001498,3,0,US,N/A
+10001499,0,0,CN,N/A
+10001500,0,0,IT,N/A
+10001501,0,0,DE,N/A
+10001502,3,0,CN,N/A
+10001503,2,0,GB,N/A
+10001504,3,0,FR,N/A
+10001505,5,0,JP,N/A
+10001506,0,0,GB,N/A
+10001507,0,0,JP,N/A
+10001508,0,0,CN,N/A
+10001509,4,0,CN,N/A
+10001510,0,0,FR,N/A
+10001511,1,0,US,N/A
+10001512,1,0,DE,N/A
+10001513,1,0,IT,N/A
+10001514,0,0,IT,N/A
+10001515,1,0,US,N/A
+10001516,2,0,JP,N/A
+10001517,1,0,GB,N/A
+10001518,4,0,IT,N/A
+10001519,1,0,FR,N/A
+10001520,0,0,JP,N/A
+10001521,4,0,DE,N/A
+10001522,4,0,DE,N/A
+10001523,1,0,JP,N/A
+10001524,2,0,IT,N/A
+10001525,2,0,FR,N/A
+10001526,3,0,IT,N/A
+10001527,0,0,GB,N/A
+10001528,4,0,FR,N/A
+10001529,2,0,US,N/A
+10001530,5,0,US,N/A
+10001531,2,0,IT,N/A
+10001532,3,0,US,N/A
+10001533,1,0,CN,N/A
+10001534,1,0,DE,N/A
+10001535,3,0,CN,N/A
+10001536,1,0,US,N/A
+10001537,0,0,GB,N/A
+10001538,5,0,IT,N/A
+10001539,5,0,US,N/A
+10001540,4,0,DE,N/A
+10001541,3,0,IT,N/A
+10001542,0,0,FR,N/A
+10001543,2,0,GB,N/A
+10001544,5,0,IT,N/A
+10001545,2,0,GB,N/A
+10001546,0,0,DE,N/A
+10001547,5,0,CN,N/A
+10001548,2,0,JP,N/A
+10001549,0,0,JP,N/A
+10001550,5,0,IT,N/A
+10001551,1,0,CN,N/A
+10001552,2,0,JP,N/A
+10001553,1,0,JP,N/A
+10001554,1,0,FR,N/A
+10001555,4,0,DE,N/A
+10001556,5,0,GB,N/A
+10001557,4,0,IT,N/A
+10001558,1,0,FR,N/A
+10001559,4,0,IT,N/A
+10001560,1,0,JP,N/A
+10001561,0,0,FR,N/A
+10001562,0,0,DE,N/A
+10001563,3,0,FR,N/A
+10001564,1,0,CN,N/A
+10001565,3,0,US,N/A
+10001566,0,0,US,N/A
+10001567,3,0,IT,N/A
+10001568,3,0,JP,N/A
+10001569,2,0,DE,N/A
+10001570,5,0,FR,N/A
+10001571,0,0,GB,N/A
+10001572,3,0,JP,N/A
+10001573,2,0,DE,N/A
+10001574,4,0,DE,N/A
+10001575,1,0,CN,N/A
+10001576,0,0,IT,N/A
+10001577,0,0,JP,N/A
+10001578,0,0,GB,N/A
+10001579,2,0,IT,N/A
+10001580,0,0,FR,N/A
+10001581,2,0,IT,N/A
+10001582,2,0,FR,N/A
+10001583,0,0,IT,N/A
+10001584,3,0,DE,N/A
+10001585,5,0,US,N/A
+10001586,4,0,US,N/A
+10001587,1,0,IT,N/A
+10001588,1,0,CN,N/A
+10001589,5,0,IT,N/A
+10001590,3,0,US,N/A
+10001591,3,0,DE,N/A
+10001592,0,0,DE,N/A
+10001593,4,0,IT,N/A
+10001594,0,0,CN,N/A
+10001595,0,0,IT,N/A
+10001596,0,0,US,N/A
+10001597,4,0,CN,N/A
+10001598,5,0,US,N/A
+10001599,2,0,JP,N/A
+10001600,3,0,FR,N/A
+10001601,2,0,CN,N/A
+10001602,0,0,IT,N/A
+10001603,1,0,DE,N/A
+10001604,2,0,US,N/A
+10001605,1,0,DE,N/A
+10001606,2,0,CN,N/A
+10001607,3,0,US,N/A
+10001608,4,0,GB,N/A
+10001609,3,0,CN,N/A
+10001610,0,0,IT,N/A
+10001611,5,0,DE,N/A
+10001612,4,0,DE,N/A
+10001613,0,0,DE,N/A
+10001614,4,0,JP,N/A
+10001615,2,0,GB,N/A
+10001616,2,0,CN,N/A
+10001617,2,0,DE,N/A
+10001618,5,0,CN,N/A
+10001619,0,0,FR,N/A
+10001620,2,0,IT,N/A
+10001621,3,0,IT,N/A
+10001622,3,0,DE,N/A
+10001623,4,0,DE,N/A
+10001624,3,0,FR,N/A
+10001625,2,0,GB,N/A
+10001626,2,0,JP,N/A
+10001627,1,0,FR,N/A
+10001628,3,0,US,N/A
+10001629,5,0,CN,N/A
+10001630,3,0,DE,N/A
+10001631,2,0,GB,N/A
+10001632,2,0,CN,N/A
+10001633,1,0,DE,N/A
+10001634,4,0,GB,N/A
+10001635,4,0,DE,N/A
+10001636,2,0,CN,N/A
+10001637,4,0,DE,N/A
+10001638,5,0,CN,N/A
+10001639,2,0,US,N/A
+10001640,5,0,US,N/A
+10001641,4,0,US,N/A
+10001642,3,0,JP,N/A
+10001643,0,0,CN,N/A
+10001644,1,0,JP,N/A
+10001645,2,0,FR,N/A
+10001646,3,0,JP,N/A
+10001647,0,0,JP,N/A
+10001648,1,0,FR,N/A
+10001649,5,0,DE,N/A
+10001650,1,0,GB,N/A
+10001651,3,0,DE,N/A
+10001652,0,0,DE,N/A
+10001653,5,0,IT,N/A
+10001654,2,0,IT,N/A
+10001655,5,0,FR,N/A
+10001656,4,0,DE,N/A
+10001657,1,0,FR,N/A
+10001658,1,0,JP,N/A
+10001659,0,0,US,N/A
+10001660,2,0,IT,N/A
+10001661,4,0,FR,N/A
+10001662,5,0,JP,N/A
+10001663,4,0,FR,N/A
+10001664,0,0,GB,N/A
+10001665,2,0,GB,N/A
+10001666,5,0,IT,N/A
+10001667,1,0,DE,N/A
+10001668,3,0,JP,N/A
+10001669,0,0,CN,N/A
+10001670,2,0,US,N/A
+10001671,5,0,IT,N/A
+10001672,3,0,IT,N/A
+10001673,0,0,JP,N/A
+10001674,1,0,IT,N/A
+10001675,0,0,CN,N/A
+10001676,4,0,US,N/A
+10001677,4,0,DE,N/A
+10001678,4,0,JP,N/A
+10001679,0,0,DE,N/A
+10001680,0,0,CN,N/A
+10001681,4,0,DE,N/A
+10001682,2,0,CN,N/A
+10001683,4,0,CN,N/A
+10001684,3,0,FR,N/A
+10001685,2,0,US,N/A
+10001686,2,0,DE,N/A
+10001687,4,0,GB,N/A
+10001688,5,0,IT,N/A
+10001689,5,0,DE,N/A
+10001690,4,0,CN,N/A
+10001691,2,0,JP,N/A
+10001692,5,0,GB,N/A
+10001693,1,0,CN,N/A
+10001694,3,0,IT,N/A
+10001695,0,0,CN,N/A
+10001696,1,0,JP,N/A
+10001697,0,0,US,N/A
+10001698,2,0,JP,N/A
+10001699,2,0,US,N/A
+10001700,1,0,DE,N/A
+10001701,4,0,JP,N/A
+10001702,5,0,JP,N/A
+10001703,0,0,IT,N/A
+10001704,3,0,JP,N/A
+10001705,4,0,JP,N/A
+10001706,1,0,IT,N/A
+10001707,1,0,IT,N/A
+10001708,1,0,GB,N/A
+10001709,0,0,US,N/A
+10001710,1,0,FR,N/A
+10001711,0,0,FR,N/A
+10001712,1,0,DE,N/A
+10001713,4,0,GB,N/A
+10001714,4,0,CN,N/A
+10001715,4,0,GB,N/A
+10001716,4,0,US,N/A
+10001717,3,0,DE,N/A
+10001718,5,0,DE,N/A
+10001719,3,0,IT,N/A
+10001720,3,0,GB,N/A
+10001721,4,0,GB,N/A
+10001722,5,0,US,N/A
+10001723,3,0,FR,N/A
+10001724,3,0,FR,N/A
+10001725,2,0,DE,N/A
+10001726,0,0,JP,N/A
+10001727,4,0,GB,N/A
+10001728,1,0,GB,N/A
+10001729,2,0,CN,N/A
+10001730,1,0,IT,N/A
+10001731,2,0,GB,N/A
+10001732,0,0,JP,N/A
+10001733,3,0,JP,N/A
+10001734,2,0,CN,N/A
+10001735,1,0,JP,N/A
+10001736,3,0,FR,N/A
+10001737,5,0,CN,N/A
+10001738,3,0,FR,N/A
+10001739,0,0,IT,N/A
+10001740,5,0,JP,N/A
+10001741,0,0,GB,N/A
+10001742,4,0,GB,N/A
+10001743,1,0,US,N/A
+10001744,4,0,US,N/A
+10001745,3,0,GB,N/A
+10001746,3,0,DE,N/A
+10001747,3,0,IT,N/A
+10001748,0,0,JP,N/A
+10001749,4,0,DE,N/A
+10001750,2,0,FR,N/A
+10001751,2,0,GB,N/A
+10001752,2,0,IT,N/A
+10001753,5,0,CN,N/A
+10001754,5,0,CN,N/A
+10001755,1,0,IT,N/A
+10001756,0,0,CN,N/A
+10001757,2,0,GB,N/A
+10001758,2,0,IT,N/A
+10001759,3,0,FR,N/A
+10001760,4,0,JP,N/A
+10001761,0,0,IT,N/A
+10001762,0,0,GB,N/A
+10001763,0,0,JP,N/A
+10001764,4,0,JP,N/A
+10001765,3,0,FR,N/A
+10001766,2,0,DE,N/A
+10001767,1,0,GB,N/A
+10001768,5,0,IT,N/A
+10001769,3,0,FR,N/A
+10001770,1,0,DE,N/A
+10001771,0,0,IT,N/A
+10001772,1,0,JP,N/A
+10001773,1,0,CN,N/A
+10001774,1,0,DE,N/A
+10001775,3,0,US,N/A
+10001776,4,0,JP,N/A
+10001777,4,0,JP,N/A
+10001778,5,0,US,N/A
+10001779,5,0,GB,N/A
+10001780,1,0,US,N/A
+10001781,5,0,US,N/A
+10001782,5,0,FR,N/A
+10001783,4,0,CN,N/A
+10001784,2,0,CN,N/A
+10001785,2,0,GB,N/A
+10001786,4,0,DE,N/A
+10001787,2,0,IT,N/A
+10001788,4,0,GB,N/A
+10001789,3,0,JP,N/A
+10001790,1,0,US,N/A
+10001791,3,0,CN,N/A
+10001792,1,0,FR,N/A
+10001793,5,0,US,N/A
+10001794,4,0,US,N/A
+10001795,3,0,GB,N/A
+10001796,3,0,CN,N/A
+10001797,5,0,GB,N/A
+10001798,3,0,DE,N/A
+10001799,4,0,US,N/A
+10001800,3,0,GB,N/A
+10001801,1,0,IT,N/A
+10001802,3,0,CN,N/A
+10001803,2,0,US,N/A
+10001804,5,0,DE,N/A
+10001805,5,0,CN,N/A
+10001806,4,0,GB,N/A
+10001807,5,0,US,N/A
+10001808,1,0,CN,N/A
+10001809,2,0,JP,N/A
+10001810,5,0,JP,N/A
+10001811,1,0,DE,N/A
+10001812,2,0,GB,N/A
+10001813,0,0,FR,N/A
+10001814,0,0,IT,N/A
+10001815,3,0,DE,N/A
+10001816,2,0,FR,N/A
+10001817,5,0,IT,N/A
+10001818,2,0,DE,N/A
+10001819,2,0,GB,N/A
+10001820,3,0,CN,N/A
+10001821,5,0,US,N/A
+10001822,0,0,CN,N/A
+10001823,2,0,JP,N/A
+10001824,0,0,IT,N/A
+10001825,5,0,CN,N/A
+10001826,1,0,IT,N/A
+10001827,1,0,GB,N/A
+10001828,0,0,CN,N/A
+10001829,0,0,GB,N/A
+10001830,1,0,GB,N/A
+10001831,4,0,US,N/A
+10001832,3,0,GB,N/A
+10001833,5,0,DE,N/A
+10001834,1,0,IT,N/A
+10001835,2,0,CN,N/A
+10001836,1,0,JP,N/A
+10001837,0,0,FR,N/A
+10001838,0,0,IT,N/A
+10001839,0,0,GB,N/A
+10001840,1,0,DE,N/A
+10001841,4,0,FR,N/A
+10001842,1,0,JP,N/A
+10001843,1,0,IT,N/A
+10001844,1,0,GB,N/A
+10001845,1,0,US,N/A
+10001846,1,0,IT,N/A
+10001847,5,0,FR,N/A
+10001848,2,0,IT,N/A
+10001849,2,0,GB,N/A
+10001850,4,0,GB,N/A
+10001851,2,0,JP,N/A
+10001852,5,0,DE,N/A
+10001853,5,0,CN,N/A
+10001854,5,0,IT,N/A
+10001855,0,0,DE,N/A
+10001856,1,0,CN,N/A
+10001857,2,0,CN,N/A
+10001858,1,0,FR,N/A
+10001859,3,0,IT,N/A
+10001860,3,0,CN,N/A
+10001861,0,0,GB,N/A
+10001862,0,0,IT,N/A
+10001863,2,0,CN,N/A
+10001864,2,0,US,N/A
+10001865,2,0,CN,N/A
+10001866,4,0,FR,N/A
+10001867,4,0,IT,N/A
+10001868,0,0,DE,N/A
+10001869,3,0,JP,N/A
+10001870,3,0,JP,N/A
+10001871,5,0,GB,N/A
+10001872,4,0,FR,N/A
+10001873,1,0,IT,N/A
+10001874,5,0,US,N/A
+10001875,5,0,IT,N/A
+10001876,4,0,US,N/A
+10001877,0,0,JP,N/A
+10001878,4,0,GB,N/A
+10001879,3,0,US,N/A
+10001880,0,0,FR,N/A
+10001881,4,0,GB,N/A
+10001882,2,0,IT,N/A
+10001883,0,0,CN,N/A
+10001884,1,0,GB,N/A
+10001885,5,0,US,N/A
+10001886,4,0,CN,N/A
+10001887,5,0,IT,N/A
+10001888,1,0,US,N/A
+10001889,1,0,FR,N/A
+10001890,2,0,US,N/A
+10001891,1,0,DE,N/A
+10001892,4,0,CN,N/A
+10001893,0,0,IT,N/A
+10001894,3,0,US,N/A
+10001895,3,0,IT,N/A
+10001896,0,0,DE,N/A
+10001897,4,0,DE,N/A
+10001898,4,0,FR,N/A
+10001899,4,0,US,N/A
+10001900,0,0,CN,N/A
+10001901,2,0,DE,N/A
+10001902,5,0,US,N/A
+10001903,4,0,JP,N/A
+10001904,5,0,CN,N/A
+10001905,1,0,GB,N/A
+10001906,0,0,IT,N/A
+10001907,1,0,US,N/A
+10001908,1,0,JP,N/A
+10001909,4,0,CN,N/A
+10001910,4,0,FR,N/A
+10001911,4,0,CN,N/A
+10001912,4,0,US,N/A
+10001913,5,0,GB,N/A
+10001914,0,0,GB,N/A
+10001915,2,0,JP,N/A
+10001916,3,0,US,N/A
+10001917,1,0,DE,N/A
+10001918,5,0,FR,N/A
+10001919,4,0,FR,N/A
+10001920,4,0,CN,N/A
+10001921,0,0,IT,N/A
+10001922,3,0,GB,N/A
+10001923,5,0,IT,N/A
+10001924,0,0,DE,N/A
+10001925,1,0,DE,N/A
+10001926,4,0,US,N/A
+10001927,3,0,GB,N/A
+10001928,0,0,GB,N/A
+10001929,3,0,GB,N/A
+10001930,1,0,DE,N/A
+10001931,4,0,US,N/A
+10001932,5,0,JP,N/A
+10001933,1,0,GB,N/A
+10001934,4,0,GB,N/A
+10001935,1,0,GB,N/A
+10001936,0,0,US,N/A
+10001937,1,0,DE,N/A
+10001938,5,0,JP,N/A
+10001939,5,0,JP,N/A
+10001940,3,0,CN,N/A
+10001941,3,0,US,N/A
+10001942,5,0,JP,N/A
+10001943,4,0,US,N/A
+10001944,4,0,FR,N/A
+10001945,1,0,US,N/A
+10001946,2,0,US,N/A
+10001947,1,0,GB,N/A
+10001948,5,0,DE,N/A
+10001949,1,0,DE,N/A
+10001950,4,0,US,N/A
+10001951,3,0,US,N/A
+10001952,0,0,IT,N/A
+10001953,5,0,IT,N/A
+10001954,4,0,CN,N/A
+10001955,2,0,GB,N/A
+10001956,3,0,FR,N/A
+10001957,1,0,JP,N/A
+10001958,1,0,CN,N/A
+10001959,4,0,GB,N/A
+10001960,2,0,JP,N/A
+10001961,5,0,DE,N/A
+10001962,5,0,FR,N/A
+10001963,2,0,IT,N/A
+10001964,3,0,JP,N/A
+10001965,2,0,IT,N/A
+10001966,1,0,GB,N/A
+10001967,0,0,GB,N/A
+10001968,3,0,US,N/A
+10001969,0,0,IT,N/A
+10001970,4,0,IT,N/A
+10001971,1,0,DE,N/A
+10001972,3,0,GB,N/A
+10001973,3,0,CN,N/A
+10001974,1,0,IT,N/A
+10001975,1,0,CN,N/A
+10001976,5,0,US,N/A
+10001977,1,0,IT,N/A
+10001978,2,0,GB,N/A
+10001979,5,0,GB,N/A
+10001980,4,0,IT,N/A
+10001981,1,0,CN,N/A
+10001982,4,0,GB,N/A
+10001983,4,0,US,N/A
+10001984,3,0,FR,N/A
+10001985,5,0,DE,N/A
+10001986,1,0,IT,N/A
+10001987,4,0,US,N/A
+10001988,5,0,DE,N/A
+10001989,2,0,DE,N/A
+10001990,0,0,GB,N/A
+10001991,0,0,US,N/A
+10001992,2,0,GB,N/A
+10001993,2,0,IT,N/A
+10001994,1,0,DE,N/A
+10001995,1,0,FR,N/A
+10001996,5,0,CN,N/A
+10001997,1,0,CN,N/A
+10001998,4,0,DE,N/A
+10001999,1,0,DE,N/A
+10002000,3,0,DE,N/A
+10002001,0,0,GB,N/A
+10002002,3,0,CN,N/A
+10002003,4,0,GB,N/A
+10002004,4,0,GB,N/A
+10002005,3,0,JP,N/A
+10002006,0,0,DE,N/A
+10002007,1,0,DE,N/A
+10002008,3,0,IT,N/A
+10002009,0,0,GB,N/A
+10002010,3,0,IT,N/A
+10002011,0,0,FR,N/A
+10002012,2,0,US,N/A
+10002013,1,0,JP,N/A
+10002014,3,0,US,N/A
+10002015,0,0,GB,N/A
+10002016,4,0,DE,N/A
+10002017,0,0,DE,N/A
+10002018,4,0,FR,N/A
+10002019,3,0,GB,N/A
+10002020,2,0,GB,N/A
+10002021,3,0,DE,N/A
+10002022,5,0,FR,N/A
+10002023,4,0,CN,N/A
+10002024,4,0,US,N/A
+10002025,3,0,IT,N/A
+10002026,3,0,DE,N/A
+10002027,2,0,CN,N/A
+10002028,1,0,DE,N/A
+10002029,1,0,US,N/A
+10002030,3,0,DE,N/A
+10002031,5,0,US,N/A
+10002032,2,0,GB,N/A
+10002033,2,0,FR,N/A
+10002034,0,0,CN,N/A
+10002035,5,0,FR,N/A
+10002036,5,0,DE,N/A
+10002037,5,0,US,N/A
+10002038,5,0,US,N/A
+10002039,5,0,GB,N/A
+10002040,4,0,US,N/A
+10002041,4,0,IT,N/A
+10002042,3,0,JP,N/A
+10002043,4,0,CN,N/A
+10002044,3,0,DE,N/A
+10002045,2,0,US,N/A
+10002046,0,0,IT,N/A
+10002047,1,0,US,N/A
+10002048,5,0,DE,N/A
+10002049,2,0,JP,N/A
+10002050,5,0,US,N/A
+10002051,3,0,GB,N/A
+10002052,3,0,IT,N/A
+10002053,0,0,GB,N/A
+10002054,3,0,DE,N/A
+10002055,3,0,JP,N/A
+10002056,4,0,DE,N/A
+10002057,0,0,DE,N/A
+10002058,5,0,CN,N/A
+10002059,0,0,IT,N/A
+10002060,1,0,CN,N/A
+10002061,4,0,CN,N/A
+10002062,3,0,CN,N/A
+10002063,4,0,DE,N/A
+10002064,2,0,JP,N/A
+10002065,1,0,FR,N/A
+10002066,2,0,FR,N/A
+10002067,1,0,US,N/A
+10002068,2,0,JP,N/A
+10002069,5,0,IT,N/A
+10002070,0,0,FR,N/A
+10002071,0,0,GB,N/A
+10002072,3,0,US,N/A
+10002073,5,0,CN,N/A
+10002074,4,0,GB,N/A
+10002075,3,0,FR,N/A
+10002076,3,0,GB,N/A
+10002077,2,0,IT,N/A
+10002078,5,0,US,N/A
+10002079,3,0,JP,N/A
+10002080,4,0,IT,N/A
+10002081,1,0,CN,N/A
+10002082,1,0,IT,N/A
+10002083,4,0,DE,N/A
+10002084,2,0,GB,N/A
+10002085,3,0,FR,N/A
+10002086,0,0,CN,N/A
+10002087,1,0,FR,N/A
+10002088,1,0,CN,N/A
+10002089,5,0,FR,N/A
+10002090,4,0,IT,N/A
+10002091,5,0,JP,N/A
+10002092,4,0,FR,N/A
+10002093,0,0,FR,N/A
+10002094,0,0,IT,N/A
+10002095,4,0,JP,N/A
+10002096,1,0,US,N/A
+10002097,4,0,FR,N/A
+10002098,0,0,FR,N/A
+10002099,4,0,IT,N/A
+10002100,2,0,US,N/A
+10002101,5,0,US,N/A
+10002102,1,0,GB,N/A
+10002103,1,0,IT,N/A
+10002104,4,0,CN,N/A
+10002105,3,0,US,N/A
+10002106,3,0,GB,N/A
+10002107,4,0,JP,N/A
+10002108,1,0,CN,N/A
+10002109,1,0,DE,N/A
+10002110,1,0,GB,N/A
+10002111,3,0,US,N/A
+10002112,5,0,FR,N/A
+10002113,5,0,DE,N/A
+10002114,0,0,IT,N/A
+10002115,1,0,DE,N/A
+10002116,5,0,FR,N/A
+10002117,5,0,US,N/A
+10002118,1,0,DE,N/A
+10002119,2,0,CN,N/A
+10002120,1,0,CN,N/A
+10002121,1,0,CN,N/A
+10002122,3,0,FR,N/A
+10002123,0,0,US,N/A
+10002124,4,0,FR,N/A
+10002125,2,0,FR,N/A
+10002126,2,0,GB,N/A
+10002127,3,0,US,N/A
+10002128,1,0,IT,N/A
+10002129,3,0,DE,N/A
+10002130,5,0,FR,N/A
+10002131,5,0,GB,N/A
+10002132,4,0,GB,N/A
+10002133,2,0,IT,N/A
+10002134,3,0,FR,N/A
+10002135,3,0,DE,N/A
+10002136,3,0,FR,N/A
+10002137,3,0,FR,N/A
+10002138,4,0,JP,N/A
+10002139,5,0,JP,N/A
+10002140,5,0,DE,N/A
+10002141,0,0,IT,N/A
+10002142,1,0,IT,N/A
+10002143,0,0,CN,N/A
+10002144,2,0,JP,N/A
+10002145,1,0,JP,N/A
+10002146,5,0,US,N/A
+10002147,2,0,FR,N/A
+10002148,3,0,IT,N/A
+10002149,2,0,JP,N/A
+10002150,2,0,IT,N/A
+10002151,5,0,CN,N/A
+10002152,2,0,DE,N/A
+10002153,4,0,US,N/A
+10002154,0,0,GB,N/A
+10002155,3,0,GB,N/A
+10002156,1,0,JP,N/A
+10002157,0,0,FR,N/A
+10002158,5,0,IT,N/A
+10002159,3,0,FR,N/A
+10002160,5,0,GB,N/A
+10002161,2,0,US,N/A
+10002162,4,0,GB,N/A
+10002163,4,0,US,N/A
+10002164,2,0,US,N/A
+10002165,0,0,IT,N/A
+10002166,4,0,CN,N/A
+10002167,4,0,US,N/A
+10002168,4,0,CN,N/A
+10002169,4,0,JP,N/A
+10002170,1,0,FR,N/A
+10002171,4,0,US,N/A
+10002172,3,0,IT,N/A
+10002173,5,0,IT,N/A
+10002174,0,0,FR,N/A
+10002175,2,0,DE,N/A
+10002176,4,0,JP,N/A
+10002177,5,0,DE,N/A
+10002178,0,0,FR,N/A
+10002179,2,0,CN,N/A
+10002180,3,0,GB,N/A
+10002181,0,0,GB,N/A
+10002182,5,0,DE,N/A
+10002183,0,0,US,N/A
+10002184,2,0,IT,N/A
+10002185,4,0,GB,N/A
+10002186,5,0,US,N/A
+10002187,5,0,DE,N/A
+10002188,5,0,DE,N/A
+10002189,2,0,CN,N/A
+10002190,5,0,GB,N/A
+10002191,1,0,IT,N/A
+10002192,3,0,GB,N/A
+10002193,5,0,FR,N/A
+10002194,5,0,JP,N/A
+10002195,2,0,GB,N/A
+10002196,1,0,FR,N/A
+10002197,3,0,GB,N/A
+10002198,4,0,JP,N/A
+10002199,3,0,CN,N/A
+10002200,3,0,JP,N/A
+10002201,4,0,JP,N/A
+10002202,4,0,IT,N/A
+10002203,4,0,IT,N/A
+10002204,5,0,FR,N/A
+10002205,4,0,IT,N/A
+10002206,3,0,JP,N/A
+10002207,2,0,DE,N/A
+10002208,5,0,US,N/A
+10002209,1,0,CN,N/A
+10002210,0,0,FR,N/A
+10002211,1,0,US,N/A
+10002212,1,0,CN,N/A
+10002213,3,0,CN,N/A
+10002214,2,0,JP,N/A
+10002215,1,0,DE,N/A
+10002216,2,0,JP,N/A
+10002217,5,0,JP,N/A
+10002218,0,0,IT,N/A
+10002219,2,0,US,N/A
+10002220,4,0,CN,N/A
+10002221,2,0,FR,N/A
+10002222,1,0,CN,N/A
+10002223,3,0,IT,N/A
+10002224,5,0,JP,N/A
+10002225,2,0,US,N/A
+10002226,5,0,FR,N/A
+10002227,1,0,JP,N/A
+10002228,2,0,DE,N/A
+10002229,1,0,CN,N/A
+10002230,1,0,GB,N/A
+10002231,1,0,FR,N/A
+10002232,5,0,GB,N/A
+10002233,5,0,DE,N/A
+10002234,5,0,DE,N/A
+10002235,3,0,FR,N/A
+10002236,3,0,JP,N/A
+10002237,3,0,DE,N/A
+10002238,4,0,US,N/A
+10002239,3,0,CN,N/A
+10002240,2,0,US,N/A
+10002241,5,0,FR,N/A
+10002242,4,0,US,N/A
+10002243,2,0,CN,N/A
+10002244,2,0,US,N/A
+10002245,5,0,GB,N/A
+10002246,1,0,JP,N/A
+10002247,4,0,CN,N/A
+10002248,2,0,GB,N/A
+10002249,5,0,JP,N/A
+10002250,0,0,DE,N/A
+10002251,3,0,JP,N/A
+10002252,4,0,FR,N/A
+10002253,4,0,JP,N/A
+10002254,5,0,JP,N/A
+10002255,0,0,GB,N/A
+10002256,2,0,CN,N/A
+10002257,5,0,IT,N/A
+10002258,4,0,US,N/A
+10002259,3,0,JP,N/A
+10002260,1,0,IT,N/A
+10002261,4,0,JP,N/A
+10002262,3,0,US,N/A
+10002263,0,0,CN,N/A
+10002264,2,0,CN,N/A
+10002265,2,0,JP,N/A
+10002266,5,0,US,N/A
+10002267,1,0,CN,N/A
+10002268,4,0,GB,N/A
+10002269,5,0,JP,N/A
+10002270,3,0,GB,N/A
+10002271,5,0,JP,N/A
+10002272,5,0,GB,N/A
+10002273,0,0,IT,N/A
+10002274,1,0,JP,N/A
+10002275,4,0,FR,N/A
+10002276,3,0,FR,N/A
+10002277,5,0,CN,N/A
+10002278,3,0,JP,N/A
+10002279,5,0,FR,N/A
+10002280,0,0,DE,N/A
+10002281,1,0,GB,N/A
+10002282,0,0,US,N/A
+10002283,5,0,JP,N/A
+10002284,2,0,IT,N/A
+10002285,1,0,CN,N/A
+10002286,5,0,US,N/A
+10002287,4,0,FR,N/A
+10002288,3,0,DE,N/A
+10002289,1,0,US,N/A
+10002290,3,0,CN,N/A
+10002291,0,0,FR,N/A
+10002292,2,0,IT,N/A
+10002293,3,0,US,N/A
+10002294,1,0,JP,N/A
+10002295,5,0,DE,N/A
+10002296,2,0,CN,N/A
+10002297,5,0,US,N/A
+10002298,0,0,GB,N/A
+10002299,2,0,FR,N/A
+10002300,2,0,DE,N/A
+10002301,0,0,US,N/A
+10002302,4,0,US,N/A
+10002303,3,0,DE,N/A
+10002304,1,0,US,N/A
+10002305,0,0,US,N/A
+10002306,1,0,IT,N/A
+10002307,2,0,CN,N/A
+10002308,0,0,GB,N/A
+10002309,5,0,JP,N/A
+10002310,0,0,CN,N/A
+10002311,1,0,FR,N/A
+10002312,0,0,US,N/A
+10002313,1,0,FR,N/A
+10002314,4,0,CN,N/A
+10002315,1,0,IT,N/A
+10002316,5,0,US,N/A
+10002317,5,0,JP,N/A
+10002318,1,0,IT,N/A
+10002319,3,0,IT,N/A
+10002320,4,0,IT,N/A
+10002321,0,0,DE,N/A
+10002322,4,0,CN,N/A
+10002323,4,0,FR,N/A
+10002324,2,0,GB,N/A
+10002325,4,0,JP,N/A
+10002326,5,0,JP,N/A
+10002327,3,0,JP,N/A
+10002328,4,0,US,N/A
+10002329,2,0,JP,N/A
+10002330,0,0,CN,N/A
+10002331,2,0,US,N/A
+10002332,5,0,IT,N/A
+10002333,0,0,IT,N/A
+10002334,0,0,CN,N/A
+10002335,4,0,DE,N/A
+10002336,3,0,US,N/A
+10002337,3,0,GB,N/A
+10002338,0,0,US,N/A
+10002339,2,0,GB,N/A
+10002340,5,0,GB,N/A
+10002341,1,0,GB,N/A
+10002342,4,0,FR,N/A
+10002343,3,0,IT,N/A
+10002344,2,0,DE,N/A
+10002345,1,0,US,N/A
+10002346,2,0,DE,N/A
+10002347,5,0,US,N/A
+10002348,0,0,DE,N/A
+10002349,4,0,FR,N/A
+10002350,5,0,CN,N/A
+10002351,3,0,US,N/A
+10002352,1,0,US,N/A
+10002353,0,0,GB,N/A
+10002354,5,0,FR,N/A
+10002355,2,0,IT,N/A
+10002356,5,0,DE,N/A
+10002357,3,0,GB,N/A
+10002358,1,0,JP,N/A
+10002359,0,0,FR,N/A
+10002360,4,0,GB,N/A
+10002361,3,0,FR,N/A
+10002362,0,0,IT,N/A
+10002363,0,0,US,N/A
+10002364,2,0,IT,N/A
+10002365,2,0,FR,N/A
+10002366,2,0,DE,N/A
+10002367,3,0,GB,N/A
+10002368,2,0,GB,N/A
+10002369,0,0,JP,N/A
+10002370,0,0,JP,N/A
+10002371,1,0,GB,N/A
+10002372,4,0,FR,N/A
+10002373,1,0,US,N/A
+10002374,1,0,FR,N/A
+10002375,0,0,CN,N/A
+10002376,3,0,JP,N/A
+10002377,2,0,US,N/A
+10002378,3,0,DE,N/A
+10002379,1,0,US,N/A
+10002380,5,0,JP,N/A
+10002381,1,0,GB,N/A
+10002382,4,0,JP,N/A
+10002383,2,0,FR,N/A
+10002384,4,0,GB,N/A
+10002385,0,0,FR,N/A
+10002386,0,0,GB,N/A
+10002387,4,0,FR,N/A
+10002388,5,0,IT,N/A
+10002389,3,0,GB,N/A
+10002390,2,0,IT,N/A
+10002391,2,0,CN,N/A
+10002392,0,0,US,N/A
+10002393,5,0,FR,N/A
+10002394,4,0,CN,N/A
+10002395,0,0,GB,N/A
+10002396,5,0,FR,N/A
+10002397,0,0,FR,N/A
+10002398,4,0,DE,N/A
+10002399,5,0,DE,N/A
+10002400,3,0,US,N/A
+10002401,0,0,GB,N/A
+10002402,3,0,US,N/A
+10002403,2,0,IT,N/A
+10002404,3,0,FR,N/A
+10002405,2,0,DE,N/A
+10002406,5,0,DE,N/A
+10002407,4,0,JP,N/A
+10002408,3,0,GB,N/A
+10002409,2,0,FR,N/A
+10002410,5,0,DE,N/A
+10002411,3,0,JP,N/A
+10002412,3,0,DE,N/A
+10002413,4,0,DE,N/A
+10002414,1,0,IT,N/A
+10002415,1,0,US,N/A
+10002416,4,0,GB,N/A
+10002417,4,0,JP,N/A
+10002418,1,0,DE,N/A
+10002419,4,0,US,N/A
+10002420,5,0,JP,N/A
+10002421,2,0,US,N/A
+10002422,0,0,IT,N/A
+10002423,4,0,CN,N/A
+10002424,0,0,DE,N/A
+10002425,4,0,CN,N/A
+10002426,0,0,US,N/A
+10002427,2,0,JP,N/A
+10002428,2,0,DE,N/A
+10002429,2,0,GB,N/A
+10002430,2,0,US,N/A
+10002431,2,0,US,N/A
+10002432,0,0,JP,N/A
+10002433,5,0,US,N/A
+10002434,3,0,CN,N/A
+10002435,4,0,GB,N/A
+10002436,5,0,JP,N/A
+10002437,0,0,GB,N/A
+10002438,3,0,FR,N/A
+10002439,2,0,JP,N/A
+10002440,3,0,FR,N/A
+10002441,1,0,DE,N/A
+10002442,0,0,JP,N/A
+10002443,3,0,FR,N/A
+10002444,2,0,US,N/A
+10002445,3,0,JP,N/A
+10002446,4,0,CN,N/A
+10002447,2,0,FR,N/A
+10002448,2,0,DE,N/A
+10002449,4,0,US,N/A
+10002450,5,0,JP,N/A
+10002451,0,0,CN,N/A
+10002452,1,0,IT,N/A
+10002453,0,0,FR,N/A
+10002454,1,0,GB,N/A
+10002455,2,0,JP,N/A
+10002456,4,0,FR,N/A
+10002457,0,0,CN,N/A
+10002458,2,0,DE,N/A
+10002459,5,0,CN,N/A
+10002460,3,0,DE,N/A
+10002461,4,0,US,N/A
+10002462,5,0,FR,N/A
+10002463,5,0,FR,N/A
+10002464,5,0,JP,N/A
+10002465,3,0,DE,N/A
+10002466,1,0,FR,N/A
+10002467,4,0,FR,N/A
+10002468,4,0,CN,N/A
+10002469,3,0,IT,N/A
+10002470,4,0,FR,N/A
+10002471,0,0,JP,N/A
+10002472,3,0,JP,N/A
+10002473,5,0,JP,N/A
+10002474,0,0,IT,N/A
+10002475,1,0,CN,N/A
+10002476,1,0,GB,N/A
+10002477,2,0,IT,N/A
+10002478,0,0,CN,N/A
+10002479,3,0,JP,N/A
+10002480,5,0,GB,N/A
+10002481,0,0,DE,N/A
+10002482,0,0,FR,N/A
+10002483,5,0,US,N/A
+10002484,3,0,US,N/A
+10002485,0,0,FR,N/A
+10002486,5,0,JP,N/A
+10002487,2,0,JP,N/A
+10002488,3,0,IT,N/A
+10002489,1,0,GB,N/A
+10002490,2,0,IT,N/A
+10002491,3,0,GB,N/A
+10002492,2,0,IT,N/A
+10002493,0,0,US,N/A
+10002494,3,0,IT,N/A
+10002495,3,0,CN,N/A
+10002496,2,0,DE,N/A
+10002497,2,0,FR,N/A
+10002498,3,0,IT,N/A
+10002499,1,0,GB,N/A
+10002500,4,0,JP,N/A
+10002501,0,0,FR,N/A
+10002502,3,0,US,N/A
+10002503,2,0,JP,N/A
+10002504,5,0,JP,N/A
+10002505,0,0,CN,N/A
+10002506,2,0,CN,N/A
+10002507,4,0,JP,N/A
+10002508,0,0,DE,N/A
+10002509,1,0,FR,N/A
+10002510,0,0,JP,N/A
+10002511,2,0,JP,N/A
+10002512,2,0,IT,N/A
+10002513,2,0,GB,N/A
+10002514,2,0,US,N/A
+10002515,3,0,DE,N/A
+10002516,3,0,IT,N/A
+10002517,1,0,FR,N/A
+10002518,2,0,FR,N/A
+10002519,5,0,DE,N/A
+10002520,1,0,FR,N/A
+10002521,3,0,FR,N/A
+10002522,5,0,US,N/A
+10002523,2,0,US,N/A
+10002524,4,0,DE,N/A
+10002525,0,0,GB,N/A
+10002526,0,0,CN,N/A
+10002527,4,0,CN,N/A
+10002528,4,0,JP,N/A
+10002529,3,0,GB,N/A
+10002530,3,0,JP,N/A
+10002531,1,0,CN,N/A
+10002532,1,0,FR,N/A
+10002533,5,0,IT,N/A
+10002534,0,0,GB,N/A
+10002535,3,0,US,N/A
+10002536,5,0,US,N/A
+10002537,1,0,US,N/A
+10002538,2,0,US,N/A
+10002539,0,0,US,N/A
+10002540,5,0,FR,N/A
+10002541,5,0,FR,N/A
+10002542,2,0,FR,N/A
+10002543,4,0,CN,N/A
+10002544,1,0,IT,N/A
+10002545,5,0,GB,N/A
+10002546,2,0,JP,N/A
+10002547,4,0,JP,N/A
+10002548,5,0,CN,N/A
+10002549,1,0,DE,N/A
+10002550,3,0,CN,N/A
+10002551,1,0,US,N/A
+10002552,0,0,CN,N/A
+10002553,1,0,JP,N/A
+10002554,5,0,IT,N/A
+10002555,4,0,US,N/A
+10002556,1,0,FR,N/A
+10002557,5,0,FR,N/A
+10002558,4,0,IT,N/A
+10002559,1,0,DE,N/A
+10002560,2,0,US,N/A
+10002561,5,0,DE,N/A
+10002562,2,0,JP,N/A
+10002563,4,0,IT,N/A
+10002564,1,0,JP,N/A
+10002565,4,0,US,N/A
+10002566,4,0,FR,N/A
+10002567,5,0,IT,N/A
+10002568,3,0,CN,N/A
+10002569,0,0,JP,N/A
+10002570,2,0,US,N/A
+10002571,5,0,GB,N/A
+10002572,4,0,GB,N/A
+10002573,4,0,US,N/A
+10002574,4,0,FR,N/A
+10002575,1,0,DE,N/A
+10002576,3,0,IT,N/A
+10002577,3,0,GB,N/A
+10002578,2,0,US,N/A
+10002579,1,0,GB,N/A
+10002580,0,0,US,N/A
+10002581,0,0,US,N/A
+10002582,4,0,IT,N/A
+10002583,2,0,FR,N/A
+10002584,3,0,CN,N/A
+10002585,3,0,JP,N/A
+10002586,0,0,JP,N/A
+10002587,3,0,JP,N/A
+10002588,3,0,GB,N/A
+10002589,5,0,FR,N/A
+10002590,3,0,FR,N/A
+10002591,1,0,IT,N/A
+10002592,4,0,FR,N/A
+10002593,2,0,CN,N/A
+10002594,4,0,CN,N/A
+10002595,1,0,JP,N/A
+10002596,3,0,CN,N/A
+10002597,3,0,GB,N/A
+10002598,1,0,FR,N/A
+10002599,0,0,DE,N/A
+10002600,2,0,JP,N/A
+10002601,1,0,DE,N/A
+10002602,5,0,CN,N/A
+10002603,2,0,CN,N/A
+10002604,5,0,GB,N/A
+10002605,4,0,DE,N/A
+10002606,5,0,JP,N/A
+10002607,5,0,CN,N/A
+10002608,2,0,US,N/A
+10002609,2,0,GB,N/A
+10002610,0,0,GB,N/A
+10002611,0,0,CN,N/A
+10002612,4,0,US,N/A
+10002613,2,0,CN,N/A
+10002614,1,0,GB,N/A
+10002615,0,0,JP,N/A
+10002616,0,0,CN,N/A
+10002617,4,0,US,N/A
+10002618,3,0,US,N/A
+10002619,4,0,CN,N/A
+10002620,3,0,FR,N/A
+10002621,0,0,GB,N/A
+10002622,0,0,GB,N/A
+10002623,1,0,GB,N/A
+10002624,2,0,US,N/A
+10002625,4,0,FR,N/A
+10002626,1,0,GB,N/A
+10002627,4,0,US,N/A
+10002628,0,0,DE,N/A
+10002629,2,0,DE,N/A
+10002630,2,0,DE,N/A
+10002631,5,0,GB,N/A
+10002632,2,0,US,N/A
+10002633,1,0,FR,N/A
+10002634,3,0,IT,N/A
+10002635,1,0,JP,N/A
+10002636,1,0,JP,N/A
+10002637,4,0,CN,N/A
+10002638,3,0,IT,N/A
+10002639,3,0,DE,N/A
+10002640,1,0,IT,N/A
+10002641,5,0,IT,N/A
+10002642,2,0,JP,N/A
+10002643,1,0,DE,N/A
+10002644,1,0,GB,N/A
+10002645,2,0,JP,N/A
+10002646,1,0,IT,N/A
+10002647,1,0,FR,N/A
+10002648,1,0,CN,N/A
+10002649,4,0,FR,N/A
+10002650,4,0,JP,N/A
+10002651,5,0,DE,N/A
+10002652,5,0,US,N/A
+10002653,5,0,DE,N/A
+10002654,3,0,JP,N/A
+10002655,5,0,GB,N/A
+10002656,3,0,GB,N/A
+10002657,2,0,US,N/A
+10002658,1,0,US,N/A
+10002659,5,0,GB,N/A
+10002660,5,0,FR,N/A
+10002661,2,0,US,N/A
+10002662,5,0,US,N/A
+10002663,0,0,IT,N/A
+10002664,5,0,IT,N/A
+10002665,4,0,IT,N/A
+10002666,4,0,CN,N/A
+10002667,0,0,JP,N/A
+10002668,0,0,DE,N/A
+10002669,0,0,CN,N/A
+10002670,4,0,IT,N/A
+10002671,2,0,US,N/A
+10002672,4,0,CN,N/A
+10002673,0,0,JP,N/A
+10002674,4,0,FR,N/A
+10002675,4,0,CN,N/A
+10002676,4,0,GB,N/A
+10002677,3,0,JP,N/A
+10002678,4,0,DE,N/A
+10002679,4,0,IT,N/A
+10002680,0,0,FR,N/A
+10002681,2,0,DE,N/A
+10002682,1,0,CN,N/A
+10002683,0,0,US,N/A
+10002684,3,0,DE,N/A
+10002685,4,0,IT,N/A
+10002686,0,0,FR,N/A
+10002687,2,0,GB,N/A
+10002688,1,0,IT,N/A
+10002689,0,0,IT,N/A
+10002690,1,0,GB,N/A
+10002691,0,0,IT,N/A
+10002692,2,0,US,N/A
+10002693,1,0,FR,N/A
+10002694,5,0,IT,N/A
+10002695,2,0,FR,N/A
+10002696,1,0,CN,N/A
+10002697,5,0,FR,N/A
+10002698,0,0,GB,N/A
+10002699,1,0,GB,N/A
+10002700,0,0,DE,N/A
+10002701,0,0,CN,N/A
+10002702,3,0,FR,N/A
+10002703,1,0,FR,N/A
+10002704,3,0,FR,N/A
+10002705,5,0,US,N/A
+10002706,5,0,DE,N/A
+10002707,1,0,DE,N/A
+10002708,5,0,DE,N/A
+10002709,5,0,US,N/A
+10002710,2,0,DE,N/A
+10002711,3,0,CN,N/A
+10002712,5,0,DE,N/A
+10002713,1,0,GB,N/A
+10002714,2,0,FR,N/A
+10002715,2,0,JP,N/A
+10002716,4,0,CN,N/A
+10002717,3,0,JP,N/A
+10002718,4,0,JP,N/A
+10002719,1,0,FR,N/A
+10002720,2,0,FR,N/A
+10002721,2,0,DE,N/A
+10002722,5,0,FR,N/A
+10002723,5,0,DE,N/A
+10002724,0,0,JP,N/A
+10002725,5,0,CN,N/A
+10002726,1,0,US,N/A
+10002727,4,0,IT,N/A
+10002728,1,0,CN,N/A
+10002729,5,0,US,N/A
+10002730,5,0,FR,N/A
+10002731,1,0,FR,N/A
+10002732,3,0,FR,N/A
+10002733,0,0,US,N/A
+10002734,4,0,JP,N/A
+10002735,3,0,IT,N/A
+10002736,4,0,IT,N/A
+10002737,2,0,IT,N/A
+10002738,5,0,DE,N/A
+10002739,3,0,JP,N/A
+10002740,2,0,CN,N/A
+10002741,2,0,FR,N/A
+10002742,3,0,US,N/A
+10002743,0,0,DE,N/A
+10002744,5,0,GB,N/A
+10002745,0,0,GB,N/A
+10002746,1,0,FR,N/A
+10002747,5,0,FR,N/A
+10002748,1,0,US,N/A
+10002749,1,0,JP,N/A
+10002750,3,0,JP,N/A
+10002751,5,0,IT,N/A
+10002752,0,0,FR,N/A
+10002753,5,0,DE,N/A
+10002754,1,0,US,N/A
+10002755,1,0,US,N/A
+10002756,1,0,JP,N/A
+10002757,3,0,DE,N/A
+10002758,0,0,FR,N/A
+10002759,3,0,CN,N/A
+10002760,3,0,US,N/A
+10002761,1,0,US,N/A
+10002762,3,0,FR,N/A
+10002763,0,0,US,N/A
+10002764,0,0,DE,N/A
+10002765,4,0,US,N/A
+10002766,4,0,US,N/A
+10002767,1,0,FR,N/A
+10002768,1,0,US,N/A
+10002769,4,0,CN,N/A
+10002770,0,0,GB,N/A
+10002771,0,0,GB,N/A
+10002772,3,0,CN,N/A
+10002773,3,0,US,N/A
+10002774,5,0,US,N/A
+10002775,5,0,FR,N/A
+10002776,4,0,IT,N/A
+10002777,3,0,IT,N/A
+10002778,1,0,CN,N/A
+10002779,3,0,FR,N/A
+10002780,2,0,IT,N/A
+10002781,4,0,JP,N/A
+10002782,1,0,US,N/A
+10002783,5,0,US,N/A
+10002784,5,0,IT,N/A
+10002785,3,0,DE,N/A
+10002786,5,0,DE,N/A
+10002787,0,0,JP,N/A
+10002788,3,0,DE,N/A
+10002789,4,0,FR,N/A
+10002790,4,0,JP,N/A
+10002791,0,0,FR,N/A
+10002792,3,0,FR,N/A
+10002793,3,0,GB,N/A
+10002794,3,0,CN,N/A
+10002795,2,0,JP,N/A
+10002796,1,0,CN,N/A
+10002797,5,0,US,N/A
+10002798,4,0,CN,N/A
+10002799,3,0,CN,N/A
+10002800,5,0,JP,N/A
+10002801,3,0,JP,N/A
+10002802,0,0,US,N/A
+10002803,2,0,DE,N/A
+10002804,0,0,IT,N/A
+10002805,2,0,FR,N/A
+10002806,5,0,GB,N/A
+10002807,5,0,JP,N/A
+10002808,4,0,DE,N/A
+10002809,4,0,FR,N/A
+10002810,4,0,GB,N/A
+10002811,2,0,US,N/A
+10002812,0,0,GB,N/A
+10002813,1,0,CN,N/A
+10002814,2,0,DE,N/A
+10002815,4,0,FR,N/A
+10002816,2,0,DE,N/A
+10002817,5,0,JP,N/A
+10002818,3,0,FR,N/A
+10002819,0,0,FR,N/A
+10002820,3,0,CN,N/A
+10002821,4,0,IT,N/A
+10002822,1,0,GB,N/A
+10002823,4,0,IT,N/A
+10002824,4,0,FR,N/A
+10002825,1,0,US,N/A
+10002826,2,0,US,N/A
+10002827,4,0,CN,N/A
+10002828,4,0,FR,N/A
+10002829,3,0,FR,N/A
+10002830,1,0,DE,N/A
+10002831,4,0,DE,N/A
+10002832,2,0,US,N/A
+10002833,5,0,CN,N/A
+10002834,2,0,US,N/A
+10002835,2,0,IT,N/A
+10002836,4,0,GB,N/A
+10002837,5,0,US,N/A
+10002838,5,0,US,N/A
+10002839,0,0,US,N/A
+10002840,0,0,DE,N/A
+10002841,5,0,CN,N/A
+10002842,3,0,GB,N/A
+10002843,4,0,JP,N/A
+10002844,2,0,IT,N/A
+10002845,3,0,FR,N/A
+10002846,0,0,GB,N/A
+10002847,0,0,US,N/A
+10002848,3,0,CN,N/A
+10002849,0,0,FR,N/A
+10002850,0,0,DE,N/A
+10002851,4,0,IT,N/A
+10002852,3,0,JP,N/A
+10002853,3,0,FR,N/A
+10002854,1,0,CN,N/A
+10002855,1,0,DE,N/A
+10002856,3,0,DE,N/A
+10002857,5,0,FR,N/A
+10002858,1,0,JP,N/A
+10002859,4,0,US,N/A
+10002860,4,0,JP,N/A
+10002861,2,0,CN,N/A
+10002862,1,0,DE,N/A
+10002863,0,0,FR,N/A
+10002864,2,0,FR,N/A
+10002865,0,0,CN,N/A
+10002866,4,0,CN,N/A
+10002867,5,0,JP,N/A
+10002868,5,0,GB,N/A
+10002869,0,0,US,N/A
+10002870,3,0,FR,N/A
+10002871,4,0,GB,N/A
+10002872,4,0,FR,N/A
+10002873,2,0,DE,N/A
+10002874,2,0,JP,N/A
+10002875,0,0,GB,N/A
+10002876,0,0,JP,N/A
+10002877,5,0,CN,N/A
+10002878,0,0,FR,N/A
+10002879,3,0,JP,N/A
+10002880,0,0,CN,N/A
+10002881,1,0,FR,N/A
+10002882,0,0,US,N/A
+10002883,4,0,FR,N/A
+10002884,3,0,DE,N/A
+10002885,5,0,FR,N/A
+10002886,4,0,CN,N/A
+10002887,3,0,CN,N/A
+10002888,2,0,DE,N/A
+10002889,5,0,GB,N/A
+10002890,4,0,JP,N/A
+10002891,4,0,GB,N/A
+10002892,5,0,IT,N/A
+10002893,5,0,IT,N/A
+10002894,4,0,DE,N/A
+10002895,1,0,CN,N/A
+10002896,1,0,JP,N/A
+10002897,4,0,CN,N/A
+10002898,4,0,JP,N/A
+10002899,3,0,CN,N/A
+10002900,1,0,FR,N/A
+10002901,1,0,IT,N/A
+10002902,4,0,DE,N/A
+10002903,2,0,FR,N/A
+10002904,0,0,JP,N/A
+10002905,1,0,US,N/A
+10002906,2,0,GB,N/A
+10002907,4,0,US,N/A
+10002908,3,0,JP,N/A
+10002909,5,0,US,N/A
+10002910,0,0,GB,N/A
+10002911,2,0,US,N/A
+10002912,4,0,US,N/A
+10002913,4,0,DE,N/A
+10002914,3,0,IT,N/A
+10002915,0,0,FR,N/A
+10002916,0,0,CN,N/A
+10002917,1,0,GB,N/A
+10002918,4,0,US,N/A
+10002919,3,0,US,N/A
+10002920,3,0,FR,N/A
+10002921,2,0,GB,N/A
+10002922,0,0,US,N/A
+10002923,2,0,JP,N/A
+10002924,3,0,CN,N/A
+10002925,4,0,JP,N/A
+10002926,2,0,GB,N/A
+10002927,0,0,CN,N/A
+10002928,0,0,IT,N/A
+10002929,4,0,GB,N/A
+10002930,3,0,GB,N/A
+10002931,2,0,IT,N/A
+10002932,4,0,IT,N/A
+10002933,3,0,US,N/A
+10002934,4,0,JP,N/A
+10002935,2,0,DE,N/A
+10002936,0,0,DE,N/A
+10002937,5,0,IT,N/A
+10002938,2,0,FR,N/A
+10002939,0,0,JP,N/A
+10002940,2,0,GB,N/A
+10002941,3,0,US,N/A
+10002942,4,0,US,N/A
+10002943,5,0,IT,N/A
+10002944,3,0,US,N/A
+10002945,3,0,CN,N/A
+10002946,4,0,FR,N/A
+10002947,4,0,JP,N/A
+10002948,0,0,FR,N/A
+10002949,0,0,CN,N/A
+10002950,0,0,GB,N/A
+10002951,3,0,FR,N/A
+10002952,0,0,IT,N/A
+10002953,2,0,US,N/A
+10002954,1,0,IT,N/A
+10002955,0,0,US,N/A
+10002956,2,0,US,N/A
+10002957,1,0,DE,N/A
+10002958,3,0,DE,N/A
+10002959,5,0,CN,N/A
+10002960,1,0,FR,N/A
+10002961,4,0,FR,N/A
+10002962,3,0,US,N/A
+10002963,4,0,DE,N/A
+10002964,3,0,JP,N/A
+10002965,4,0,CN,N/A
+10002966,1,0,GB,N/A
+10002967,0,0,IT,N/A
+10002968,4,0,FR,N/A
+10002969,2,0,JP,N/A
+10002970,5,0,FR,N/A
+10002971,1,0,FR,N/A
+10002972,4,0,JP,N/A
+10002973,5,0,CN,N/A
+10002974,1,0,DE,N/A
+10002975,4,0,US,N/A
+10002976,5,0,US,N/A
+10002977,0,0,GB,N/A
+10002978,1,0,JP,N/A
+10002979,1,0,CN,N/A
+10002980,5,0,US,N/A
+10002981,3,0,JP,N/A
+10002982,0,0,IT,N/A
+10002983,5,0,FR,N/A
+10002984,4,0,IT,N/A
+10002985,3,0,GB,N/A
+10002986,2,0,US,N/A
+10002987,0,0,CN,N/A
+10002988,2,0,FR,N/A
+10002989,0,0,US,N/A
+10002990,0,0,CN,N/A
+10002991,0,0,US,N/A
+10002992,4,0,FR,N/A
+10002993,4,0,CN,N/A
+10002994,4,0,FR,N/A
+10002995,0,0,CN,N/A
+10002996,2,0,DE,N/A
+10002997,2,0,IT,N/A
+10002998,1,0,CN,N/A
+10002999,2,0,US,N/A
+10003000,5,0,JP,N/A
+10003001,4,0,FR,N/A
+10003002,4,0,FR,N/A
+10003003,5,0,CN,N/A
+10003004,2,0,GB,N/A
+10003005,3,0,FR,N/A
+10003006,5,0,FR,N/A
+10003007,5,0,GB,N/A
+10003008,4,0,DE,N/A
+10003009,3,0,US,N/A
+10003010,4,0,DE,N/A
+10003011,5,0,JP,N/A
+10003012,1,0,FR,N/A
+10003013,5,0,US,N/A
+10003014,4,0,US,N/A
+10003015,1,0,US,N/A
+10003016,0,0,FR,N/A
+10003017,0,0,CN,N/A
+10003018,1,0,GB,N/A
+10003019,3,0,DE,N/A
+10003020,3,0,FR,N/A
+10003021,1,0,IT,N/A
+10003022,1,0,IT,N/A
+10003023,0,0,GB,N/A
+10003024,4,0,US,N/A
+10003025,5,0,IT,N/A
+10003026,0,0,US,N/A
+10003027,1,0,IT,N/A
+10003028,3,0,US,N/A
+10003029,4,0,IT,N/A
+10003030,4,0,DE,N/A
+10003031,4,0,IT,N/A
+10003032,1,0,US,N/A
+10003033,4,0,GB,N/A
+10003034,3,0,FR,N/A
+10003035,0,0,DE,N/A
+10003036,0,0,US,N/A
+10003037,3,0,JP,N/A
+10003038,2,0,GB,N/A
+10003039,2,0,US,N/A
+10003040,1,0,CN,N/A
+10003041,5,0,JP,N/A
+10003042,2,0,CN,N/A
+10003043,2,0,JP,N/A
+10003044,2,0,CN,N/A
+10003045,2,0,GB,N/A
+10003046,1,0,GB,N/A
+10003047,0,0,FR,N/A
+10003048,2,0,US,N/A
+10003049,5,0,US,N/A
+10003050,5,0,US,N/A
+10003051,1,0,GB,N/A
+10003052,4,0,CN,N/A
+10003053,1,0,GB,N/A
+10003054,4,0,FR,N/A
+10003055,3,0,FR,N/A
+10003056,3,0,IT,N/A
+10003057,5,0,FR,N/A
+10003058,1,0,DE,N/A
+10003059,5,0,IT,N/A
+10003060,3,0,US,N/A
+10003061,4,0,DE,N/A
+10003062,3,0,DE,N/A
+10003063,3,0,DE,N/A
+10003064,4,0,JP,N/A
+10003065,4,0,CN,N/A
+10003066,1,0,IT,N/A
+10003067,0,0,IT,N/A
+10003068,2,0,CN,N/A
+10003069,4,0,CN,N/A
+10003070,0,0,JP,N/A
+10003071,0,0,US,N/A
+10003072,0,0,JP,N/A
+10003073,4,0,GB,N/A
+10003074,5,0,FR,N/A
+10003075,2,0,JP,N/A
+10003076,0,0,JP,N/A
+10003077,3,0,DE,N/A
+10003078,3,0,FR,N/A
+10003079,4,0,IT,N/A
+10003080,4,0,IT,N/A
+10003081,0,0,DE,N/A
+10003082,3,0,CN,N/A
+10003083,4,0,GB,N/A
+10003084,3,0,JP,N/A
+10003085,0,0,GB,N/A
+10003086,0,0,DE,N/A
+10003087,0,0,GB,N/A
+10003088,5,0,CN,N/A
+10003089,2,0,DE,N/A
+10003090,4,0,US,N/A
+10003091,2,0,CN,N/A
+10003092,0,0,GB,N/A
+10003093,2,0,IT,N/A
+10003094,4,0,DE,N/A
+10003095,3,0,IT,N/A
+10003096,3,0,FR,N/A
+10003097,0,0,CN,N/A
+10003098,1,0,DE,N/A
+10003099,3,0,GB,N/A
+10003100,2,0,GB,N/A
+10003101,1,0,CN,N/A
+10003102,4,0,IT,N/A
+10003103,5,0,GB,N/A
+10003104,3,0,GB,N/A
+10003105,4,0,JP,N/A
+10003106,2,0,US,N/A
+10003107,1,0,CN,N/A
+10003108,2,0,IT,N/A
+10003109,4,0,JP,N/A
+10003110,0,0,CN,N/A
+10003111,1,0,CN,N/A
+10003112,5,0,IT,N/A
+10003113,1,0,JP,N/A
+10003114,2,0,DE,N/A
+10003115,1,0,US,N/A
+10003116,4,0,IT,N/A
+10003117,2,0,DE,N/A
+10003118,2,0,JP,N/A
+10003119,2,0,IT,N/A
+10003120,4,0,FR,N/A
+10003121,4,0,GB,N/A
+10003122,3,0,JP,N/A
+10003123,3,0,FR,N/A
+10003124,2,0,JP,N/A
+10003125,4,0,US,N/A
+10003126,1,0,FR,N/A
+10003127,4,0,FR,N/A
+10003128,4,0,IT,N/A
+10003129,2,0,US,N/A
+10003130,3,0,IT,N/A
+10003131,2,0,CN,N/A
+10003132,5,0,DE,N/A
+10003133,3,0,CN,N/A
+10003134,4,0,FR,N/A
+10003135,1,0,DE,N/A
+10003136,0,0,FR,N/A
+10003137,4,0,JP,N/A
+10003138,0,0,GB,N/A
+10003139,0,0,FR,N/A
+10003140,2,0,CN,N/A
+10003141,4,0,IT,N/A
+10003142,4,0,DE,N/A
+10003143,1,0,IT,N/A
+10003144,1,0,JP,N/A
+10003145,4,0,GB,N/A
+10003146,4,0,DE,N/A
+10003147,0,0,US,N/A
+10003148,2,0,FR,N/A
+10003149,0,0,CN,N/A
+10003150,4,0,FR,N/A
+10003151,0,0,JP,N/A
+10003152,0,0,CN,N/A
+10003153,1,0,FR,N/A
+10003154,4,0,DE,N/A
+10003155,4,0,IT,N/A
+10003156,1,0,GB,N/A
+10003157,1,0,FR,N/A
+10003158,3,0,US,N/A
+10003159,1,0,US,N/A
+10003160,1,0,JP,N/A
+10003161,1,0,DE,N/A
+10003162,0,0,JP,N/A
+10003163,1,0,GB,N/A
+10003164,0,0,GB,N/A
+10003165,0,0,DE,N/A
+10003166,2,0,GB,N/A
+10003167,5,0,JP,N/A
+10003168,5,0,JP,N/A
+10003169,1,0,IT,N/A
+10003170,2,0,FR,N/A
+10003171,3,0,JP,N/A
+10003172,0,0,GB,N/A
+10003173,3,0,CN,N/A
+10003174,4,0,GB,N/A
+10003175,0,0,GB,N/A
+10003176,2,0,GB,N/A
+10003177,0,0,IT,N/A
+10003178,5,0,JP,N/A
+10003179,1,0,DE,N/A
+10003180,3,0,FR,N/A
+10003181,4,0,GB,N/A
+10003182,0,0,GB,N/A
+10003183,4,0,IT,N/A
+10003184,2,0,GB,N/A
+10003185,0,0,GB,N/A
+10003186,4,0,GB,N/A
+10003187,3,0,US,N/A
+10003188,3,0,JP,N/A
+10003189,5,0,DE,N/A
+10003190,0,0,US,N/A
+10003191,2,0,DE,N/A
+10003192,0,0,US,N/A
+10003193,3,0,US,N/A
+10003194,1,0,CN,N/A
+10003195,4,0,DE,N/A
+10003196,5,0,DE,N/A
+10003197,1,0,US,N/A
+10003198,0,0,JP,N/A
+10003199,1,0,GB,N/A
+10003200,1,0,US,N/A
+10003201,3,0,IT,N/A
+10003202,5,0,JP,N/A
+10003203,0,0,JP,N/A
+10003204,3,0,CN,N/A
+10003205,1,0,JP,N/A
+10003206,0,0,FR,N/A
+10003207,5,0,FR,N/A
+10003208,4,0,US,N/A
+10003209,1,0,JP,N/A
+10003210,5,0,GB,N/A
+10003211,4,0,DE,N/A
+10003212,3,0,FR,N/A
+10003213,0,0,CN,N/A
+10003214,4,0,IT,N/A
+10003215,4,0,CN,N/A
+10003216,0,0,GB,N/A
+10003217,5,0,IT,N/A
+10003218,3,0,CN,N/A
+10003219,3,0,US,N/A
+10003220,5,0,CN,N/A
+10003221,2,0,DE,N/A
+10003222,4,0,JP,N/A
+10003223,5,0,DE,N/A
+10003224,5,0,US,N/A
+10003225,4,0,JP,N/A
+10003226,1,0,DE,N/A
+10003227,0,0,US,N/A
+10003228,2,0,CN,N/A
+10003229,4,0,IT,N/A
+10003230,4,0,CN,N/A
+10003231,1,0,US,N/A
+10003232,4,0,US,N/A
+10003233,0,0,JP,N/A
+10003234,2,0,DE,N/A
+10003235,0,0,IT,N/A
+10003236,3,0,IT,N/A
+10003237,2,0,GB,N/A
+10003238,2,0,IT,N/A
+10003239,2,0,IT,N/A
+10003240,2,0,DE,N/A
+10003241,5,0,US,N/A
+10003242,2,0,US,N/A
+10003243,4,0,US,N/A
+10003244,1,0,DE,N/A
+10003245,2,0,CN,N/A
+10003246,1,0,IT,N/A
+10003247,0,0,DE,N/A
+10003248,2,0,DE,N/A
+10003249,4,0,US,N/A
+10003250,5,0,DE,N/A
+10003251,4,0,IT,N/A
+10003252,1,0,DE,N/A
+10003253,0,0,FR,N/A
+10003254,3,0,CN,N/A
+10003255,2,0,US,N/A
+10003256,1,0,JP,N/A
+10003257,5,0,CN,N/A
+10003258,3,0,JP,N/A
+10003259,5,0,IT,N/A
+10003260,0,0,FR,N/A
+10003261,2,0,CN,N/A
+10003262,5,0,DE,N/A
+10003263,3,0,FR,N/A
+10003264,3,0,GB,N/A
+10003265,0,0,US,N/A
+10003266,3,0,US,N/A
+10003267,3,0,GB,N/A
+10003268,4,0,US,N/A
+10003269,2,0,IT,N/A
+10003270,1,0,FR,N/A
+10003271,2,0,CN,N/A
+10003272,4,0,GB,N/A
+10003273,5,0,DE,N/A
+10003274,3,0,CN,N/A
+10003275,1,0,DE,N/A
+10003276,3,0,GB,N/A
+10003277,3,0,JP,N/A
+10003278,0,0,JP,N/A
+10003279,4,0,FR,N/A
+10003280,1,0,JP,N/A
+10003281,4,0,FR,N/A
+10003282,4,0,IT,N/A
+10003283,2,0,DE,N/A
+10003284,1,0,DE,N/A
+10003285,1,0,DE,N/A
+10003286,3,0,DE,N/A
+10003287,3,0,JP,N/A
+10003288,0,0,JP,N/A
+10003289,1,0,US,N/A
+10003290,5,0,GB,N/A
+10003291,1,0,IT,N/A
+10003292,3,0,IT,N/A
+10003293,1,0,US,N/A
+10003294,1,0,DE,N/A
+10003295,3,0,JP,N/A
+10003296,0,0,JP,N/A
+10003297,0,0,JP,N/A
+10003298,4,0,US,N/A
+10003299,1,0,JP,N/A
+10003300,2,0,IT,N/A
+10003301,2,0,JP,N/A
+10003302,5,0,GB,N/A
+10003303,1,0,GB,N/A
+10003304,0,0,GB,N/A
+10003305,3,0,DE,N/A
+10003306,0,0,JP,N/A
+10003307,3,0,US,N/A
+10003308,0,0,DE,N/A
+10003309,3,0,FR,N/A
+10003310,1,0,CN,N/A
+10003311,4,0,FR,N/A
+10003312,0,0,IT,N/A
+10003313,5,0,US,N/A
+10003314,2,0,IT,N/A
+10003315,1,0,IT,N/A
+10003316,0,0,GB,N/A
+10003317,2,0,DE,N/A
+10003318,5,0,US,N/A
+10003319,1,0,GB,N/A
+10003320,0,0,FR,N/A
+10003321,2,0,FR,N/A
+10003322,4,0,JP,N/A
+10003323,4,0,FR,N/A
+10003324,3,0,DE,N/A
+10003325,2,0,GB,N/A
+10003326,2,0,GB,N/A
+10003327,0,0,FR,N/A
+10003328,0,0,IT,N/A
+10003329,3,0,US,N/A
+10003330,0,0,FR,N/A
+10003331,5,0,IT,N/A
+10003332,4,0,IT,N/A
+10003333,0,0,GB,N/A
+10003334,0,0,US,N/A
+10003335,2,0,IT,N/A
+10003336,5,0,US,N/A
+10003337,1,0,US,N/A
+10003338,5,0,IT,N/A
+10003339,5,0,JP,N/A
+10003340,1,0,FR,N/A
+10003341,0,0,JP,N/A
+10003342,0,0,JP,N/A
+10003343,3,0,JP,N/A
+10003344,5,0,CN,N/A
+10003345,3,0,JP,N/A
+10003346,1,0,JP,N/A
+10003347,3,0,FR,N/A
+10003348,4,0,DE,N/A
+10003349,1,0,FR,N/A
+10003350,1,0,US,N/A
+10003351,2,0,CN,N/A
+10003352,2,0,IT,N/A
+10003353,0,0,IT,N/A
+10003354,1,0,GB,N/A
+10003355,0,0,GB,N/A
+10003356,4,0,FR,N/A
+10003357,4,0,IT,N/A
+10003358,5,0,FR,N/A
+10003359,0,0,FR,N/A
+10003360,0,0,FR,N/A
+10003361,1,0,CN,N/A
+10003362,1,0,DE,N/A
+10003363,0,0,IT,N/A
+10003364,2,0,US,N/A
+10003365,2,0,GB,N/A
+10003366,2,0,GB,N/A
+10003367,2,0,DE,N/A
+10003368,2,0,IT,N/A
+10003369,5,0,DE,N/A
+10003370,4,0,CN,N/A
+10003371,3,0,JP,N/A
+10003372,3,0,DE,N/A
+10003373,2,0,CN,N/A
+10003374,4,0,FR,N/A
+10003375,2,0,GB,N/A
+10003376,4,0,DE,N/A
+10003377,5,0,IT,N/A
+10003378,0,0,DE,N/A
+10003379,1,0,US,N/A
+10003380,5,0,FR,N/A
+10003381,1,0,DE,N/A
+10003382,3,0,IT,N/A
+10003383,0,0,CN,N/A
+10003384,0,0,CN,N/A
+10003385,3,0,FR,N/A
+10003386,4,0,FR,N/A
+10003387,0,0,GB,N/A
+10003388,0,0,CN,N/A
+10003389,0,0,GB,N/A
+10003390,2,0,CN,N/A
+10003391,5,0,DE,N/A
+10003392,3,0,FR,N/A
+10003393,0,0,CN,N/A
+10003394,1,0,CN,N/A
+10003395,4,0,CN,N/A
+10003396,2,0,JP,N/A
+10003397,0,0,IT,N/A
+10003398,3,0,DE,N/A
+10003399,4,0,FR,N/A
+10003400,3,0,JP,N/A
+10003401,5,0,FR,N/A
+10003402,3,0,JP,N/A
+10003403,3,0,JP,N/A
+10003404,4,0,CN,N/A
+10003405,1,0,DE,N/A
+10003406,3,0,JP,N/A
+10003407,2,0,DE,N/A
+10003408,3,0,CN,N/A
+10003409,2,0,JP,N/A
+10003410,2,0,IT,N/A
+10003411,3,0,US,N/A
+10003412,1,0,CN,N/A
+10003413,4,0,GB,N/A
+10003414,4,0,JP,N/A
+10003415,4,0,GB,N/A
+10003416,2,0,IT,N/A
+10003417,0,0,DE,N/A
+10003418,4,0,JP,N/A
+10003419,0,0,GB,N/A
+10003420,2,0,IT,N/A
+10003421,2,0,FR,N/A
+10003422,1,0,US,N/A
+10003423,2,0,FR,N/A
+10003424,5,0,JP,N/A
+10003425,5,0,IT,N/A
+10003426,0,0,JP,N/A
+10003427,2,0,IT,N/A
+10003428,4,0,JP,N/A
+10003429,5,0,FR,N/A
+10003430,3,0,GB,N/A
+10003431,4,0,IT,N/A
+10003432,4,0,CN,N/A
+10003433,4,0,IT,N/A
+10003434,3,0,FR,N/A
+10003435,1,0,IT,N/A
+10003436,4,0,CN,N/A
+10003437,4,0,DE,N/A
+10003438,3,0,DE,N/A
+10003439,5,0,CN,N/A
+10003440,3,0,JP,N/A
+10003441,4,0,US,N/A
+10003442,4,0,JP,N/A
+10003443,3,0,JP,N/A
+10003444,4,0,DE,N/A
+10003445,3,0,CN,N/A
+10003446,0,0,JP,N/A
+10003447,2,0,JP,N/A
+10003448,2,0,CN,N/A
+10003449,2,0,CN,N/A
+10003450,2,0,IT,N/A
+10003451,0,0,GB,N/A
+10003452,2,0,JP,N/A
+10003453,2,0,JP,N/A
+10003454,5,0,CN,N/A
+10003455,3,0,GB,N/A
+10003456,1,0,JP,N/A
+10003457,2,0,IT,N/A
+10003458,4,0,US,N/A
+10003459,3,0,GB,N/A
+10003460,4,0,US,N/A
+10003461,3,0,DE,N/A
+10003462,3,0,CN,N/A
+10003463,4,0,JP,N/A
+10003464,3,0,IT,N/A
+10003465,2,0,US,N/A
+10003466,4,0,IT,N/A
+10003467,3,0,JP,N/A
+10003468,3,0,DE,N/A
+10003469,5,0,JP,N/A
+10003470,5,0,US,N/A
+10003471,2,0,GB,N/A
+10003472,5,0,DE,N/A
+10003473,0,0,JP,N/A
+10003474,4,0,JP,N/A
+10003475,5,0,US,N/A
+10003476,5,0,JP,N/A
+10003477,1,0,GB,N/A
+10003478,5,0,GB,N/A
+10003479,0,0,GB,N/A
+10003480,0,0,CN,N/A
+10003481,1,0,DE,N/A
+10003482,1,0,GB,N/A
+10003483,5,0,DE,N/A
+10003484,0,0,FR,N/A
+10003485,4,0,DE,N/A
+10003486,1,0,CN,N/A
+10003487,0,0,GB,N/A
+10003488,2,0,GB,N/A
+10003489,3,0,US,N/A
+10003490,4,0,JP,N/A
+10003491,5,0,FR,N/A
+10003492,3,0,CN,N/A
+10003493,0,0,GB,N/A
+10003494,1,0,IT,N/A
+10003495,4,0,JP,N/A
+10003496,3,0,JP,N/A
+10003497,1,0,GB,N/A
+10003498,3,0,JP,N/A
+10003499,1,0,CN,N/A
+10003500,1,0,IT,N/A
+10003501,4,0,JP,N/A
+10003502,1,0,CN,N/A
+10003503,5,0,DE,N/A
+10003504,4,0,IT,N/A
+10003505,3,0,IT,N/A
+10003506,1,0,FR,N/A
+10003507,2,0,US,N/A
+10003508,4,0,IT,N/A
+10003509,3,0,IT,N/A
+10003510,5,0,CN,N/A
+10003511,4,0,FR,N/A
+10003512,1,0,FR,N/A
+10003513,3,0,IT,N/A
+10003514,0,0,IT,N/A
+10003515,3,0,JP,N/A
+10003516,1,0,IT,N/A
+10003517,4,0,DE,N/A
+10003518,1,0,CN,N/A
+10003519,1,0,CN,N/A
+10003520,3,0,IT,N/A
+10003521,1,0,FR,N/A
+10003522,5,0,US,N/A
+10003523,5,0,GB,N/A
+10003524,1,0,FR,N/A
+10003525,1,0,FR,N/A
+10003526,1,0,DE,N/A
+10003527,4,0,JP,N/A
+10003528,3,0,CN,N/A
+10003529,2,0,FR,N/A
+10003530,0,0,IT,N/A
+10003531,4,0,DE,N/A
+10003532,5,0,DE,N/A
+10003533,2,0,IT,N/A
+10003534,1,0,IT,N/A
+10003535,3,0,IT,N/A
+10003536,5,0,IT,N/A
+10003537,3,0,JP,N/A
+10003538,0,0,US,N/A
+10003539,2,0,JP,N/A
+10003540,5,0,IT,N/A
+10003541,1,0,JP,N/A
+10003542,2,0,US,N/A
+10003543,4,0,DE,N/A
+10003544,1,0,DE,N/A
+10003545,1,0,GB,N/A
+10003546,5,0,US,N/A
+10003547,4,0,GB,N/A
+10003548,3,0,JP,N/A
+10003549,5,0,JP,N/A
+10003550,0,0,JP,N/A
+10003551,0,0,JP,N/A
+10003552,1,0,IT,N/A
+10003553,2,0,US,N/A
+10003554,0,0,JP,N/A
+10003555,2,0,JP,N/A
+10003556,5,0,JP,N/A
+10003557,2,0,FR,N/A
+10003558,2,0,FR,N/A
+10003559,0,0,FR,N/A
+10003560,3,0,IT,N/A
+10003561,1,0,CN,N/A
+10003562,3,0,US,N/A
+10003563,2,0,FR,N/A
+10003564,5,0,CN,N/A
+10003565,5,0,IT,N/A
+10003566,1,0,FR,N/A
+10003567,4,0,IT,N/A
+10003568,2,0,DE,N/A
+10003569,5,0,CN,N/A
+10003570,3,0,FR,N/A
+10003571,3,0,DE,N/A
+10003572,3,0,JP,N/A
+10003573,1,0,FR,N/A
+10003574,1,0,US,N/A
+10003575,2,0,FR,N/A
+10003576,2,0,IT,N/A
+10003577,3,0,FR,N/A
+10003578,2,0,CN,N/A
+10003579,2,0,FR,N/A
+10003580,4,0,CN,N/A
+10003581,1,0,JP,N/A
+10003582,3,0,GB,N/A
+10003583,4,0,IT,N/A
+10003584,5,0,GB,N/A
+10003585,5,0,JP,N/A
+10003586,1,0,GB,N/A
+10003587,5,0,IT,N/A
+10003588,1,0,FR,N/A
+10003589,3,0,IT,N/A
+10003590,2,0,IT,N/A
+10003591,1,0,US,N/A
+10003592,1,0,CN,N/A
+10003593,0,0,DE,N/A
+10003594,0,0,US,N/A
+10003595,1,0,JP,N/A
+10003596,1,0,DE,N/A
+10003597,3,0,IT,N/A
+10003598,3,0,FR,N/A
+10003599,2,0,FR,N/A
+10003600,4,0,DE,N/A
+10003601,4,0,IT,N/A
+10003602,3,0,US,N/A
+10003603,4,0,FR,N/A
+10003604,2,0,DE,N/A
+10003605,3,0,JP,N/A
+10003606,4,0,CN,N/A
+10003607,1,0,JP,N/A
+10003608,3,0,FR,N/A
+10003609,5,0,IT,N/A
+10003610,3,0,GB,N/A
+10003611,5,0,GB,N/A
+10003612,4,0,DE,N/A
+10003613,2,0,JP,N/A
+10003614,4,0,JP,N/A
+10003615,3,0,GB,N/A
+10003616,3,0,JP,N/A
+10003617,3,0,CN,N/A
+10003618,5,0,FR,N/A
+10003619,2,0,FR,N/A
+10003620,4,0,DE,N/A
+10003621,2,0,GB,N/A
+10003622,5,0,GB,N/A
+10003623,5,0,DE,N/A
+10003624,4,0,CN,N/A
+10003625,5,0,IT,N/A
+10003626,1,0,US,N/A
+10003627,0,0,FR,N/A
+10003628,0,0,FR,N/A
+10003629,4,0,FR,N/A
+10003630,4,0,CN,N/A
+10003631,3,0,GB,N/A
+10003632,1,0,US,N/A
+10003633,1,0,IT,N/A
+10003634,5,0,FR,N/A
+10003635,5,0,GB,N/A
+10003636,4,0,JP,N/A
+10003637,5,0,IT,N/A
+10003638,3,0,IT,N/A
+10003639,2,0,US,N/A
+10003640,4,0,JP,N/A
+10003641,3,0,FR,N/A
+10003642,5,0,CN,N/A
+10003643,0,0,DE,N/A
+10003644,5,0,US,N/A
+10003645,2,0,GB,N/A
+10003646,4,0,JP,N/A
+10003647,5,0,GB,N/A
+10003648,1,0,JP,N/A
+10003649,3,0,CN,N/A
+10003650,4,0,DE,N/A
+10003651,1,0,JP,N/A
+10003652,3,0,FR,N/A
+10003653,5,0,JP,N/A
+10003654,1,0,JP,N/A
+10003655,4,0,US,N/A
+10003656,0,0,FR,N/A
+10003657,4,0,CN,N/A
+10003658,3,0,US,N/A
+10003659,3,0,DE,N/A
+10003660,3,0,CN,N/A
+10003661,0,0,CN,N/A
+10003662,0,0,US,N/A
+10003663,5,0,JP,N/A
+10003664,0,0,GB,N/A
+10003665,4,0,FR,N/A
+10003666,5,0,JP,N/A
+10003667,4,0,DE,N/A
+10003668,5,0,JP,N/A
+10003669,3,0,GB,N/A
+10003670,1,0,GB,N/A
+10003671,4,0,FR,N/A
+10003672,5,0,CN,N/A
+10003673,0,0,DE,N/A
+10003674,4,0,CN,N/A
+10003675,5,0,IT,N/A
+10003676,0,0,US,N/A
+10003677,1,0,CN,N/A
+10003678,4,0,US,N/A
+10003679,1,0,IT,N/A
+10003680,0,0,IT,N/A
+10003681,5,0,IT,N/A
+10003682,0,0,IT,N/A
+10003683,4,0,DE,N/A
+10003684,5,0,US,N/A
+10003685,1,0,FR,N/A
+10003686,1,0,DE,N/A
+10003687,1,0,IT,N/A
+10003688,1,0,IT,N/A
+10003689,0,0,US,N/A
+10003690,4,0,CN,N/A
+10003691,4,0,FR,N/A
+10003692,2,0,DE,N/A
+10003693,0,0,DE,N/A
+10003694,3,0,CN,N/A
+10003695,1,0,US,N/A
+10003696,3,0,US,N/A
+10003697,3,0,IT,N/A
+10003698,3,0,JP,N/A
+10003699,5,0,DE,N/A
+10003700,1,0,GB,N/A
+10003701,2,0,CN,N/A
+10003702,4,0,JP,N/A
+10003703,5,0,FR,N/A
+10003704,3,0,JP,N/A
+10003705,4,0,CN,N/A
+10003706,5,0,DE,N/A
+10003707,2,0,FR,N/A
+10003708,5,0,JP,N/A
+10003709,0,0,JP,N/A
+10003710,1,0,IT,N/A
+10003711,1,0,IT,N/A
+10003712,4,0,JP,N/A
+10003713,3,0,CN,N/A
+10003714,0,0,JP,N/A
+10003715,3,0,IT,N/A
+10003716,0,0,DE,N/A
+10003717,4,0,GB,N/A
+10003718,1,0,IT,N/A
+10003719,4,0,FR,N/A
+10003720,3,0,JP,N/A
+10003721,3,0,IT,N/A
+10003722,0,0,US,N/A
+10003723,1,0,IT,N/A
+10003724,5,0,US,N/A
+10003725,4,0,FR,N/A
+10003726,3,0,DE,N/A
+10003727,5,0,FR,N/A
+10003728,2,0,DE,N/A
+10003729,2,0,JP,N/A
+10003730,0,0,DE,N/A
+10003731,0,0,FR,N/A
+10003732,1,0,FR,N/A
+10003733,4,0,CN,N/A
+10003734,0,0,FR,N/A
+10003735,2,0,GB,N/A
+10003736,1,0,US,N/A
+10003737,3,0,CN,N/A
+10003738,5,0,CN,N/A
+10003739,2,0,DE,N/A
+10003740,4,0,IT,N/A
+10003741,3,0,IT,N/A
+10003742,3,0,JP,N/A
+10003743,1,0,DE,N/A
+10003744,3,0,DE,N/A
+10003745,3,0,IT,N/A
+10003746,0,0,JP,N/A
+10003747,0,0,CN,N/A
+10003748,0,0,JP,N/A
+10003749,2,0,FR,N/A
+10003750,1,0,US,N/A
+10003751,5,0,FR,N/A
+10003752,5,0,US,N/A
+10003753,3,0,IT,N/A
+10003754,1,0,IT,N/A
+10003755,1,0,DE,N/A
+10003756,4,0,US,N/A
+10003757,0,0,CN,N/A
+10003758,3,0,JP,N/A
+10003759,5,0,FR,N/A
+10003760,3,0,US,N/A
+10003761,5,0,US,N/A
+10003762,0,0,GB,N/A
+10003763,5,0,US,N/A
+10003764,3,0,DE,N/A
+10003765,3,0,DE,N/A
+10003766,5,0,JP,N/A
+10003767,2,0,JP,N/A
+10003768,1,0,JP,N/A
+10003769,1,0,US,N/A
+10003770,0,0,FR,N/A
+10003771,3,0,JP,N/A
+10003772,2,0,GB,N/A
+10003773,1,0,GB,N/A
+10003774,2,0,DE,N/A
+10003775,3,0,CN,N/A
+10003776,1,0,DE,N/A
+10003777,5,0,CN,N/A
+10003778,2,0,FR,N/A
+10003779,4,0,CN,N/A
+10003780,3,0,JP,N/A
+10003781,2,0,IT,N/A
+10003782,4,0,DE,N/A
+10003783,4,0,GB,N/A
+10003784,1,0,US,N/A
+10003785,3,0,FR,N/A
+10003786,2,0,DE,N/A
+10003787,0,0,JP,N/A
+10003788,4,0,JP,N/A
+10003789,5,0,GB,N/A
+10003790,1,0,FR,N/A
+10003791,1,0,US,N/A
+10003792,0,0,IT,N/A
+10003793,0,0,IT,N/A
+10003794,5,0,JP,N/A
+10003795,1,0,US,N/A
+10003796,5,0,IT,N/A
+10003797,2,0,CN,N/A
+10003798,5,0,IT,N/A
+10003799,1,0,US,N/A
+10003800,0,0,US,N/A
+10003801,2,0,FR,N/A
+10003802,4,0,US,N/A
+10003803,3,0,JP,N/A
+10003804,5,0,IT,N/A
+10003805,4,0,JP,N/A
+10003806,4,0,GB,N/A
+10003807,5,0,CN,N/A
+10003808,0,0,JP,N/A
+10003809,2,0,IT,N/A
+10003810,5,0,DE,N/A
+10003811,3,0,GB,N/A
+10003812,3,0,DE,N/A
+10003813,0,0,FR,N/A
+10003814,5,0,FR,N/A
+10003815,4,0,US,N/A
+10003816,3,0,GB,N/A
+10003817,3,0,GB,N/A
+10003818,1,0,US,N/A
+10003819,4,0,FR,N/A
+10003820,4,0,IT,N/A
+10003821,1,0,GB,N/A
+10003822,3,0,FR,N/A
+10003823,0,0,GB,N/A
+10003824,2,0,CN,N/A
+10003825,2,0,IT,N/A
+10003826,4,0,GB,N/A
+10003827,5,0,DE,N/A
+10003828,4,0,JP,N/A
+10003829,1,0,GB,N/A
+10003830,1,0,GB,N/A
+10003831,4,0,DE,N/A
+10003832,3,0,GB,N/A
+10003833,3,0,JP,N/A
+10003834,3,0,DE,N/A
+10003835,4,0,GB,N/A
+10003836,5,0,DE,N/A
+10003837,2,0,CN,N/A
+10003838,5,0,JP,N/A
+10003839,4,0,FR,N/A
+10003840,3,0,US,N/A
+10003841,0,0,FR,N/A
+10003842,2,0,FR,N/A
+10003843,5,0,GB,N/A
+10003844,2,0,JP,N/A
+10003845,0,0,FR,N/A
+10003846,0,0,US,N/A
+10003847,3,0,US,N/A
+10003848,0,0,JP,N/A
+10003849,3,0,JP,N/A
+10003850,4,0,US,N/A
+10003851,2,0,CN,N/A
+10003852,4,0,JP,N/A
+10003853,2,0,CN,N/A
+10003854,2,0,FR,N/A
+10003855,0,0,US,N/A
+10003856,3,0,IT,N/A
+10003857,3,0,FR,N/A
+10003858,3,0,IT,N/A
+10003859,2,0,FR,N/A
+10003860,3,0,CN,N/A
+10003861,4,0,DE,N/A
+10003862,1,0,FR,N/A
+10003863,0,0,FR,N/A
+10003864,2,0,CN,N/A
+10003865,2,0,FR,N/A
+10003866,5,0,DE,N/A
+10003867,2,0,CN,N/A
+10003868,4,0,CN,N/A
+10003869,3,0,GB,N/A
+10003870,3,0,IT,N/A
+10003871,3,0,IT,N/A
+10003872,0,0,CN,N/A
+10003873,5,0,DE,N/A
+10003874,5,0,GB,N/A
+10003875,5,0,FR,N/A
+10003876,1,0,GB,N/A
+10003877,0,0,DE,N/A
+10003878,3,0,GB,N/A
+10003879,5,0,CN,N/A
+10003880,1,0,FR,N/A
+10003881,5,0,JP,N/A
+10003882,0,0,IT,N/A
+10003883,3,0,US,N/A
+10003884,4,0,US,N/A
+10003885,0,0,JP,N/A
+10003886,4,0,FR,N/A
+10003887,3,0,DE,N/A
+10003888,3,0,JP,N/A
+10003889,5,0,US,N/A
+10003890,0,0,IT,N/A
+10003891,2,0,FR,N/A
+10003892,5,0,GB,N/A
+10003893,1,0,JP,N/A
+10003894,1,0,FR,N/A
+10003895,4,0,FR,N/A
+10003896,0,0,FR,N/A
+10003897,0,0,JP,N/A
+10003898,0,0,US,N/A
+10003899,2,0,IT,N/A
+10003900,2,0,US,N/A
+10003901,3,0,JP,N/A
+10003902,1,0,CN,N/A
+10003903,1,0,US,N/A
+10003904,1,0,CN,N/A
+10003905,2,0,US,N/A
+10003906,4,0,JP,N/A
+10003907,3,0,GB,N/A
+10003908,0,0,GB,N/A
+10003909,4,0,FR,N/A
+10003910,0,0,GB,N/A
+10003911,0,0,JP,N/A
+10003912,2,0,GB,N/A
+10003913,1,0,US,N/A
+10003914,4,0,DE,N/A
+10003915,5,0,CN,N/A
+10003916,3,0,JP,N/A
+10003917,3,0,IT,N/A
+10003918,1,0,DE,N/A
+10003919,0,0,IT,N/A
+10003920,0,0,DE,N/A
+10003921,1,0,IT,N/A
+10003922,5,0,GB,N/A
+10003923,0,0,JP,N/A
+10003924,5,0,US,N/A
+10003925,5,0,CN,N/A
+10003926,4,0,CN,N/A
+10003927,3,0,US,N/A
+10003928,2,0,CN,N/A
+10003929,1,0,CN,N/A
+10003930,3,0,JP,N/A
+10003931,2,0,GB,N/A
+10003932,4,0,IT,N/A
+10003933,5,0,IT,N/A
+10003934,5,0,CN,N/A
+10003935,2,0,US,N/A
+10003936,3,0,IT,N/A
+10003937,0,0,US,N/A
+10003938,3,0,IT,N/A
+10003939,5,0,DE,N/A
+10003940,3,0,CN,N/A
+10003941,0,0,JP,N/A
+10003942,4,0,US,N/A
+10003943,0,0,IT,N/A
+10003944,5,0,GB,N/A
+10003945,5,0,GB,N/A
+10003946,4,0,JP,N/A
+10003947,0,0,FR,N/A
+10003948,1,0,FR,N/A
+10003949,4,0,DE,N/A
+10003950,1,0,GB,N/A
+10003951,0,0,DE,N/A
+10003952,3,0,US,N/A
+10003953,5,0,IT,N/A
+10003954,4,0,IT,N/A
+10003955,3,0,GB,N/A
+10003956,2,0,CN,N/A
+10003957,2,0,IT,N/A
+10003958,5,0,CN,N/A
+10003959,1,0,FR,N/A
+10003960,1,0,DE,N/A
+10003961,1,0,FR,N/A
+10003962,4,0,US,N/A
+10003963,0,0,JP,N/A
+10003964,4,0,GB,N/A
+10003965,3,0,GB,N/A
+10003966,5,0,DE,N/A
+10003967,3,0,DE,N/A
+10003968,2,0,FR,N/A
+10003969,1,0,DE,N/A
+10003970,1,0,US,N/A
+10003971,3,0,CN,N/A
+10003972,3,0,JP,N/A
+10003973,1,0,CN,N/A
+10003974,2,0,CN,N/A
+10003975,0,0,FR,N/A
+10003976,2,0,US,N/A
+10003977,2,0,CN,N/A
+10003978,0,0,FR,N/A
+10003979,2,0,IT,N/A
+10003980,0,0,GB,N/A
+10003981,3,0,DE,N/A
+10003982,0,0,JP,N/A
+10003983,1,0,IT,N/A
+10003984,0,0,CN,N/A
+10003985,3,0,IT,N/A
+10003986,4,0,GB,N/A
+10003987,5,0,GB,N/A
+10003988,0,0,IT,N/A
+10003989,0,0,JP,N/A
+10003990,5,0,CN,N/A
+10003991,1,0,US,N/A
+10003992,4,0,JP,N/A
+10003993,2,0,GB,N/A
+10003994,5,0,IT,N/A
+10003995,3,0,FR,N/A
+10003996,0,0,GB,N/A
+10003997,1,0,CN,N/A
+10003998,4,0,JP,N/A
+10003999,4,0,DE,N/A
+10004000,4,0,IT,N/A
+10004001,5,0,JP,N/A
+10004002,5,0,IT,N/A
+10004003,3,0,CN,N/A
+10004004,4,0,US,N/A
+10004005,2,0,GB,N/A
+10004006,0,0,CN,N/A
+10004007,4,0,US,N/A
+10004008,4,0,DE,N/A
+10004009,5,0,FR,N/A
+10004010,0,0,CN,N/A
+10004011,0,0,GB,N/A
+10004012,0,0,FR,N/A
+10004013,0,0,GB,N/A
+10004014,4,0,US,N/A
+10004015,3,0,CN,N/A
+10004016,1,0,GB,N/A
+10004017,1,0,JP,N/A
+10004018,3,0,FR,N/A
+10004019,3,0,JP,N/A
+10004020,2,0,GB,N/A
+10004021,1,0,IT,N/A
+10004022,1,0,GB,N/A
+10004023,1,0,FR,N/A
+10004024,3,0,CN,N/A
+10004025,0,0,GB,N/A
+10004026,1,0,DE,N/A
+10004027,1,0,CN,N/A
+10004028,5,0,JP,N/A
+10004029,5,0,GB,N/A
+10004030,0,0,IT,N/A
+10004031,5,0,DE,N/A
+10004032,5,0,US,N/A
+10004033,2,0,FR,N/A
+10004034,1,0,DE,N/A
+10004035,2,0,CN,N/A
+10004036,4,0,JP,N/A
+10004037,2,0,GB,N/A
+10004038,2,0,JP,N/A
+10004039,1,0,FR,N/A
+10004040,0,0,GB,N/A
+10004041,4,0,JP,N/A
+10004042,1,0,DE,N/A
+10004043,3,0,CN,N/A
+10004044,0,0,GB,N/A
+10004045,4,0,FR,N/A
+10004046,4,0,FR,N/A
+10004047,0,0,IT,N/A
+10004048,3,0,JP,N/A
+10004049,3,0,DE,N/A
+10004050,3,0,DE,N/A
+10004051,4,0,US,N/A
+10004052,3,0,GB,N/A
+10004053,3,0,DE,N/A
+10004054,4,0,GB,N/A
+10004055,0,0,GB,N/A
+10004056,3,0,CN,N/A
+10004057,4,0,GB,N/A
+10004058,2,0,FR,N/A
+10004059,1,0,DE,N/A
+10004060,1,0,CN,N/A
+10004061,0,0,JP,N/A
+10004062,3,0,CN,N/A
+10004063,5,0,DE,N/A
+10004064,2,0,FR,N/A
+10004065,5,0,GB,N/A
+10004066,3,0,US,N/A
+10004067,1,0,FR,N/A
+10004068,3,0,US,N/A
+10004069,5,0,US,N/A
+10004070,1,0,FR,N/A
+10004071,5,0,CN,N/A
+10004072,2,0,GB,N/A
+10004073,4,0,JP,N/A
+10004074,2,0,FR,N/A
+10004075,1,0,CN,N/A
+10004076,0,0,US,N/A
+10004077,3,0,DE,N/A
+10004078,1,0,JP,N/A
+10004079,1,0,DE,N/A
+10004080,4,0,DE,N/A
+10004081,0,0,DE,N/A
+10004082,0,0,CN,N/A
+10004083,0,0,DE,N/A
+10004084,2,0,CN,N/A
+10004085,1,0,DE,N/A
+10004086,4,0,IT,N/A
+10004087,0,0,FR,N/A
+10004088,5,0,JP,N/A
+10004089,4,0,IT,N/A
+10004090,2,0,JP,N/A
+10004091,2,0,FR,N/A
+10004092,4,0,JP,N/A
+10004093,2,0,IT,N/A
+10004094,1,0,JP,N/A
+10004095,2,0,IT,N/A
+10004096,5,0,US,N/A
+10004097,3,0,IT,N/A
+10004098,2,0,IT,N/A
+10004099,3,0,IT,N/A
+10004100,0,0,FR,N/A
+10004101,0,0,IT,N/A
+10004102,1,0,JP,N/A
+10004103,1,0,DE,N/A
+10004104,2,0,FR,N/A
+10004105,0,0,IT,N/A
+10004106,1,0,DE,N/A
+10004107,0,0,US,N/A
+10004108,2,0,IT,N/A
+10004109,3,0,IT,N/A
+10004110,2,0,US,N/A
+10004111,5,0,US,N/A
+10004112,4,0,IT,N/A
+10004113,0,0,DE,N/A
+10004114,1,0,FR,N/A
+10004115,1,0,FR,N/A
+10004116,3,0,JP,N/A
+10004117,3,0,US,N/A
+10004118,4,0,CN,N/A
+10004119,5,0,US,N/A
+10004120,1,0,DE,N/A
+10004121,2,0,CN,N/A
+10004122,3,0,US,N/A
+10004123,1,0,GB,N/A
+10004124,0,0,IT,N/A
+10004125,3,0,DE,N/A
+10004126,2,0,FR,N/A
+10004127,2,0,GB,N/A
+10004128,2,0,GB,N/A
+10004129,0,0,IT,N/A
+10004130,4,0,CN,N/A
+10004131,0,0,FR,N/A
+10004132,5,0,FR,N/A
+10004133,0,0,US,N/A
+10004134,0,0,FR,N/A
+10004135,4,0,FR,N/A
+10004136,1,0,DE,N/A
+10004137,5,0,DE,N/A
+10004138,2,0,GB,N/A
+10004139,1,0,US,N/A
+10004140,3,0,GB,N/A
+10004141,2,0,CN,N/A
+10004142,3,0,CN,N/A
+10004143,4,0,JP,N/A
+10004144,3,0,FR,N/A
+10004145,1,0,JP,N/A
+10004146,2,0,GB,N/A
+10004147,0,0,US,N/A
+10004148,3,0,DE,N/A
+10004149,1,0,IT,N/A
+10004150,1,0,IT,N/A
+10004151,1,0,IT,N/A
+10004152,0,0,CN,N/A
+10004153,4,0,JP,N/A
+10004154,5,0,CN,N/A
+10004155,2,0,US,N/A
+10004156,2,0,CN,N/A
+10004157,5,0,US,N/A
+10004158,5,0,DE,N/A
+10004159,1,0,CN,N/A
+10004160,3,0,FR,N/A
+10004161,4,0,DE,N/A
+10004162,1,0,CN,N/A
+10004163,3,0,US,N/A
+10004164,4,0,CN,N/A
+10004165,3,0,GB,N/A
+10004166,4,0,IT,N/A
+10004167,5,0,IT,N/A
+10004168,1,0,CN,N/A
+10004169,4,0,JP,N/A
+10004170,4,0,GB,N/A
+10004171,0,0,FR,N/A
+10004172,5,0,GB,N/A
+10004173,4,0,DE,N/A
+10004174,5,0,IT,N/A
+10004175,4,0,GB,N/A
+10004176,1,0,IT,N/A
+10004177,2,0,GB,N/A
+10004178,1,0,CN,N/A
+10004179,5,0,CN,N/A
+10004180,5,0,IT,N/A
+10004181,0,0,DE,N/A
+10004182,0,0,IT,N/A
+10004183,4,0,JP,N/A
+10004184,5,0,US,N/A
+10004185,2,0,CN,N/A
+10004186,5,0,CN,N/A
+10004187,1,0,IT,N/A
+10004188,4,0,DE,N/A
+10004189,4,0,IT,N/A
+10004190,2,0,GB,N/A
+10004191,1,0,FR,N/A
+10004192,2,0,JP,N/A
+10004193,2,0,IT,N/A
+10004194,1,0,CN,N/A
+10004195,2,0,IT,N/A
+10004196,0,0,JP,N/A
+10004197,4,0,JP,N/A
+10004198,2,0,IT,N/A
+10004199,2,0,DE,N/A
+10004200,2,0,GB,N/A
+10004201,1,0,GB,N/A
+10004202,1,0,GB,N/A
+10004203,3,0,FR,N/A
+10004204,2,0,FR,N/A
+10004205,2,0,US,N/A
+10004206,4,0,DE,N/A
+10004207,0,0,FR,N/A
+10004208,5,0,DE,N/A
+10004209,1,0,FR,N/A
+10004210,5,0,GB,N/A
+10004211,2,0,GB,N/A
+10004212,1,0,IT,N/A
+10004213,2,0,CN,N/A
+10004214,1,0,US,N/A
+10004215,3,0,JP,N/A
+10004216,3,0,DE,N/A
+10004217,5,0,GB,N/A
+10004218,3,0,JP,N/A
+10004219,4,0,FR,N/A
+10004220,4,0,US,N/A
+10004221,3,0,DE,N/A
+10004222,1,0,CN,N/A
+10004223,3,0,FR,N/A
+10004224,0,0,GB,N/A
+10004225,4,0,IT,N/A
+10004226,0,0,IT,N/A
+10004227,3,0,DE,N/A
+10004228,4,0,FR,N/A
+10004229,2,0,CN,N/A
+10004230,0,0,JP,N/A
+10004231,3,0,CN,N/A
+10004232,4,0,CN,N/A
+10004233,1,0,CN,N/A
+10004234,5,0,GB,N/A
+10004235,2,0,US,N/A
+10004236,2,0,JP,N/A
+10004237,0,0,FR,N/A
+10004238,1,0,DE,N/A
+10004239,0,0,GB,N/A
+10004240,3
<TRUNCATED>
[35/50] [abbrv] kylin git commit: KYLIN-2212 add more test queries
Posted by li...@apache.org.
KYLIN-2212 add more test queries
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/af429e5c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/af429e5c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/af429e5c
Branch: refs/heads/master-cdh5.7
Commit: af429e5cbbda2a64159b25bc5ce4ad70f2f1a1f2
Parents: 8f3239b
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Dec 5 15:05:35 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Dec 5 15:05:35 2016 +0800
----------------------------------------------------------------------
.../metadata/filter/LogicalTupleFilter.java | 3 +-
.../src/test/resources/query/sql/query101.sql | 13 ++++++++
.../src/test/resources/query/sql/query102.sql | 13 ++++++++
.../src/test/resources/query/sql/query103.sql | 13 ++++++++
.../test/resources/query/sql_like/query21.sql | 31 ++++++++++++++++++++
5 files changed, 71 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 61657fb..1744309 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -64,8 +64,7 @@ public class LogicalTupleFilter extends TupleFilter {
public TupleFilter reverse() {
switch (operator) {
case NOT:
- throw new IllegalStateException("not( not in ()) is invalid syntax");
- //return reverseNestedNots(this, 0);
+ throw new IllegalStateException("NOT will be replaced in org.apache.kylin.query.relnode.OLAPFilterRel.TupleFilterVisitor");
case AND:
case OR:
LogicalTupleFilter reverse = new LogicalTupleFilter(REVERSE_OP_MAP.get(operator));
http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/kylin-it/src/test/resources/query/sql/query101.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query101.sql b/kylin-it/src/test/resources/query/sql/query101.sql
new file mode 100644
index 0000000..fb42bca
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query101.sql
@@ -0,0 +1,13 @@
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') )
+ group by meta_categ_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/kylin-it/src/test/resources/query/sql/query102.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query102.sql b/kylin-it/src/test/resources/query/sql/query102.sql
new file mode 100644
index 0000000..bd1e15e
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query102.sql
@@ -0,0 +1,13 @@
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') and meta_categ_name not in ('Crafts','Computers'))
+ group by meta_categ_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/kylin-it/src/test/resources/query/sql/query103.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query103.sql b/kylin-it/src/test/resources/query/sql/query103.sql
new file mode 100644
index 0000000..c5f9bf9
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query103.sql
@@ -0,0 +1,13 @@
+
+select meta_categ_name, count(1) as cnt, sum(price) as GMV
+
+ from test_kylin_fact
+ left JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ left JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ left JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+ where not ( meta_categ_name not in ('', 'a','Computers') or meta_categ_name not in ('Crafts','Computers'))
+ group by meta_categ_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/af429e5c/kylin-it/src/test/resources/query/sql_like/query21.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query21.sql b/kylin-it/src/test/resources/query/sql_like/query21.sql
new file mode 100644
index 0000000..368a15c
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query21.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select lstg_format_name as lstg_format_name, count(*) as cnt
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where not(lstg_format_name not like '%BIN%')
+group by lstg_format_name
\ No newline at end of file
[18/50] [abbrv] kylin git commit: measure encoding length bug
Posted by li...@apache.org.
measure encoding length bug
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1a5295ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1a5295ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1a5295ba
Branch: refs/heads/master-cdh5.7
Commit: 1a5295bad21fc18766bb3c7ba6eee59678d42185
Parents: 5822492
Author: luguosheng <55...@qq.com>
Authored: Thu Dec 1 17:54:35 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Dec 1 18:17:35 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 4 ++--
webapp/app/js/controllers/cubeEdit.js | 2 +-
webapp/app/js/controllers/cubeMeasures.js | 8 ++++----
3 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index f2b36fe..8192419 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -47,7 +47,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
var _encoding = item.encoding;
var _valueLength ;
var baseKey=item.encoding.replace(/:\d+/,'');
- if(needLengthKeyList.indexOf(baseKey)>=-1){
+ if(needLengthKeyList.indexOf(baseKey)!=-1){
var result=/:(\d+)/.exec(item.encoding);
_valueLength=result?result[1]:0;
}
@@ -78,7 +78,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
var version=$scope.getTypeVersion(item.encoding);
var encodingType=$scope.removeVersion(item.encoding);
- if(needLengthKeyList.indexOf(encodingType)>=-1){
+ if(needLengthKeyList.indexOf(encodingType)!=-1){
encoding = encodingType+":"+item.valueLength;
}else{
encoding = encodingType;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index 0704a62..85bd4b1 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -78,7 +78,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
$scope.store.supportedEncoding = $scope.cubeConfig.encodings;
})
$scope.createFilter=function(type){
- if(type.indexOf("varchar")<=0){
+ if(type.indexOf("varchar")==-1){
return ['fixed_length_hex'];
}else if(type!="date"){
return ['date'];
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a5295ba/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 085338f..18f53c2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -45,7 +45,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
}
}
$scope.createFilter=function(type){
- if(type.indexOf("varchar")<=0){
+ if(type.indexOf("varchar")==-1){
return ['fixed_length_hex'];
}else if(type!="date"){
return ['date'];
@@ -60,7 +60,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
var encodings =$scope.store.supportedEncoding,filterEncoding=[];
var filerList=$scope.createFilter(type);
if($scope.isEdit) {
- if (name && $scope.newMeasure.function.configuration) {
+ if (name && $scope.newMeasure.function.configuration&&$scope.newMeasure.function.configuration['topn.encoding.' + name]) {
var version = $scope.newMeasure.function.configuration['topn.encoding_version.' + name] || 1;
filterEncoding = VdmUtil.getFilterObjectListByOrFilterVal(encodings, 'value', $scope.newMeasure.function.configuration['topn.encoding.' + name].replace(/:\d+/, "") + (version ? "[v" + version + "]" : "[v1]"), 'suggest', true);
}else{
@@ -119,7 +119,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
var version=$scope.newMeasure.function.configuration['topn.encoding_version.'+_name]||1;
item=$scope.removeVersion(item);
var baseKey=item.replace(/:\d+/,'');
- if(needLengthKeyList.indexOf(baseKey)>=-1){
+ if(needLengthKeyList.indexOf(baseKey)!=-1){
var result=/:(\d+)/.exec(item);
_valueLength=result?result[1]:0;
}
@@ -231,7 +231,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
var versionKey='topn.encoding_version.'+item.name;
var version=$scope.getTypeVersion(item.encoding);
var encoding="";
- if(needLengthKeyList.indexOf($scope.removeVersion(item.encoding))>=-1){
+ if(needLengthKeyList.indexOf($scope.removeVersion(item.encoding))!=-1){
encoding = $scope.removeVersion(item.encoding)+":"+item.valueLength;
}else{
encoding = $scope.removeVersion(item.encoding);
[10/50] [abbrv] kylin git commit: Revert "KYLIN-2195 could run
get-properties.sh in source code dir"
Posted by li...@apache.org.
Revert "KYLIN-2195 could run get-properties.sh in source code dir"
This reverts commit 8506e4d4f75fbbda518997cac97e3333229fa5a0.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3186d176
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3186d176
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3186d176
Branch: refs/heads/master-cdh5.7
Commit: 3186d176113596c71a47d98a927564cefff3b123
Parents: 8506e4d
Author: lidongsjtu <li...@apache.org>
Authored: Tue Nov 29 19:11:50 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Nov 29 19:11:50 2016 +0800
----------------------------------------------------------------------
build/bin/get-properties.sh | 12 ++----------
1 file changed, 2 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3186d176/build/bin/get-properties.sh
----------------------------------------------------------------------
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 0fa4f6f..170442d 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -19,21 +19,13 @@
source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
-
if [ $# != 1 ]
then
echo 'invalid input'
exit -1
fi
-if [[ $CI_MODE == 'true' ]]; then
- cd $dir
- job_jar=$(ls ../../assembly/target/kylin-*-job.jar)
- tool_jar=$(ls ../../tool/target/kylin-tool-*.jar|grep -v assembly)
-else
- job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
- tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
-fi
-
+job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
+tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
echo "$result"
[08/50] [abbrv] kylin git commit: KYLIN-1875 Add JoinsTree,
refactor ModelChooser
Posted by li...@apache.org.
KYLIN-1875 Add JoinsTree, refactor ModelChooser
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bda8787d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bda8787d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bda8787d
Branch: refs/heads/master-cdh5.7
Commit: bda8787d1a5c9bbe7d94798091763e1004a1eb61
Parents: a1a2a4a
Author: Yang Li <li...@apache.org>
Authored: Mon Nov 28 07:49:11 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Tue Nov 29 07:25:29 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/DictionaryManager.java | 6 +-
.../kylin/metadata/model/DataModelDesc.java | 30 +-
.../apache/kylin/metadata/model/JoinDesc.java | 11 +
.../apache/kylin/metadata/model/JoinsTree.java | 130 ++++++
.../apache/kylin/metadata/model/TblColRef.java | 4 +
.../kylin/metadata/MetadataManagerTest.java | 11 +-
.../kylin/metadata/model/JoinsTreeTest.java | 74 ++++
.../test_kylin_snowflake_sales_cube.json | 268 ++++++++++++
.../test_kylin_snowflake_model_desc.json | 161 --------
.../test_kylin_snowflake_sales_model.json | 87 ++++
.../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json | 28 ++
.../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json | 408 +++++++++++++++++++
.../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json | 152 +++++++
.../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json | 24 ++
.../localmeta/table/SNOWTEST.KYLIN_SALES.json | 56 +++
.../apache/kylin/query/relnode/OLAPContext.java | 2 +
.../kylin/query/relnode/OLAPTableScan.java | 7 +-
.../kylin/query/routing/ModelChooser.java | 55 +--
18 files changed, 1293 insertions(+), 221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 820299c..6178234 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -339,7 +339,7 @@ public class DictionaryManager {
// find a lookup table that the col joins as FK
for (TableRef lookup : model.getLookupTables()) {
- JoinDesc lookupJoin = model.getPKSideJoinMap().get(lookup);
+ JoinDesc lookupJoin = model.getJoinByPKSide(lookup);
int find = ArrayUtils.indexOf(lookupJoin.getForeignKeyColumns(), col);
if (find < 0)
continue;
@@ -357,8 +357,8 @@ public class DictionaryManager {
if (join.isInnerJoin() == false)
return false;
- TableRef table = join.getForeignKeyColumns()[0].getTableRef();
- join = model.getPKSideJoinMap().get(table);
+ TableRef table = join.getFKSide();
+ join = model.getJoinByPKSide(table);
}
return true;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index ff92def..d917571 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -97,8 +97,7 @@ public class DataModelDesc extends RootPersistentEntity {
private Set<TableRef> allTableRefs = Sets.newLinkedHashSet();
private Map<String, TableRef> aliasMap = Maps.newHashMap(); // alias => TableRef, a table has exactly one alias
private Map<String, TableRef> tableNameMap = Maps.newHashMap(); // name => TableRef, a table maybe referenced by multiple names
- private Map<TableRef, JoinDesc> pkSideJoinMap = Maps.newHashMap(); // table (PK side) => JoinTable
- private Map<String, List<JoinDesc>> fkSideJoinMap = Maps.newHashMap(); // table (FK side) => JoinDesc
+ private JoinsTree joinsTree;
/**
* Error messages during resolving json metadata
@@ -151,12 +150,12 @@ public class DataModelDesc extends RootPersistentEntity {
return joinTables;
}
- public Map<TableRef, JoinDesc> getPKSideJoinMap() {
- return pkSideJoinMap;
+ public JoinDesc getJoinByPKSide(TableRef table) {
+ return joinsTree.getJoinByPKSide(table);
}
-
- public Map<String, List<JoinDesc>> getFKSideJoinMap() {
- return fkSideJoinMap;
+
+ public JoinsTree getJoinsTree() {
+ return joinsTree;
}
@Deprecated
@@ -282,6 +281,7 @@ public class DataModelDesc extends RootPersistentEntity {
initJoinTablesForUpgrade();
initTableAlias(tables);
initJoinColumns();
+ initJoinsTree();
ModelDimensionDesc.capicalizeStrings(dimensions);
initPartitionDesc();
}
@@ -360,8 +360,6 @@ public class DataModelDesc extends RootPersistentEntity {
}
private void initJoinColumns() {
- pkSideJoinMap.clear();
- fkSideJoinMap.clear();
for (JoinTableDesc joinTable : joinTables) {
TableRef dimTable = joinTable.getTableRef();
@@ -413,15 +411,15 @@ public class DataModelDesc extends RootPersistentEntity {
logger.warn("PK " + dimTable + "." + pkCols[i].getName() + "." + pkCols[i].getDatatype() + " are not consistent with FK " + fkTable + "." + fkCols[i].getName() + "." + fkCols[i].getDatatype());
}
}
+ }
+ }
- // pk/fk side join maps
- pkSideJoinMap.put(dimTable, join);
- List<JoinDesc> list = fkSideJoinMap.get(fkTable.getTableIdentity());
- if (list == null) {
- fkSideJoinMap.put(fkTable.getTableIdentity(), list = Lists.newArrayListWithCapacity(4));
- }
- list.add(join);
+ private void initJoinsTree() {
+ List<JoinDesc> joins = new ArrayList<>();
+ for (JoinTableDesc joinTable : joinTables) {
+ joins.add(joinTable.getJoin());
}
+ joinsTree = new JoinsTree(rootFactTableRef, joins);
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
index d3c0745..6489244 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java
@@ -110,6 +110,14 @@ public class JoinDesc {
Preconditions.checkState(tableRef == cols[i].getTableRef());
}
+ public TableRef getPKSide() {
+ return primaryKeyColumns[0].getTableRef();
+ }
+
+ public TableRef getFKSide() {
+ return foreignKeyColumns[0].getTableRef();
+ }
+
public void sortByFK() {
Preconditions.checkState(primaryKey.length == foreignKey.length && primaryKey.length == primaryKeyColumns.length && foreignKey.length == foreignKeyColumns.length);
boolean cont = true;
@@ -174,6 +182,9 @@ public class JoinDesc {
// equals() without alias
public boolean matches(JoinDesc other) {
+ if (other == null)
+ return false;
+
if (!this.type.equalsIgnoreCase(other.getType()))
return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
new file mode 100644
index 0000000..a0b267d
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+public class JoinsTree {
+
+ final Map<String, Chain> tableChains = new LinkedHashMap<>();
+
+ public JoinsTree(TableRef rootTable, List<JoinDesc> joins) {
+ for (JoinDesc join : joins) {
+ for (TblColRef col : join.getForeignKeyColumns())
+ Preconditions.checkState(col.isQualified());
+ for (TblColRef col : join.getPrimaryKeyColumns())
+ Preconditions.checkState(col.isQualified());
+ }
+
+ tableChains.put(rootTable.getAlias(), new Chain(rootTable, null, null));
+
+ for (JoinDesc join : joins) {
+ TableRef pkSide = join.getPKSide();
+ Chain fkSide = tableChains.get(join.getFKSide().getAlias());
+ tableChains.put(pkSide.getAlias(), new Chain(pkSide, join, fkSide));
+ }
+ }
+
+ public Map<String, String> matches(JoinsTree another) {
+ return matches(another, Collections.<String, String> emptyMap());
+ }
+
+ public Map<String, String> matches(JoinsTree another, Map<String, String> constraints) {
+ Map<String, String> matchUp = new HashMap<>();
+
+ for (Chain chain : tableChains.values()) {
+ if (matchInTree(chain, another, constraints, matchUp) == false)
+ return null;
+ }
+
+ return matchUp;
+ }
+
+ private boolean matchInTree(Chain chain, JoinsTree another, Map<String, String> constraints, Map<String, String> matchUp) {
+ String thisAlias = chain.table.getAlias();
+ if (matchUp.containsKey(thisAlias))
+ return true;
+
+ String constraint = constraints.get(thisAlias);
+ if (constraint != null) {
+ return matchChain(chain, another.tableChains.get(constraint), matchUp);
+ }
+
+ for (Chain anotherChain : another.tableChains.values()) {
+ if (matchChain(chain, anotherChain, matchUp)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean matchChain(Chain chain, Chain anotherChain, Map<String, String> matchUp) {
+ String thisAlias = chain.table.getAlias();
+ String anotherAlias = anotherChain.table.getAlias();
+
+ String curMatch = matchUp.get(thisAlias);
+ if (curMatch != null)
+ return curMatch.equals(anotherAlias);
+ if (curMatch == null && matchUp.values().contains(anotherAlias))
+ return false;
+
+ boolean matches = false;
+ if (chain.join == null) {
+ matches = anotherChain.join == null && chain.table.getTableDesc().equals(anotherChain.table.getTableDesc());
+ } else {
+ matches = chain.join.matches(anotherChain.join) && matchChain(chain.fkSide, anotherChain.fkSide, matchUp);
+ }
+
+ if (matches) {
+ matchUp.put(thisAlias, anotherAlias);
+ }
+ return matches;
+ }
+
+ public JoinDesc getJoinByPKSide(TableRef table) {
+ Chain chain = tableChains.get(table.getAlias());
+ if (chain == null)
+ return null;
+ else
+ return chain.join;
+ }
+
+ static class Chain {
+ TableRef table; // pk side
+ JoinDesc join;
+ Chain fkSide;
+
+ public Chain(TableRef table, JoinDesc join, Chain fkSide) {
+ this.table = table;
+ this.join = join;
+ this.fkSide = fkSide;
+ if (join != null) {
+ Preconditions.checkArgument(table == join.getPKSide());
+ Preconditions.checkArgument(fkSide.table == join.getFKSide());
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 2cfbafc..bf8d36b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -123,6 +123,10 @@ public class TblColRef implements Serializable {
return table;
}
+ public boolean isQualified() {
+ return table != null;
+ }
+
public String getTableAlias() {
return table != null ? table.getAlias() : "UNKNOWN_ALIAS";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index fcae8d4..204b4e0 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -76,17 +76,20 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
@Test
public void testSnowflakeDataModel() throws Exception {
- DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_model_desc");
+ DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_sales_model");
Assert.assertTrue(model.getDimensions().size() > 0);
try {
- model.findTable("TEST_KYLIN_COUNTRY");
+ model.findTable("KYLIN_COUNTRY");
Assert.fail();
} catch (IllegalArgumentException ex) {
// excepted
}
- Assert.assertNotNull(model.findColumn("BUYER_COUNTRY"));
- Assert.assertNotNull(model.findColumn("SELLER_COUNTRY"));
+
+ Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
+ Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
+ Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
+ Assert.assertNotNull(model.findColumn("BUYER_ID"));
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
new file mode 100644
index 0000000..f52ef45
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.JoinsTree.Chain;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ */
+public class JoinsTreeTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testBasics() {
+ MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model");
+ JoinsTree joinsTree = model.getJoinsTree();
+
+ Chain chain = joinsTree.tableChains.get("BUYER_COUNTRY");
+ assertTrue(chain.table == model.findTable("BUYER_COUNTRY"));
+ assertTrue(chain.fkSide.table == model.findTable("BUYER_ACCOUNT"));
+ assertTrue(chain.fkSide.fkSide.table == model.findTable("KYLIN_SALES"));
+ assertTrue(chain.fkSide.fkSide.join == null);
+ assertTrue(chain.fkSide.fkSide.fkSide == null);
+ }
+
+ @Test
+ public void testMatch() {
+ MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model");
+ JoinsTree joinsTree = model.getJoinsTree();
+
+ Map<String, String> matchUp = joinsTree.matches(joinsTree);
+ for (Entry<String, String> e : matchUp.entrySet()) {
+ assertTrue(e.getKey().equals(e.getValue()));
+ }
+ assertTrue(model.getAllTables().size() == matchUp.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
new file mode 100644
index 0000000..ee0d68c
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
@@ -0,0 +1,268 @@
+{
+ "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "last_modified" : 1451468470824,
+ "name" : "test_kylin_snowflake_sales_cube",
+ "model_name" : "test_kylin_snowflake_sales_model",
+ "description" : null,
+ "null_string" : null,
+ "dimensions" : [ {
+ "name" : "TRANS_ID",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "TRANS_ID",
+ "derived" : null
+ }, {
+ "name" : "CAL_DT",
+ "table" : "SNOWTEST.KYLIN_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
+ }, {
+ "name" : "META_CATEG_NAME",
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEG_LVL2_NAME",
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEG_LVL3_NAME",
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ }, {
+ "name" : "SELLER_ID",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "SELLER_ID",
+ "derived" : null
+ }, {
+ "name" : "BUYER_ID",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "BUYER_ID",
+ "derived" : null
+ }, {
+ "name" : "ACCOUNT_BUYER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL",
+ "derived" : null
+ }, {
+ "name" : "ACCOUNT_SELLER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL",
+ "derived" : null
+ }, {
+ "name" : "BUYER_COUNTRY",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY",
+ "derived" : null
+ }, {
+ "name" : "SELLER_COUNTRY",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY",
+ "derived" : null
+ }, {
+ "name" : "BUYER_COUNTRY_NAME",
+ "table" : "BUYER_COUNTRY",
+ "column" : "NAME",
+ "derived" : null
+ }, {
+ "name" : "SELLER_COUNTRY_NAME",
+ "table" : "SELLER_COUNTRY",
+ "column" : "NAME",
+ "derived" : null
+ }, {
+ "name" : "OPS_USER_ID",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "OPS_USER_ID",
+ "derived" : null
+ }, {
+ "name" : "OPS_REGION",
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "column" : "OPS_REGION",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
+ }
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ }
+ }, {
+ "name" : "SELLER_CNT_HLL",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
+ },
+ "returntype" : "hllc(10)"
+ }
+ }, {
+ "name" : "TOP_SELLER",
+ "function" : {
+ "expression" : "TOP_N",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
+ }
+ },
+ "returntype" : "topn(100)"
+ }
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "BUYER_ID",
+ "encoding" : "integer:4"
+ }, {
+ "column" : "SELLER_ID",
+ "encoding" : "integer:4"
+ }, {
+ "column" : "TRANS_ID",
+ "encoding" : "integer:4"
+ }, {
+ "column" : "PART_DT",
+ "encoding" : "date"
+ }, {
+ "column" : "LEAF_CATEG_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "META_CATEG_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "CATEG_LVL2_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "CATEG_LVL3_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_COUNTRY.NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_COUNTRY.NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "LSTG_FORMAT_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "LSTG_SITE_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "OPS_USER_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "OPS_REGION",
+ "encoding" : "dict"
+ } ]
+ },
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "F1",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
+ } ]
+ }, {
+ "name" : "F2",
+ "columns" : [ {
+ "qualifier" : "M",
+ "measure_refs" : [ "SELLER_CNT_HLL", "TOP_SELLER" ]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : [ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
+ "mandatory_dims" : [ "PART_DT" ],
+ "joint_dims" : [ [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME" ], [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_COUNTRY.NAME" ],
+ [ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID" ], [ "OPS_USER_ID", "OPS_REGION" ] ]
+ }
+ }, {
+ "includes" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID",
+ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID",
+ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
+ "joint_dims" : [ ]
+ }
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "partition_date_start" : 1325376000000,
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 2,
+ "storage_type" : 2,
+ "override_kylin_properties" : {
+ "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json
deleted file mode 100644
index b06a217..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_model_desc.json
+++ /dev/null
@@ -1,161 +0,0 @@
-{
- "uuid": "ac0f4ee2-1dcb-4b07-a38e-4c298563e0e3",
- "name": "test_kylin_snowflake_model_desc",
- "lookups": [
- {
- "table": "EDW.TEST_CAL_DT",
- "join": {
- "type": "left",
- "primary_key": [
- "CAL_DT"
- ],
- "foreign_key": [
- "CAL_DT"
- ]
- }
- },
- {
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join": {
- "type": "left",
- "primary_key": [
- "LEAF_CATEG_ID",
- "SITE_ID"
- ],
- "foreign_key": [
- "LEAF_CATEG_ID",
- "LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "EDW.TEST_SITES",
- "join": {
- "type": "left",
- "primary_key": [
- "SITE_ID"
- ],
- "foreign_key": [
- "LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "join": {
- "type": "left",
- "primary_key": [
- "SELLER_TYPE_CD"
- ],
- "foreign_key": [
- "SLR_SEGMENT_CD"
- ]
- }
- },
- {
- "table": "DEFAULT.TEST_KYLIN_COUNTRY",
- "alias": "BUYER_COUNTRY",
- "join": {
- "type": "left",
- "primary_key": [
- "COUNTRY"
- ],
- "foreign_key": [
- "BUYER_COUNTRY"
- ]
- }
- },
- {
- "table": "DEFAULT.TEST_KYLIN_COUNTRY",
- "alias": "SELLER_COUNTRY",
- "join": {
- "type": "left",
- "primary_key": [
- "COUNTRY"
- ],
- "foreign_key": [
- "SELLER_COUNTRY"
- ]
- }
- }
- ],
- "dimensions": [
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "lstg_format_name",
- "LSTG_SITE_ID",
- "SLR_SEGMENT_CD",
- "TRANS_ID",
- "CAL_DT",
- "LEAF_CATEG_ID",
- "SELLER_ID",
- "BUYER_COUNTRY",
- "SELLER_COUNTRY"
- ]
- },
- {
- "table": "default.test_category_groupings",
- "columns": [
- "leaf_categ_id",
- "site_id",
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER",
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name"
- ]
- },
- {
- "table": "edw.test_sites",
- "columns": [
- "site_id",
- "site_name",
- "cre_user"
- ]
- },
- {
- "table": "edw.test_seller_type_dim",
- "columns": [
- "seller_type_cd",
- "seller_type_desc"
- ]
- },
- {
- "table": "edw.test_cal_dt",
- "columns": [
- "cal_dt",
- "week_beg_dt"
- ]
- },
- {
- "table": "BUYER_COUNTRY",
- "columns": [
- "country",
- "name"
- ]
- },
- {
- "table": "SELLER_COUNTRY",
- "columns": [
- "country",
- "name"
- ]
- }
- ],
- "metrics": [
- "PRICE",
- "ITEM_COUNT",
- "SELLER_ID"
- ],
- "last_modified": 1422435345352,
- "fact_table": "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition": null,
- "partition_desc": {
- "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start": 0,
- "partition_type": "APPEND"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
new file mode 100644
index 0000000..84c6979
--- /dev/null
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
@@ -0,0 +1,87 @@
+{
+ "uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f",
+ "name" : "test_kylin_snowflake_sales_model",
+ "lookups" : [ {
+ "table" : "SNOWTEST.KYLIN_CAL_DT",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "CAL_DT" ],
+ "foreign_key" : [ "PART_DT" ]
+ }
+ }, {
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
+ "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
+ }
+ }, {
+ "table" : "SNOWTEST.KYLIN_ACCOUNT",
+ "alias" : "BUYER_ACCOUNT",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "ACCOUNT_ID" ],
+ "foreign_key" : [ "BUYER_ID" ]
+ }
+ }, {
+ "table" : "SNOWTEST.KYLIN_ACCOUNT",
+ "alias" : "SELLER_ACCOUNT",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "ACCOUNT_ID" ],
+ "foreign_key" : [ "SELLER_ID" ]
+ }
+ }, {
+ "table" : "SNOWTEST.KYLIN_COUNTRY",
+ "alias" : "BUYER_COUNTRY",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "COUNTRY" ],
+ "foreign_key" : [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY" ]
+ }
+ }, {
+ "table" : "SNOWTEST.KYLIN_COUNTRY",
+ "alias" : "SELLER_COUNTRY",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "COUNTRY" ],
+ "foreign_key" : [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY" ]
+ }
+ }],
+ "dimensions" : [ {
+ "table" : "SNOWTEST.KYLIN_SALES",
+ "columns" : [ "TRANS_ID", "SELLER_ID", "BUYER_ID", "PART_DT", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION" ]
+ }, {
+ "table" : "SNOWTEST.KYLIN_CAL_DT",
+ "columns" : ["CAL_DT", "WEEK_BEG_DT"]
+ }, {
+ "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
+ "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "SITE_ID" ]
+ }, {
+ "table" : "BUYER_ACCOUNT",
+ "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ]
+ }, {
+ "table" : "SELLER_ACCOUNT",
+ "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ]
+ }, {
+ "table" : "BUYER_COUNTRY",
+ "columns" : [ "COUNTRY", "NAME" ]
+ }, {
+ "table" : "SELLER_COUNTRY",
+ "columns" : [ "COUNTRY", "NAME" ]
+ } ],
+ "metrics": [
+ "PRICE",
+ "ITEM_COUNT",
+ "SELLER_ID",
+ "LSTG_FORMAT_NAME"
+ ],
+ "last_modified" : 1422435345362,
+ "fact_table" : "SNOWTEST.KYLIN_SALES",
+ "filter_condition" : null,
+ "partition_desc" : {
+ "partition_date_column" : "SNOWTEST.KYLIN_SALES.PART_DT",
+ "partition_date_start" : 1325376000000,
+ "partition_type" : "APPEND"
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
new file mode 100644
index 0000000..8d39cbc
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
@@ -0,0 +1,28 @@
+{
+ "uuid" : "f386e39e-40d7-44c2-9eb3-41b365632231",
+
+ "name" : "KYLIN_ACCOUNT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "ACCOUNT_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "ACCOUNT_BUYER_LEVEL",
+ "datatype" : "int"
+ }, {
+ "id" : "3",
+ "name" : "ACCOUNT_SELLER_LEVEL",
+ "datatype" : "int"
+ }, {
+ "id" : "4",
+ "name" : "ACCOUNT_COUNTRY",
+ "datatype" : "string"
+ }, {
+ "id" : "5",
+ "name" : "ACCOUNT_CONTACT",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
new file mode 100644
index 0000000..655d4e7
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
@@ -0,0 +1,408 @@
+{
+ "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+
+ "name" : "KYLIN_CAL_DT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "CAL_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "2",
+ "name" : "YEAR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "QTR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "4",
+ "name" : "MONTH_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "5",
+ "name" : "WEEK_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "6",
+ "name" : "AGE_FOR_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "AGE_FOR_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "8",
+ "name" : "AGE_FOR_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "9",
+ "name" : "AGE_FOR_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "10",
+ "name" : "AGE_FOR_DT_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "11",
+ "name" : "AGE_FOR_RTL_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "12",
+ "name" : "AGE_FOR_RTL_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "13",
+ "name" : "AGE_FOR_RTL_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "14",
+ "name" : "AGE_FOR_RTL_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "15",
+ "name" : "AGE_FOR_CS_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "16",
+ "name" : "DAY_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "17",
+ "name" : "DAY_OF_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "18",
+ "name" : "DAY_OF_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "19",
+ "name" : "DAY_OF_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "20",
+ "name" : "DAY_OF_WEEK_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "21",
+ "name" : "WEEK_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "22",
+ "name" : "WEEK_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "23",
+ "name" : "MONTH_OF_QTR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "24",
+ "name" : "MONTH_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "25",
+ "name" : "MONTH_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "26",
+ "name" : "QTR_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "27",
+ "name" : "QTR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "28",
+ "name" : "YEAR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "29",
+ "name" : "YEAR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "30",
+ "name" : "QTR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "MONTH_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "32",
+ "name" : "WEEK_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "33",
+ "name" : "CAL_DT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "34",
+ "name" : "CAL_DT_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "35",
+ "name" : "CAL_DT_SHORT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "36",
+ "name" : "YTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "37",
+ "name" : "QTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "38",
+ "name" : "MTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "39",
+ "name" : "WTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "40",
+ "name" : "SEASON_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "41",
+ "name" : "DAY_IN_YEAR_COUNT",
+ "datatype" : "smallint"
+ }, {
+ "id" : "42",
+ "name" : "DAY_IN_QTR_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "43",
+ "name" : "DAY_IN_MONTH_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "44",
+ "name" : "DAY_IN_WEEK_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "45",
+ "name" : "RTL_YEAR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "46",
+ "name" : "RTL_QTR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "47",
+ "name" : "RTL_MONTH_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "48",
+ "name" : "RTL_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "49",
+ "name" : "CS_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "50",
+ "name" : "CAL_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "51",
+ "name" : "DAY_OF_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "52",
+ "name" : "MONTH_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "53",
+ "name" : "PRD_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "54",
+ "name" : "PRD_FLAG",
+ "datatype" : "string"
+ }, {
+ "id" : "55",
+ "name" : "PRD_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "56",
+ "name" : "PRD_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "57",
+ "name" : "QTR_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "58",
+ "name" : "QTR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "59",
+ "name" : "QTR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "60",
+ "name" : "RETAIL_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "61",
+ "name" : "RETAIL_YEAR",
+ "datatype" : "string"
+ }, {
+ "id" : "62",
+ "name" : "RETAIL_START_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "63",
+ "name" : "RETAIL_WK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "64",
+ "name" : "WEEK_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "65",
+ "name" : "WEEK_NUM_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "66",
+ "name" : "WEEK_BEG_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "67",
+ "name" : "WEEK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "68",
+ "name" : "WEEK_IN_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "69",
+ "name" : "WEEK_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "70",
+ "name" : "WEEK_BEG_END_DESC_MDY",
+ "datatype" : "string"
+ }, {
+ "id" : "71",
+ "name" : "WEEK_BEG_END_DESC_MD",
+ "datatype" : "string"
+ }, {
+ "id" : "72",
+ "name" : "YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "73",
+ "name" : "YEAR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "74",
+ "name" : "CAL_DT_MNS_1YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "75",
+ "name" : "CAL_DT_MNS_2YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "76",
+ "name" : "CAL_DT_MNS_1QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "77",
+ "name" : "CAL_DT_MNS_2QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "78",
+ "name" : "CAL_DT_MNS_1MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "79",
+ "name" : "CAL_DT_MNS_2MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "80",
+ "name" : "CAL_DT_MNS_1WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "81",
+ "name" : "CAL_DT_MNS_2WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "82",
+ "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "83",
+ "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "84",
+ "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "85",
+ "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "86",
+ "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "87",
+ "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "88",
+ "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "89",
+ "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "90",
+ "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "91",
+ "name" : "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "92",
+ "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "93",
+ "name" : "SEASON_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "94",
+ "name" : "YTM_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "95",
+ "name" : "YTQ_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "96",
+ "name" : "YTW_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "97",
+ "name" : "KYLIN_CAL_DT_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "98",
+ "name" : "KYLIN_CAL_DT_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "99",
+ "name" : "KYLIN_CAL_DT_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "100",
+ "name" : "KYLIN_CAL_DT_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
new file mode 100644
index 0000000..abee6d1
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
@@ -0,0 +1,152 @@
+{
+ "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
+
+ "name" : "KYLIN_CATEGORY_GROUPINGS",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "LEAF_CATEG_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "LEAF_CATEG_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "3",
+ "name" : "SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "4",
+ "name" : "CATEG_BUSN_MGR",
+ "datatype" : "string"
+ }, {
+ "id" : "5",
+ "name" : "CATEG_BUSN_UNIT",
+ "datatype" : "string"
+ }, {
+ "id" : "6",
+ "name" : "REGN_CATEG",
+ "datatype" : "string"
+ }, {
+ "id" : "7",
+ "name" : "USER_DEFINED_FIELD1",
+ "datatype" : "string"
+ }, {
+ "id" : "8",
+ "name" : "USER_DEFINED_FIELD3",
+ "datatype" : "string"
+ }, {
+ "id" : "9",
+ "name" : "KYLIN_GROUPINGS_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "10",
+ "name" : "KYLIN_GROUPINGS_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "11",
+ "name" : "KYLIN_GROUPINGS_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "12",
+ "name" : "KYLIN_GROUPINGS_UPD_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "13",
+ "name" : "META_CATEG_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "14",
+ "name" : "META_CATEG_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "15",
+ "name" : "CATEG_LVL2_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "16",
+ "name" : "CATEG_LVL3_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "17",
+ "name" : "CATEG_LVL4_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "18",
+ "name" : "CATEG_LVL5_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "19",
+ "name" : "CATEG_LVL6_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "20",
+ "name" : "CATEG_LVL7_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "21",
+ "name" : "CATEG_LVL2_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "22",
+ "name" : "CATEG_LVL3_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "23",
+ "name" : "CATEG_LVL4_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "24",
+ "name" : "CATEG_LVL5_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "25",
+ "name" : "CATEG_LVL6_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "26",
+ "name" : "CATEG_LVL7_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "27",
+ "name" : "CATEG_FLAGS",
+ "datatype" : "decimal"
+ }, {
+ "id" : "28",
+ "name" : "ADULT_CATEG_YN",
+ "datatype" : "string"
+ }, {
+ "id" : "29",
+ "name" : "DOMAIN_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "30",
+ "name" : "USER_DEFINED_FIELD5",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "VCS_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "32",
+ "name" : "GCS_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "33",
+ "name" : "MOVE_TO",
+ "datatype" : "decimal"
+ }, {
+ "id" : "34",
+ "name" : "SAP_CATEGORY_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "35",
+ "name" : "SRC_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "36",
+ "name" : "BSNS_VRTCL_NAME",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
new file mode 100644
index 0000000..64708c7
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
@@ -0,0 +1,24 @@
+{
+ "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882",
+
+ "name" : "KYLIN_COUNTRY",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "COUNTRY",
+ "datatype" : "string"
+ }, {
+ "id" : "2",
+ "name" : "LATITUDE",
+ "datatype" : "double"
+ }, {
+ "id" : "3",
+ "name" : "LONGITUDE",
+ "datatype" : "double"
+ }, {
+ "id" : "4",
+ "name" : "NAME",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
new file mode 100644
index 0000000..4aa82b5
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
@@ -0,0 +1,56 @@
+{
+ "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
+
+ "name" : "KYLIN_SALES",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "TRANS_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "PART_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "LSTG_FORMAT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "4",
+ "name" : "LEAF_CATEG_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "5",
+ "name" : "LSTG_SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "6",
+ "name" : "SLR_SEGMENT_CD",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "PRICE",
+ "datatype" : "decimal(19,4)"
+ }, {
+ "id" : "8",
+ "name" : "ITEM_COUNT",
+ "datatype" : "bigint"
+ }, {
+ "id" : "9",
+ "name" : "SELLER_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "10",
+ "name" : "BUYER_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "11",
+ "name" : "OPS_USER_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "12",
+ "name" : "OPS_REGION",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 615d802..5fb2c5c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -33,6 +33,7 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinsTree;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
@@ -127,6 +128,7 @@ public class OLAPContext {
public Set<TblColRef> filterColumns = new HashSet<>();
public TupleFilter filter;
public List<JoinDesc> joins = new LinkedList<>();
+ public JoinsTree joinsTree;
private List<TblColRef> sortColumns;
private List<SQLDigest.OrderEnum> sortOrders;
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 3c2bf48..14758c9 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -239,10 +239,13 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
return new ColumnRowType(columns);
}
+ public TableRef getTableRef() {
+ return columnRowType.getColumnByIndex(0).getTableRef();
+ }
+
@SuppressWarnings("deprecation")
public TblColRef makeRewriteColumn(String name) {
- TableRef tableRef = columnRowType.getColumnByIndex(0).getTableRef();
- return tableRef.makeFakeColumn(name);
+ return getTableRef().makeFakeColumn(name);
}
public void fixColumnRowTypeWithModel(DataModelDesc model, Map<String, String> aliasMap) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/bda8787d/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
index f1f5a48..ccb2900 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/ModelChooser.java
@@ -30,6 +30,7 @@ import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.JoinsTree;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
@@ -38,7 +39,7 @@ import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.relnode.OLAPTableScan;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
-import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -73,47 +74,31 @@ public class ModelChooser {
private static Map<String, String> matches(DataModelDesc model, List<OLAPContext> contexts) {
Map<String, String> result = Maps.newHashMap();
- // the greedy match is not perfect but works for the moment
- Map<String, List<JoinDesc>> modelJoinsMap = model.getFKSideJoinMap();
for (OLAPContext ctx : contexts) {
- for (JoinDesc queryJoin : ctx.joins) {
- String fkTable = queryJoin.getForeignKeyColumns()[0].getTable();
- List<JoinDesc> modelJoins = modelJoinsMap.get(fkTable);
- if (modelJoins == null)
- return null;
-
- JoinDesc matchJoin = null;
- for (JoinDesc modelJoin : modelJoins) {
- if (modelJoin.matches(queryJoin)) {
- matchJoin = modelJoin;
- break;
- }
- }
- if (matchJoin == null)
- return null;
+ TableRef firstTable = ctx.firstTableScan.getTableRef();
- matchesAdd(queryJoin.getForeignKeyColumns()[0].getTableAlias(), matchJoin.getForeignKeyColumns()[0].getTableAlias(), result);
- matchesAdd(queryJoin.getPrimaryKeyColumns()[0].getTableAlias(), matchJoin.getPrimaryKeyColumns()[0].getTableAlias(), result);
- }
-
- OLAPTableScan firstTable = ctx.firstTableScan;
- String firstTableAlias = firstTable.getAlias();
- if (result.containsKey(firstTableAlias) == false) {
- TableRef tableRef = model.findFirstTable(firstTable.getOlapTable().getTableName());
- if (tableRef == null)
- return null;
- matchesAdd(firstTableAlias, tableRef.getAlias(), result);
+ Map<String, String> matchUp = null;
+
+ if (ctx.joins.isEmpty() && model.isLookupTable(firstTable.getTableIdentity())) {
+ // one lookup table
+ String modelAlias = model.findFirstTable(firstTable.getTableIdentity()).getAlias();
+ matchUp = ImmutableMap.of(firstTable.getAlias(), modelAlias);
+ } else {
+ // normal big joins
+ if (ctx.joinsTree == null) {
+ ctx.joinsTree = new JoinsTree(firstTable, ctx.joins);
+ }
+ matchUp = ctx.joinsTree.matches(model.getJoinsTree(), result);
}
+
+ if (matchUp == null)
+ return null;
+
+ result.putAll(matchUp);
}
-
return result;
}
- private static void matchesAdd(String origAlias, String targetAlias, Map<String, String> result) {
- String existingTarget = result.put(origAlias, targetAlias);
- Preconditions.checkState(existingTarget == null || existingTarget.equals(targetAlias));
- }
-
private static Map<DataModelDesc, Set<IRealization>> makeOrderedModelMap(List<OLAPContext> contexts) {
// the first context, which is the top most context, contains all columns from all contexts
OLAPContext first = contexts.get(0);
[15/50] [abbrv] kylin git commit:
org.apache.kylin.metadata.model.FunctionDesc#configuration should use default
linkedhashmap to keep order
Posted by li...@apache.org.
org.apache.kylin.metadata.model.FunctionDesc#configuration should use default linkedhashmap to keep order
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/93bf0d02
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/93bf0d02
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/93bf0d02
Branch: refs/heads/master-cdh5.7
Commit: 93bf0d0214fe446d794b61e02fb18ff8097611d1
Parents: 30cb1ac
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Dec 1 11:20:53 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Dec 1 11:40:25 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/JacksonBean.java | 16 ++++++++++++++++
.../apache/kylin/common/util/JacksonTest.java | 19 +++++++++++++++----
.../kylin/metadata/model/FunctionDesc.java | 8 ++++----
3 files changed, 35 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
index 42357f2..ffff6d1 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -23,6 +23,10 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class JacksonBean {
@@ -32,6 +36,10 @@ public class JacksonBean {
@JsonInclude(JsonInclude.Include.NON_NULL)
private int b;
+ @JsonProperty("configuration")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ private Map<String, String> configuration = new HashMap<String, String>();
+
public String getA() {
return a;
}
@@ -48,6 +56,14 @@ public class JacksonBean {
this.b = b;
}
+ public Map<String, String> getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Map<String, String> configuration) {
+ this.configuration = configuration;
+ }
+
@Override
public String toString() {
return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}';
http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
index 81be7eb..0d7097a 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -18,22 +18,33 @@
package org.apache.kylin.common.util;
-import java.io.IOException;
-
import org.junit.Test;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
public class JacksonTest {
@Test
public void foo() throws IOException {
+ HashMap a = new HashMap<String, String>();
+ a.put("1", "1");
+ a.put("3", "3");
+ a.put("2", "2");
+
+
JacksonBean bean = new JacksonBean();
bean.setA("valuea");
+ bean.setConfiguration(a);
String s = JsonUtil.writeValueAsString(bean);
System.out.println(s);
- JacksonBean desBean = (JacksonBean) JsonUtil.readValue("{\"a\":\"valuea\"}", JacksonBean.class);
+ JacksonBean desBean = (JacksonBean) JsonUtil.readValue("{\"a\":\"valuea\",\"b\":0,\"configuration\":{\"2\":\"2\",\"3\":\"3\",\"1\":\"1\"}}", JacksonBean.class);
+
String x2 = JsonUtil.writeValueAsString(desBean);
- System.out.println(desBean);
System.out.println(x2);
+
+ System.out.println(desBean);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/93bf0d02/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index ae7f805..7b41552 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -19,8 +19,8 @@
package org.apache.kylin.metadata.model;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.LinkedHashMap;
+import java.util.Map;
import java.util.Set;
import org.apache.kylin.measure.MeasureType;
@@ -67,7 +67,7 @@ public class FunctionDesc {
@JsonProperty("configuration")
@JsonInclude(JsonInclude.Include.NON_EMPTY)
- private HashMap<String, String> configuration = new LinkedHashMap<String, String>();
+ private Map<String, String> configuration = new LinkedHashMap<String, String>();
private DataType returnDataType;
private MeasureType<?> measureType;
@@ -238,11 +238,11 @@ public class FunctionDesc {
this.returnDataType = DataType.getType(returnType);
}
- public HashMap<String, String> getConfiguration() {
+ public Map<String, String> getConfiguration() {
return configuration;
}
- public void setConfiguration(HashMap<String, String> configurations) {
+ public void setConfiguration(Map<String, String> configurations) {
this.configuration = configurations;
}
[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
Posted by li...@apache.org.
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/469d9cc5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/469d9cc5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/469d9cc5
Branch: refs/heads/master-cdh5.7
Commit: 469d9cc5c7df0a648d652635a7dc6b9ef4383182
Parents: 081ed0d
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Dec 7 12:28:13 2016 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/hbase-site.xml | 19 +---
.../kylin/provision/BuildCubeWithEngine.java | 12 +--
pom.xml | 12 +--
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 ++++----------------
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 +++---
.../apache/kylin/rest/service/CubeService.java | 35 +++-----
.../apache/kylin/rest/service/QueryService.java | 24 +++--
.../apache/kylin/rest/service/UserService.java | 17 ++--
.../kylin/storage/hbase/HBaseConnection.java | 44 ++++-----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++----
.../kylin/storage/hbase/HBaseStorage.java | 3 +-
.../storage/hbase/cube/SimpleHBaseStore.java | 20 ++---
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
.../storage/hbase/cube/v1/CubeStorageQuery.java | 4 +-
.../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
.../cube/v1/SerializedHBaseTupleIterator.java | 4 +-
.../observer/AggregateRegionObserver.java | 4 +-
.../observer/AggregationScanner.java | 14 ++-
.../observer/ObserverAggregationCache.java | 10 ++-
.../coprocessor/observer/ObserverEnabler.java | 4 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
.../storage/hbase/steps/DeprecatedGCStep.java | 23 ++---
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++---
.../storage/hbase/util/CleanHtableCLI.java | 12 +--
.../storage/hbase/util/CubeMigrationCLI.java | 36 ++++----
.../hbase/util/CubeMigrationCheckCLI.java | 17 ++--
.../hbase/util/DeployCoprocessorCLI.java | 27 +++---
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 +++----
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++--
.../hbase/util/HBaseRegionSizeCalculator.java | 35 ++++----
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 +++----
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
.../storage/hbase/util/StorageCleanupJob.java | 20 +++--
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
.../observer/AggregateRegionObserverTest.java | 26 ++----
.../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
50 files changed, 417 insertions(+), 472 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
- <property>
- <name>hbase.client.pause</name>
- <value>100</value>
- <description>General client pause value. Used mostly as value to wait
- before running a retry of a failed get, region lookup, etc.
- See hbase.client.retries.number for description of how we backoff from
- this initial pause amount and how this pause works w/ retries.</description>
- </property>
- <property>
- <name>hbase.client.retries.number</name>
- <value>5</value>
- <description>Maximum retries. Used as maximum for all retryable
- operations such as the getting of a cell's value, starting a row update,
- etc. Retry interval is a rough function based on hbase.client.pause. At
- first we retry at this interval but then with backoff, we pretty quickly reach
- retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
- ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 67b62d5..8bd1d67 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -35,8 +35,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -58,6 +57,7 @@ import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.apache.kylin.tool.StorageCleanupJob;
@@ -431,10 +431,10 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
- String tableName = segment.getStorageLocationIdentifier();
- try (HTable table = new HTable(conf, tableName)) {
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
+ String tableName = segment.getStorageLocationIdentifier();
+
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 51479c8..6d3425e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,20 +46,20 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.6.0</hadoop2.version>
- <yarn.version>2.6.0</yarn.version>
+ <hadoop2.version>2.7.1</hadoop2.version>
+ <yarn.version>2.7.1</yarn.version>
<!-- Hive versions -->
- <hive.version>0.14.0</hive.version>
- <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
+ <hive.version>1.2.1</hive.version>
+ <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
<kafka.version>0.10.0.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.6.0</curator.version>
+ <curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index ea68855..8095bf8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
/**
*/
@@ -36,6 +36,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- HTableInterface getTable(String tableName) throws IOException;
+ Table getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index d9326f5..cc76b87 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
+ private Table mockedAclTable;
+ private Table mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index d0aa0ed..972eea9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements HTableInterface {
+public class MockHTable implements Table {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,14 +114,6 @@ public class MockHTable implements HTableInterface {
this.columnFamilies.add(columnFamily);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[] getTableName() {
- return tableName.getBytes();
- }
-
@Override
public TableName getName() {
return null;
@@ -200,8 +192,8 @@ public class MockHTable implements HTableInterface {
}
@Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- return new Boolean[0];
+ public boolean[] existsAll(List<Get> list) throws IOException {
+ return new boolean[0];
}
/**
@@ -306,15 +298,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- // FIXME: implement
- return null;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -446,7 +429,7 @@ public class MockHTable implements HTableInterface {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow(nkvs);
+ filter.filterRow();
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -535,6 +518,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -555,7 +543,7 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDeleteFamily()) {
+ if (kv.isDelete()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -592,6 +580,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -605,7 +598,7 @@ public class MockHTable implements HTableInterface {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, true);
+ return incrementColumnValue(row, family, qualifier, amount, null);
}
@Override
@@ -617,37 +610,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if (check(row, family, qualifier, null)) {
- Put put = new Put(row);
- put.add(family, qualifier, Bytes.toBytes(amount));
- put(put);
- return amount;
- }
- long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
- data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
- return newValue;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isAutoFlush() {
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void flushCommits() throws IOException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public void close() throws IOException {
}
@@ -673,29 +635,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public void setAutoFlush(boolean autoFlush) {
- throw new NotImplementedException();
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- throw new NotImplementedException();
-
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- throw new NotImplementedException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index 1d520c4..d1a1384 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,7 +21,8 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -58,11 +59,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index d693a67..3e3efec 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- HTableInterface htable = null;
+ Table htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,17 +226,16 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -250,7 +249,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -266,7 +265,6 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -284,7 +282,7 @@ public class AclService implements MutableAclService {
throw e;
}
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -295,17 +293,16 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 85c9284..c0f8e6f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -28,9 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.WeakHashMap;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -406,33 +404,24 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- HTable table = null;
+ Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- try {
- table = new HTable(hconf, tableName);
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
-
- regionCount = sizeMap.size();
-
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
- } finally {
- IOUtils.closeQuietly(table);
+ for (long s : sizeMap.values()) {
+ tableSize += s;
}
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 8810c85..2c031cf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -47,11 +47,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
@@ -161,14 +161,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -194,14 +193,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -213,12 +211,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- HTableInterface htable = null;
+ Table htable = null;
try {
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = conn.getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 07c7c6f..ab54882 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
- put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,14 +163,13 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -185,7 +184,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -216,7 +215,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- HTableInterface htable = null;
+ Table htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..b769391 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -64,7 +64,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
+ private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -75,7 +75,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (HConnection conn : connPool.values()) {
+ for (Connection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -144,7 +144,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -213,9 +213,9 @@ public class HBaseConnection {
// ============================================================================
- // returned HConnection can be shared by multiple threads and does not require close()
+ // returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static HConnection get(String url) {
+ public static Connection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -223,13 +223,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- HConnection connection = connPool.get(url);
+ Connection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = HConnectionManager.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
connPool.put(url, connection);
}
@@ -248,8 +248,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static boolean tableExists(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -269,18 +269,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
+ public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
+ Admin hbase = conn.getAdmin();
+ TableName tableName = TableName.valueOf(table);
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(table, newFamilyDescriptor(family));
+ hbase.addColumn(tableName, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -333,8 +333,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void deleteTable(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (!tableExists(conn, tableName)) {
@@ -344,10 +344,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(tableName)) {
- hbase.disableTable(tableName);
+ if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
+ hbase.disableTable(TableName.valueOf(tableName));
}
- hbase.deleteTable(tableName);
+ hbase.deleteTable(TableName.valueOf(tableName));
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 1d19983..714a265 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -31,14 +31,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +70,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- HConnection getConnection() throws IOException {
+ Connection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -120,7 +121,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,13 +239,12 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -252,7 +252,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -265,8 +265,6 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
- table.flushCommits();
-
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -275,7 +273,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -288,7 +286,6 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
- table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -310,7 +307,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -318,7 +315,7 @@ public class HBaseResourceStore extends ResourceStore {
}
}
- private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -337,7 +334,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
@@ -363,7 +360,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -371,8 +368,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.addColumn(B_FAMILY, B_COLUMN, content);
+ put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index 43b65cb..d36d722 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -18,7 +18,6 @@
package org.apache.kylin.storage.hbase;
-import com.google.common.base.Preconditions;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
@@ -36,6 +35,8 @@ import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition;
+import com.google.common.base.Preconditions;
+
@SuppressWarnings("unused")
//used by reflection
public class HBaseStorage implements IStorage {
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b141190..f63d9c2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,12 +26,13 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -86,14 +87,13 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final HTableInterface table;
+ final BufferedMutator table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getTable(htableName);
- table.setAutoFlush(false, true);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getBufferedMutator(htableName);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.put(put);
+ table.mutate(put);
}
@Override
public void close() throws IOException {
- table.flushCommits();
+ table.flush();
table.close();
}
}
class Reader implements IGTScanner {
- final HTableInterface table;
+ final Table table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 8ac3832..982a044 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -25,11 +25,12 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
@@ -70,7 +71,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
protected final List<RowValueDecoder> rowValueDecoders;
private final StorageContext context;
private final String tableName;
- private final HTableInterface table;
+ private final Table table;
protected CubeTupleConverter tupleConverter;
protected final Iterator<HBaseKeyRange> rangeIterator;
@@ -88,7 +89,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
private int advMeasureRowsRemaining;
private int advMeasureRowIndex;
- public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, HConnection conn, //
+ public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, Connection conn, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, //
List<RowValueDecoder> rowValueDecoders, StorageContext context, TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
@@ -108,7 +109,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
this.rangeIterator = keyRanges.iterator();
try {
- this.table = conn.getTable(tableName);
+ this.table = conn.getTable(TableName.valueOf(tableName));
} catch (Throwable t) {
throw new StorageException("Error when open connection to table " + tableName, t);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 02aa64a..ea5a393 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -33,7 +33,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Dictionary;
@@ -156,7 +156,7 @@ public class CubeStorageQuery implements IStorageQuery {
setCoprocessor(groupsCopD, valueDecoders, context); // enable coprocessor if beneficial
setLimit(filter, context);
- HConnection conn = HBaseConnection.get(context.getConnUrl());
+ Connection conn = HBaseConnection.get(context.getConnUrl());
// notice we're passing filterD down to storage instead of flatFilter
return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context, returnTupleInfo);
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 8a20c65..3d30767 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
/**
* @author yangli9
@@ -50,7 +51,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -60,7 +61,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -94,4 +95,9 @@ public class RegionScannerAdapter implements RegionScanner {
return Long.MAX_VALUE;
}
+ @Override
+ public int getBatch() {
+ return -1;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
index e8dd5b9..d033c77 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -57,7 +57,7 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
private int scanCount;
private ITuple next;
- public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
+ public SerializedHBaseTupleIterator(Connection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, //
StorageContext context, TupleInfo returnTupleInfo) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index 7139ca7..7e25e4c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.gridtable.StorageSideBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
- HRegion region = ctxt.getEnvironment().getRegion();
+ Region region = ctxt.getEnvironment().getRegion();
region.startRegionOperation();
try {
synchronized (innerScanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index a900ea1..d64f48f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
@@ -116,8 +117,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- return outerScanner.next(result, limit);
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.next(result, scannerContext);
}
@Override
@@ -126,8 +127,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return outerScanner.nextRaw(result, limit);
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.nextRaw(result, scannerContext);
}
@Override
@@ -160,6 +161,11 @@ public class AggregationScanner implements RegionScanner {
return outerScanner.getMvccReadPoint();
}
+ @Override
+ public int getBatch() {
+ return outerScanner.getBatch();
+ }
+
private static class Stats {
long inputRows = 0;
long inputBytes = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index 8404262..331e34d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
@@ -112,7 +113,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -122,7 +123,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -161,6 +162,11 @@ public class ObserverAggregationCache extends AggregationCache {
// AggregateRegionObserver.LOG.info("Kylin Scanner getMvccReadPoint()");
return Long.MAX_VALUE;
}
+
+ @Override
+ public int getBatch() {
+ return innerScanner.getBatch();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
index 394b3e2..9fd33f5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
@@ -23,9 +23,9 @@ import java.util.Collection;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -60,7 +60,7 @@ public class ObserverEnabler {
static final Map<String, Boolean> CUBE_OVERRIDES = Maps.newConcurrentMap();
public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, //
- Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, HTableInterface table, Scan scan) throws IOException {
+ Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, Table table, Scan scan) throws IOException {
if (context.isCoprocessorEnabled() == false) {
return table.getScanner(scan);
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index d99f80e..f879e2b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,8 +26,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -50,10 +51,10 @@ import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,7 +117,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -171,7 +172,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final boolean[] abnormalFinish = new boolean[1];
try {
- HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
+ Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 3cefc5f..a52af90 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -24,11 +24,12 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -154,8 +155,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index da9c932..f0949da 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -142,7 +142,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -179,7 +179,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 2814ad6..feb4842 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -26,7 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -79,7 +80,8 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin admin = conn.getAdmin();
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -92,7 +94,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(tableName)) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -101,7 +103,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
IOUtils.closeQuietly(admin);
@@ -110,8 +112,7 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -126,8 +127,7 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 46a828e..2d1c03a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -29,9 +29,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.exception.ExecuteException;
@@ -100,19 +101,21 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index d5b36df..6587d4e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -49,7 +49,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final HTableInterface hTable;
+ private final Table hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -58,7 +58,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -117,7 +117,6 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
- hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
[44/50] [abbrv] kylin git commit: KYLIN-1971 ColumnDesc.equals() bug
fix
Posted by li...@apache.org.
KYLIN-1971 ColumnDesc.equals() bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a53eafaa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a53eafaa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a53eafaa
Branch: refs/heads/master-cdh5.7
Commit: a53eafaa30d5bb8bf5b52404bb6bbc1dcd9673e8
Parents: b5c0588
Author: Li Yang <li...@apache.org>
Authored: Tue Dec 6 18:39:03 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Dec 6 18:39:03 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a53eafaa/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 7d9133d..403eaaf 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -194,7 +194,7 @@ public class ColumnDesc implements Serializable {
if (table == null) {
if (other.table != null)
return false;
- } else if (!table.equals(other.table))
+ } else if (!table.getIdentity().equals(other.table.getIdentity()))
return false;
if (datatype == null) {
[02/50] [abbrv] kylin git commit: KYLIN-2195 support
backward-compatibility properties in get-properties.sh
Posted by li...@apache.org.
KYLIN-2195 support backward-compatibility properties in get-properties.sh
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1448e57
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1448e57
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1448e57
Branch: refs/heads/master-cdh5.7
Commit: b1448e5789ce65d5b94ed246df01fa4c269515b2
Parents: a2ecf18
Author: lidongsjtu <li...@apache.org>
Authored: Mon Nov 28 23:37:41 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Nov 28 23:37:41 2016 +0800
----------------------------------------------------------------------
build/bin/get-properties.sh | 4 +-
.../org/apache/kylin/tool/KylinConfigCLI.java | 42 ++++++++++++++++++++
2 files changed, 45 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b1448e57/build/bin/get-properties.sh
----------------------------------------------------------------------
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 1a086ea..170442d 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -25,5 +25,7 @@ then
exit -1
fi
-result=`cat ${KYLIN_HOME}/conf/kylin.properties | grep -w "^$1" | grep -v '^#' | awk -F= '{ n = index($0,"="); print substr($0,n+1)}' | cut -c 1- |tail -1`
+job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
+tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
+result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
echo "$result"
http://git-wip-us.apache.org/repos/asf/kylin/blob/b1448e57/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
new file mode 100644
index 0000000..e1a5b99
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.tool;
+
+import java.util.Properties;
+
+import org.apache.kylin.common.BackwardCompatibilityConfig;
+import org.apache.kylin.common.KylinConfig;
+
+public class KylinConfigCLI {
+ public static void main(String[] args) {
+ if (args.length != 1) {
+ System.err.println("Usage: KylinConfigCLI conf_name");
+ System.err.println("Example: KylinConfigCLI kylin.server.mode");
+ System.exit(1);
+ }
+
+ Properties config = KylinConfig.getKylinProperties();
+ BackwardCompatibilityConfig bcc = new BackwardCompatibilityConfig();
+ String value = config.getProperty(bcc.check(args[0]));
+ if (value == null) {
+ value = "";
+ }
+ System.out.println(value);
+ }
+}
[45/50] [abbrv] kylin git commit: KYLIN-2251: fix httpcore dependency
conflict in JDBC Driver
Posted by li...@apache.org.
KYLIN-2251: fix httpcore dependency conflict in JDBC Driver
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea13af4d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea13af4d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea13af4d
Branch: refs/heads/master-cdh5.7
Commit: ea13af4d8aa6a5b25fc328ee1adaf2b454eddea0
Parents: a53eafa
Author: Billy Liu <bi...@apache.org>
Authored: Wed Dec 7 09:01:52 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Dec 7 09:01:52 2016 +0800
----------------------------------------------------------------------
jdbc/pom.xml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea13af4d/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 4f42913..22d8eca 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -39,14 +39,14 @@
<dependencies>
<dependency>
- <groupId>org.apache.calcite.avatica</groupId>
- <artifactId>avatica</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica</artifactId>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
[43/50] [abbrv] kylin git commit: KYLIN-2250 HiveMRInput accepts
intermediate_view_table_name
Posted by li...@apache.org.
KYLIN-2250 HiveMRInput accepts intermediate_view_table_name
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b5c05880
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b5c05880
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b5c05880
Branch: refs/heads/master-cdh5.7
Commit: b5c058801b916ddc6e00085d652ad6a0497ce7d9
Parents: 7699973
Author: Cheng Wang <ch...@kyligence.io>
Authored: Tue Dec 6 15:48:21 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Dec 6 16:46:40 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../apache/kylin/cube/cli/DictionaryGeneratorCLI.java | 2 +-
.../org/apache/kylin/metadata/model/TableDesc.java | 14 +++++++++-----
.../org/apache/kylin/source/hive/HiveMRInput.java | 13 +++++++++----
4 files changed, 20 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 0c80c07..bda1423 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -274,7 +274,7 @@ public class CubeManager implements IRealizationProvider {
SnapshotManager snapshotMgr = getSnapshotManager();
TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable));
- if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+ if (tableDesc.isView()) {
String tableName = tableDesc.getMaterializedName();
tableDesc.setDatabase(config.getHiveDatabaseForIntermediateTable());
tableDesc.setName(tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index 163c6ca..3e1ab0d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -98,7 +98,7 @@ public class DictionaryGeneratorCLI {
} else {
MetadataManager metadataManager = MetadataManager.getInstance(config);
TableDesc tableDesc = new TableDesc(metadataManager.getTableDesc(srcTable));
- if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+ if (tableDesc.isView()) {
TableDesc materializedTbl = new TableDesc();
materializedTbl.setDatabase(config.getHiveDatabaseForIntermediateTable());
materializedTbl.setName(tableDesc.getMaterializedName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 659de07..ab8c465 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -26,8 +26,8 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.StringSplitter;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
/**
* Table Metadata from Source. All name should be uppercase.
@@ -36,7 +36,9 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class TableDesc extends RootPersistentEntity implements ISourceAware {
- public static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW";
+ private static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW";
+ private static final String materializedTableNamePrefix = "kylin_intermediate_";
+
@JsonProperty("name")
private String name;
@JsonProperty("columns")
@@ -46,8 +48,6 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
@JsonProperty("table_type")
private String tableType;
- private static final String materializedTableNamePrefix = "kylin_intermediate_";
-
private DatabaseDesc database = new DatabaseDesc();
private String identity = null;
@@ -96,6 +96,10 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
}
return identity;
}
+
+ public boolean isView() {
+ return TABLE_TYPE_VIRTUAL_VIEW.equals(tableType);
+ }
public static String concatResourcePath(String tableIdentity) {
return ResourceStore.TABLE_RESOURCE_ROOT + "/" + tableIdentity + ".json";
@@ -211,7 +215,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
return false;
if (!Arrays.equals(columns, tableDesc.columns))
return false;
-
+
return getIdentity().equals(tableDesc.getIdentity());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b5c05880/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index aa603d7..fc2b982 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -60,6 +60,11 @@ import com.google.common.collect.Sets;
public class HiveMRInput implements IMRInput {
+ public static String getTableNameForHCat(TableDesc table) {
+ String tableName = table.isView() ? table.getMaterializedName() : table.getName();
+ return String.format("%s.%s", table.getDatabase(), tableName).toUpperCase();
+ }
+
@Override
public IMRBatchCubingInputSide getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
return new BatchCubingInputSide(flatDesc);
@@ -67,9 +72,9 @@ public class HiveMRInput implements IMRInput {
@Override
public IMRTableInputFormat getTableInputFormat(TableDesc table) {
- return new HiveTableInputFormat(table.getIdentity());
+ return new HiveTableInputFormat(getTableNameForHCat(table));
}
-
+
@Override
public IMRBatchMergeInputSide getBatchMergeInputSide(ISegment seg) {
return new IMRBatchMergeInputSide() {
@@ -167,7 +172,7 @@ public class HiveMRInput implements IMRInput {
for (JoinTableDesc lookupDesc : flatDesc.getDataModel().getJoinTables()) {
TableDesc tableDesc = metadataManager.getTableDesc(lookupDesc.getTable());
- if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(tableDesc.getTableType())) {
+ if (tableDesc.isView()) {
lookupViewsTables.add(tableDesc);
}
}
@@ -180,7 +185,7 @@ public class HiveMRInput implements IMRInput {
hiveCmdBuilder.addStatement(useDatabaseHql);
hiveCmdBuilder.addStatement(JoinedFlatTable.generateHiveSetStatements(conf));
for (TableDesc lookUpTableDesc : lookupViewsTables) {
- if (TableDesc.TABLE_TYPE_VIRTUAL_VIEW.equalsIgnoreCase(lookUpTableDesc.getTableType())) {
+ if (lookUpTableDesc.isView()) {
StringBuilder createIntermediateTableHql = new StringBuilder();
createIntermediateTableHql.append("DROP TABLE IF EXISTS " + lookUpTableDesc.getMaterializedName() + ";\n");
createIntermediateTableHql.append("CREATE TABLE IF NOT EXISTS " + lookUpTableDesc.getMaterializedName() + "\n");
[30/50] [abbrv] kylin git commit: minor: normalize scientific decimal
values
Posted by li...@apache.org.
minor: normalize scientific decimal values
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fe9efa86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fe9efa86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fe9efa86
Branch: refs/heads/master-cdh5.7
Commit: fe9efa86cd34da31ec41dc176140ee4ee2f49fed
Parents: eb686a9
Author: Hongbin Ma <ma...@apache.org>
Authored: Sun Dec 4 21:26:18 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Dec 4 21:26:18 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/metadata/tuple/Tuple.java | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fe9efa86/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
index 721a719..d51ae9e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/tuple/Tuple.java
@@ -115,6 +115,8 @@ public class Tuple implements ITuple {
// BigDecimal during cube build for best precision
if ("double".equals(dataType) && fieldValue instanceof BigDecimal) {
fieldValue = ((BigDecimal) fieldValue).doubleValue();
+ } else if ("decimal".equals(dataType) && fieldValue instanceof BigDecimal) {
+ fieldValue = normalizeDecimal((BigDecimal) fieldValue);
} else if ("integer".equals(dataType) && fieldValue instanceof Number) {
fieldValue = ((Number) fieldValue).intValue();
} else if ("smallint".equals(dataType) && fieldValue instanceof Number) {
@@ -145,6 +147,14 @@ public class Tuple implements ITuple {
return o;
}
+ private static BigDecimal normalizeDecimal(BigDecimal input) {
+ if (input.scale() < 0) {
+ return input.setScale(0);
+ } else {
+ return input;
+ }
+ }
+
public boolean hasColumn(TblColRef column) {
return info.hasColumn(column);
}
@@ -199,7 +209,7 @@ public class Tuple implements ITuple {
case "double":
return Double.valueOf(strValue);
case "decimal":
- return new BigDecimal(strValue);
+ return normalizeDecimal(new BigDecimal(strValue));
case "float":
return Float.valueOf(strValue);
case "boolean":
[40/50] [abbrv] kylin git commit: minor,
generate item_count value for sample data
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/d2aaf270/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
----------------------------------------------------------------------
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index 7349b37..e13b7ad 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,10000 +1,10000 @@
-0,2012-12-14,Others,88750,0,11,36.2828,0,10000349,10002313,ANALYST,Beijing
-1,2012-08-28,Others,175750,0,13,23.8563,0,10000927,10004376,ANALYST,Beijing
-2,2012-02-16,ABIN,148324,15,13,88.3418,0,10000005,10006710,ADMIN,Shanghai
-3,2013-10-19,FP-non GTC,37831,0,13,47.3015,0,10000209,10003717,ANALYST,Beijing
-4,2012-10-22,Others,140746,100,11,83.454,0,10000154,10006076,ADMIN,Shanghai
-5,2013-01-25,FP-GTC,16509,0,-99,22.9896,0,10000372,10007436,ADMIN,Shanghai
-6,2013-04-04,Others,963,0,13,88.5907,0,10000648,10009869,MODELER,Hongkong
-7,2012-04-11,Others,15687,0,15,88.194,0,10000866,10000400,ADMIN,Shanghai
-8,2013-01-19,ABIN,60606,3,13,77.9727,0,10000936,10000687,MODELER,Hongkong
-9,2012-04-30,FP-non GTC,106246,0,14,52.295,0,10000047,10009223,MODELER,Hongkong
-10,2013-02-03,Auction,45333,0,16,56.3584,0,10000470,10006759,MODELER,Hongkong
-11,2012-09-02,FP-GTC,25147,0,-99,15.3553,0,10000203,10000062,ANALYST,Beijing
-12,2013-07-30,Others,24760,0,16,25.077,0,10000382,10003539,ADMIN,Shanghai
-13,2012-04-17,Auction,31519,0,16,72.7384,0,10000069,10001245,ADMIN,Shanghai
-14,2013-04-30,FP-GTC,51582,0,15,75.82,0,10000085,10008148,ANALYST,Beijing
-15,2013-03-23,FP-GTC,16509,0,15,8.6653,0,10000559,10008079,MODELER,Hongkong
-16,2013-04-01,ABIN,175750,3,16,64.2802,0,10000507,10007797,MODELER,Hongkong
-17,2013-07-12,ABIN,50677,0,-99,24.5987,0,10000938,10006224,ADMIN,Shanghai
-18,2012-04-23,FP-GTC,45333,0,5,72.6553,0,10000863,10003824,MODELER,Hongkong
-19,2012-12-19,ABIN,3838,0,12,87.4142,0,10000612,10007157,ADMIN,Shanghai
-20,2012-08-15,FP-GTC,62179,0,16,67.4238,0,10000839,10003452,MODELER,Hongkong
-21,2013-08-22,Auction,1357,0,5,18.36,0,10000303,10000114,ANALYST,Beijing
-22,2013-08-23,ABIN,31519,0,5,67.6344,0,10000877,10007417,ANALYST,Beijing
-23,2013-09-12,FP-GTC,223,0,14,71.7898,0,10000848,10000627,MODELER,Hongkong
-24,2012-03-14,FP-GTC,132939,0,15,26.1496,0,10000815,10005292,MODELER,Hongkong
-25,2013-07-13,Others,67703,3,16,83.9661,0,10000858,10008123,MODELER,Hongkong
-26,2013-12-15,FP-non GTC,100847,0,5,34.3451,0,10000387,10007806,ADMIN,Shanghai
-27,2012-08-30,Auction,62179,0,11,21.0496,0,10000393,10007360,ANALYST,Beijing
-28,2013-01-10,FP-GTC,48027,0,13,23.4142,0,10000757,10002135,MODELER,Hongkong
-29,2013-01-25,ABIN,145970,0,5,26.7842,0,10000402,10009682,ANALYST,Beijing
-30,2012-07-05,Auction,48027,0,-99,46.732,0,10000972,10009522,MODELER,Hongkong
-31,2012-11-07,FP-GTC,20213,0,-99,19.727,0,10000873,10009944,ANALYST,Beijing
-32,2013-06-20,ABIN,100847,0,5,76.2783,0,10000409,10005395,ADMIN,Shanghai
-33,2012-10-08,ABIN,26262,0,5,48.7447,0,10000029,10001836,ADMIN,Shanghai
-34,2013-04-30,ABIN,87118,0,15,63.2187,0,10000529,10000928,ANALYST,Beijing
-35,2013-04-24,FP-non GTC,20213,0,16,39.5057,0,10000329,10000768,ANALYST,Beijing
-36,2012-05-13,Others,164261,0,13,84.1246,0,10000688,10000727,ADMIN,Shanghai
-37,2013-08-02,ABIN,31519,0,5,37.1504,0,10000905,10000719,MODELER,Hongkong
-38,2012-04-03,Auction,164261,0,11,92.0974,0,10000341,10001337,ADMIN,Shanghai
-39,2012-09-02,FP-GTC,31519,0,14,87.7829,0,10000460,10008484,ANALYST,Beijing
-40,2013-10-05,ABIN,13987,0,16,74.0719,0,10000381,10002975,MODELER,Hongkong
-41,2012-01-13,FP-GTC,48027,0,15,71.2637,0,10000915,10005778,ADMIN,Shanghai
-42,2013-03-01,Others,13836,0,14,16.7288,0,10000953,10009385,ADMIN,Shanghai
-43,2012-09-12,FP-non GTC,16509,0,5,12.2933,0,10000277,10002286,MODELER,Hongkong
-44,2012-02-07,Auction,45333,0,5,64.977,0,10000040,10005583,MODELER,Hongkong
-45,2013-12-14,FP-non GTC,158798,0,16,72.4413,0,10000500,10007635,MODELER,Hongkong
-46,2013-10-13,Auction,31519,0,5,79.3053,0,10000816,10006423,MODELER,Hongkong
-47,2012-06-11,ABIN,4943,0,5,11.6942,0,10000696,10001237,ANALYST,Beijing
-48,2012-10-18,ABIN,80053,0,-99,54.0933,0,10000604,10006851,MODELER,Hongkong
-49,2012-11-03,Auction,20886,0,5,9.8258,0,10000764,10001220,MODELER,Hongkong
-50,2012-01-15,Auction,44079,0,14,13.0371,0,10000343,10001259,MODELER,Hongkong
-51,2012-07-03,FP-GTC,159184,0,-99,92.5314,0,10000724,10009008,ADMIN,Shanghai
-52,2012-02-16,FP-GTC,314,0,13,49.1825,0,10000334,10004005,ADMIN,Shanghai
-53,2012-06-17,FP-non GTC,31387,3,12,0.6677,0,10000603,10002595,ANALYST,Beijing
-54,2012-07-15,FP-non GTC,32876,0,-99,50.9634,0,10000248,10006890,ANALYST,Beijing
-55,2012-07-24,FP-GTC,150265,15,16,57.5645,0,10000653,10007269,ADMIN,Shanghai
-56,2012-10-06,Auction,145970,0,5,44.3091,0,10000082,10001640,ADMIN,Shanghai
-57,2013-06-14,ABIN,80053,0,5,20.2388,0,10000632,10002068,MODELER,Hongkong
-58,2013-10-10,FP-non GTC,15868,0,11,28.629,0,10000523,10004108,MODELER,Hongkong
-59,2012-09-30,ABIN,31673,0,12,75.8658,0,10000719,10009011,ADMIN,Shanghai
-60,2012-04-02,FP-GTC,63861,0,5,25.8748,0,10000311,10000289,ANALYST,Beijing
-61,2012-03-16,Others,314,0,-99,9.2733,0,10000409,10001533,MODELER,Hongkong
-62,2012-09-30,FP-non GTC,60340,0,14,74.661,0,10000790,10006831,MODELER,Hongkong
-63,2012-03-20,Others,87118,0,5,79.3781,0,10000384,10000632,MODELER,Hongkong
-64,2012-09-12,FP-GTC,1120,3,5,61.4408,0,10000645,10001462,ANALYST,Beijing
-65,2013-12-10,FP-non GTC,20886,0,11,14.814,0,10000943,10008334,ANALYST,Beijing
-66,2012-12-05,FP-GTC,106246,0,12,24.938,0,10000858,10007796,ANALYST,Beijing
-67,2012-08-21,Auction,43479,0,12,22.2008,0,10000310,10001440,MODELER,Hongkong
-68,2012-11-30,FP-GTC,175750,0,15,35.1161,0,10000440,10007199,ADMIN,Shanghai
-69,2013-10-10,Others,95672,0,11,82.3718,0,10000128,10003194,ADMIN,Shanghai
-70,2012-10-17,Auction,80053,0,15,9.8166,0,10000105,10009975,ADMIN,Shanghai
-71,2012-03-16,FP-non GTC,57784,0,5,66.2369,0,10000439,10008763,MODELER,Hongkong
-72,2012-11-14,ABIN,65,0,11,17.5513,0,10000855,10007000,MODELER,Hongkong
-73,2013-06-24,FP-GTC,16509,0,15,53.9195,0,10000130,10005095,MODELER,Hongkong
-74,2012-08-13,FP-non GTC,65,0,5,69.8447,0,10000548,10005613,MODELER,Hongkong
-75,2012-01-03,Others,38238,0,11,90.3722,0,10000739,10004281,ADMIN,Shanghai
-76,2013-11-24,FP-non GTC,175750,3,13,47.3441,0,10000291,10003352,ANALYST,Beijing
-77,2013-11-08,Auction,106246,0,5,38.6729,0,10000310,10007150,ADMIN,Shanghai
-78,2013-04-07,Others,73506,0,-99,69.8999,0,10000235,10003270,ADMIN,Shanghai
-79,2012-10-15,FP-GTC,314,0,13,44.1598,0,10000265,10001620,ADMIN,Shanghai
-80,2012-01-20,FP-GTC,80287,0,-99,85.7263,0,10000704,10002716,ANALYST,Beijing
-81,2012-08-17,ABIN,32876,0,16,21.2934,0,10000163,10007391,ANALYST,Beijing
-82,2013-07-09,Auction,37831,0,15,38.2213,0,10000557,10006017,ADMIN,Shanghai
-83,2013-09-21,Others,45333,0,13,25.5384,0,10000184,10003754,MODELER,Hongkong
-84,2013-10-28,FP-non GTC,11554,0,16,89.9654,0,10000940,10005821,MODELER,Hongkong
-85,2013-04-05,Auction,11554,0,14,41.8676,0,10000384,10007941,ANALYST,Beijing
-86,2013-03-25,FP-GTC,43398,0,14,75.011,0,10000697,10006079,ANALYST,Beijing
-87,2012-05-03,ABIN,33038,15,12,64.6735,0,10000147,10001734,ADMIN,Shanghai
-88,2012-10-13,ABIN,95672,3,14,3.2128,0,10000547,10003232,MODELER,Hongkong
-89,2012-09-11,FP-GTC,37831,0,16,60.3228,0,10000899,10002001,MODELER,Hongkong
-90,2012-08-31,Auction,314,0,5,13.248,0,10000350,10005727,ANALYST,Beijing
-91,2012-12-25,Others,165888,0,16,60.3492,0,10000367,10000234,ANALYST,Beijing
-92,2012-12-26,FP-non GTC,50677,0,5,58.6078,0,10000523,10003148,ADMIN,Shanghai
-93,2012-05-24,FP-GTC,145970,0,5,33.1342,0,10000926,10009430,MODELER,Hongkong
-94,2012-06-27,Auction,174106,3,13,55.3039,0,10000430,10006553,ANALYST,Beijing
-95,2012-01-02,ABIN,87118,0,14,48.2433,0,10000731,10006539,ADMIN,Shanghai
-96,2013-12-14,FP-GTC,50508,0,5,51.5822,0,10000977,10000022,ANALYST,Beijing
-97,2012-12-30,Auction,63889,0,12,3.8393,0,10000957,10008304,MODELER,Hongkong
-98,2013-10-26,FP-non GTC,24541,0,5,23.6812,0,10000350,10005174,ANALYST,Beijing
-99,2013-12-29,ABIN,38238,0,15,99.4389,0,10000605,10002057,ANALYST,Beijing
-100,2012-05-26,FP-GTC,100847,0,15,11.0672,0,10000291,10002184,ADMIN,Shanghai
-101,2012-12-18,FP-non GTC,164261,0,11,54.9352,0,10000509,10000385,ANALYST,Beijing
-102,2013-03-12,ABIN,95672,0,15,85.7152,0,10000742,10000188,ADMIN,Shanghai
-103,2013-02-22,ABIN,43398,0,11,65.197,0,10000294,10007930,ADMIN,Shanghai
-104,2012-02-20,Others,175750,0,14,63.2187,0,10000364,10007008,MODELER,Hongkong
-105,2013-08-06,Others,32876,0,13,50.8111,0,10000542,10004538,MODELER,Hongkong
-106,2013-06-27,Others,13987,0,13,2.3165,0,10000979,10003960,ADMIN,Shanghai
-107,2012-07-17,Others,13987,0,16,25.2136,0,10000744,10006962,ADMIN,Shanghai
-108,2013-10-12,FP-non GTC,16509,0,5,15.3986,0,10000652,10000372,MODELER,Hongkong
-109,2012-12-07,Others,158798,0,5,1.6111,0,10000149,10007138,ANALYST,Beijing
-110,2013-02-09,Auction,32876,0,5,37.3237,0,10000467,10007844,ADMIN,Shanghai
-111,2012-10-26,Others,44079,0,12,18.187,0,10000420,10003664,ADMIN,Shanghai
-112,2012-04-10,ABIN,132939,0,13,88.595,0,10000103,10004100,MODELER,Hongkong
-113,2013-04-08,FP-non GTC,159184,0,5,92.4421,0,10000933,10005974,ADMIN,Shanghai
-114,2012-05-30,Others,80287,0,14,68.1663,0,10000195,10001912,ADMIN,Shanghai
-115,2012-02-23,FP-GTC,88750,0,11,19.4075,0,10000038,10000200,ADMIN,Shanghai
-116,2013-10-25,ABIN,11554,0,11,27.2512,0,10000228,10006680,ANALYST,Beijing
-117,2012-12-02,FP-GTC,41940,0,14,49.5504,0,10000927,10003981,ANALYST,Beijing
-118,2013-09-16,FP-GTC,314,0,-99,3.4785,0,10000304,10002124,MODELER,Hongkong
-119,2013-11-01,Others,31519,3,15,46.1783,0,10000771,10008920,MODELER,Hongkong
-120,2012-10-21,FP-non GTC,31519,0,15,1.7696,0,10000507,10001384,MODELER,Hongkong
-121,2013-08-25,Auction,95173,0,13,89.2083,0,10000868,10006103,MODELER,Hongkong
-122,2013-11-09,FP-non GTC,150047,3,12,58.2634,0,10000497,10004364,ANALYST,Beijing
-123,2012-02-04,FP-GTC,175750,0,11,41.385,0,10000947,10000922,ANALYST,Beijing
-124,2012-10-17,ABIN,26249,0,14,10.5789,0,10000965,10001034,ADMIN,Shanghai
-125,2012-03-16,FP-non GTC,963,0,5,17.8949,0,10000908,10009714,ANALYST,Beijing
-126,2012-08-06,Others,94847,0,13,96.0332,0,10000595,10002452,ANALYST,Beijing
-127,2013-12-23,FP-non GTC,24760,0,16,90.7096,0,10000700,10006693,ADMIN,Shanghai
-128,2013-03-29,FP-GTC,9426,3,13,50.9023,0,10000090,10004582,ADMIN,Shanghai
-129,2013-03-25,Others,20886,0,-99,35.0414,0,10000171,10002472,ANALYST,Beijing
-130,2013-12-10,Auction,95672,0,15,29.8872,0,10000216,10009995,MODELER,Hongkong
-131,2012-03-14,Others,12688,0,14,10.4067,0,10000626,10003664,MODELER,Hongkong
-132,2013-11-29,FP-non GTC,31519,0,13,89.2334,0,10000199,10008582,ANALYST,Beijing
-133,2013-04-29,ABIN,31519,0,-99,99.246,0,10000565,10005784,ADMIN,Shanghai
-134,2012-02-01,FP-GTC,161567,15,11,56.3989,0,10000651,10007644,ANALYST,Beijing
-135,2012-03-17,FP-GTC,4943,0,16,0.4422,0,10000829,10004772,MODELER,Hongkong
-136,2012-11-06,FP-GTC,156614,0,12,2.3454,0,10000370,10009337,ANALYST,Beijing
-137,2012-12-03,Auction,155226,0,13,51.1354,0,10000867,10008319,MODELER,Hongkong
-138,2012-09-09,FP-GTC,99985,0,12,9.1236,0,10000281,10001454,ANALYST,Beijing
-139,2012-06-16,ABIN,99985,0,16,16.407,0,10000143,10000141,MODELER,Hongkong
-140,2012-05-06,FP-GTC,99985,0,14,6.4304,0,10000514,10009863,MODELER,Hongkong
-141,2013-10-20,FP-GTC,36250,0,13,55.6895,0,10000390,10006419,ADMIN,Shanghai
-142,2012-10-11,ABIN,106246,0,15,30.6373,0,10000980,10003318,ADMIN,Shanghai
-143,2013-08-08,Others,103178,15,12,41.4904,0,10000751,10007021,ADMIN,Shanghai
-144,2012-09-05,Auction,314,0,16,3.2615,0,10000280,10008590,MODELER,Hongkong
-145,2012-08-07,ABIN,48904,0,13,57.7688,0,10000245,10005116,ANALYST,Beijing
-146,2013-07-05,ABIN,314,0,14,28.9444,0,10000888,10001907,MODELER,Hongkong
-147,2012-09-10,FP-non GTC,95672,0,-99,54.0914,0,10000601,10008909,ANALYST,Beijing
-148,2012-09-18,Auction,25147,0,12,74.4422,0,10000679,10000970,ANALYST,Beijing
-149,2012-08-24,FP-non GTC,175750,3,16,45.8282,0,10000397,10000959,ANALYST,Beijing
-150,2013-05-29,Others,26249,0,14,12.7681,0,10000967,10006295,ANALYST,Beijing
-151,2012-11-17,FP-GTC,94847,0,14,74.4181,0,10000175,10009505,ANALYST,Beijing
-152,2012-09-22,ABIN,175750,0,15,85.6879,0,10000453,10008084,ADMIN,Shanghai
-153,2013-09-17,ABIN,533,0,11,85.5838,0,10000642,10003672,ADMIN,Shanghai
-154,2013-03-28,Others,60340,0,12,72.6495,0,10000397,10003829,ADMIN,Shanghai
-155,2013-08-09,Others,216,0,15,88.6918,0,10000867,10000486,ADMIN,Shanghai
-156,2013-12-19,Others,43398,0,-99,88.0444,0,10000482,10004753,MODELER,Hongkong
-157,2013-12-02,FP-non GTC,15115,0,-99,29.6027,0,10000538,10007924,MODELER,Hongkong
-158,2012-11-23,Others,46575,0,5,49.3124,0,10000678,10003077,ANALYST,Beijing
-159,2013-06-25,FP-non GTC,16145,0,15,7.4413,0,10000897,10004554,MODELER,Hongkong
-160,2012-01-22,FP-GTC,164,0,14,71.8071,0,10000510,10000523,ANALYST,Beijing
-161,2013-04-24,FP-GTC,1504,0,12,95.5706,0,10000554,10002121,MODELER,Hongkong
-162,2012-03-05,FP-non GTC,12688,0,5,47.7646,0,10000175,10005221,ADMIN,Shanghai
-163,2013-02-15,FP-non GTC,16145,0,13,40.9511,0,10000647,10002696,MODELER,Hongkong
-164,2013-01-18,Others,36250,0,13,77.3572,0,10000687,10002478,ADMIN,Shanghai
-165,2012-06-04,Others,15115,0,15,25.34,0,10000493,10000370,MODELER,Hongkong
-166,2013-10-13,FP-GTC,165888,0,-99,58.1861,0,10000753,10003626,ANALYST,Beijing
-167,2012-05-21,Auction,145970,0,5,37.2504,0,10000292,10002719,ADMIN,Shanghai
-168,2012-04-26,Auction,11848,0,13,29.2954,0,10000804,10007432,MODELER,Hongkong
-169,2013-09-26,FP-GTC,61323,0,13,84.7095,0,10000761,10005178,ANALYST,Beijing
-170,2013-03-06,Auction,100847,0,-99,81.798,0,10000593,10006532,ANALYST,Beijing
-171,2012-08-08,ABIN,51582,0,15,15.979,0,10000138,10008335,ADMIN,Shanghai
-172,2012-01-09,Others,1504,0,14,34.0508,0,10000954,10003235,ANALYST,Beijing
-173,2013-10-20,ABIN,20886,0,12,36.9353,0,10000408,10007540,ADMIN,Shanghai
-174,2013-06-25,Auction,53064,0,11,99.7714,0,10000409,10003295,MODELER,Hongkong
-175,2013-05-03,FP-GTC,63861,0,5,39.2974,0,10000668,10003615,MODELER,Hongkong
-176,2013-01-26,FP-GTC,100847,0,13,59.5009,0,10000536,10005423,ADMIN,Shanghai
-177,2012-09-19,FP-GTC,164261,0,13,15.7583,0,10000231,10007013,ANALYST,Beijing
-178,2013-11-16,FP-GTC,164261,0,16,83.6234,0,10000927,10006597,ADMIN,Shanghai
-179,2013-05-07,FP-non GTC,20485,0,5,52.91,0,10000491,10005645,MODELER,Hongkong
-180,2013-08-15,Others,32996,15,12,54.7122,0,10000411,10008504,MODELER,Hongkong
-181,2013-10-31,FP-GTC,63861,0,14,0.3599,0,10000903,10005471,MODELER,Hongkong
-182,2012-06-13,FP-non GTC,159184,0,11,49.955,0,10000849,10000545,ANALYST,Beijing
-183,2012-12-25,Auction,20213,0,-99,11.4752,0,10000583,10004067,ANALYST,Beijing
-184,2013-12-15,Auction,759,0,13,33.6991,0,10000390,10004439,ADMIN,Shanghai
-185,2012-04-30,Others,87118,0,16,51.463,0,10000572,10009658,MODELER,Hongkong
-186,2012-07-14,Others,10866,0,11,36.9614,0,10000467,10001311,ANALYST,Beijing
-187,2013-06-25,FP-non GTC,1357,0,12,17.5618,0,10000426,10000482,ANALYST,Beijing
-188,2013-02-06,Auction,13836,0,12,74.3179,0,10000346,10005540,ANALYST,Beijing
-189,2012-06-03,ABIN,60340,0,12,82.3722,0,10000531,10005121,ANALYST,Beijing
-190,2013-07-23,FP-GTC,139973,0,-99,75.2548,0,10000385,10004656,ADMIN,Shanghai
-191,2013-07-06,Others,11554,0,14,40.1129,0,10000724,10000830,ADMIN,Shanghai
-192,2013-08-10,FP-non GTC,31519,0,-99,5.2432,0,10000356,10000921,ADMIN,Shanghai
-193,2013-07-31,Auction,31673,0,14,51.1089,0,10000598,10005431,ADMIN,Shanghai
-194,2013-12-05,Auction,314,0,13,19.9031,0,10000068,10007321,ADMIN,Shanghai
-195,2012-12-30,ABIN,50508,0,5,73.8659,0,10000482,10005414,MODELER,Hongkong
-196,2012-09-22,Auction,165888,0,12,81.9826,0,10000832,10000194,MODELER,Hongkong
-197,2012-07-16,Auction,223,0,5,76.0284,0,10000354,10002836,MODELER,Hongkong
-198,2013-01-26,Others,20485,0,12,71.6623,0,10000552,10000202,ANALYST,Beijing
-199,2012-11-07,FP-non GTC,57013,0,16,73.2931,0,10000801,10000942,ANALYST,Beijing
-200,2012-01-26,FP-non GTC,175750,0,13,18.897,0,10000649,10001073,ADMIN,Shanghai
-201,2012-06-09,FP-non GTC,60340,0,12,11.1614,0,10000830,10007167,ADMIN,Shanghai
-202,2012-06-20,Auction,57990,3,5,69.1183,0,10000557,10004252,MODELER,Hongkong
-203,2013-01-01,FP-non GTC,45333,0,13,94.0586,0,10000253,10003835,ANALYST,Beijing
-204,2013-07-26,FP-non GTC,139973,0,11,39.7239,0,10000441,10000478,ADMIN,Shanghai
-205,2012-09-15,FP-GTC,24541,0,15,0.3239,0,10000379,10009850,ANALYST,Beijing
-206,2013-09-22,Auction,31519,3,12,80.0315,0,10000436,10001924,ANALYST,Beijing
-207,2012-10-27,FP-GTC,223,0,15,39.379,0,10000420,10002426,ANALYST,Beijing
-208,2012-11-20,FP-non GTC,533,0,5,49.4951,0,10000390,10006171,ADMIN,Shanghai
-209,2013-12-07,FP-GTC,1349,0,15,93.3008,0,10000398,10008965,MODELER,Hongkong
-210,2012-05-30,FP-non GTC,46575,0,12,66.7652,0,10000915,10006004,ADMIN,Shanghai
-211,2012-09-07,FP-non GTC,95672,0,14,29.597,0,10000231,10004893,MODELER,Hongkong
-212,2013-04-26,Auction,31673,0,-99,15.8347,0,10000260,10002693,ANALYST,Beijing
-213,2012-09-11,FP-non GTC,50508,0,14,80.5444,0,10000263,10002496,ADMIN,Shanghai
-214,2013-09-03,FP-non GTC,95173,0,16,33.1487,0,10000493,10002289,ADMIN,Shanghai
-215,2012-10-03,FP-non GTC,44079,0,15,99.7408,0,10000828,10009523,ADMIN,Shanghai
-216,2012-12-26,ABIN,16145,0,14,40.7826,0,10000062,10005334,ANALYST,Beijing
-217,2012-07-17,Auction,26249,0,15,28.6663,0,10000534,10002400,ADMIN,Shanghai
-218,2013-06-30,ABIN,63861,0,14,75.6832,0,10000944,10007279,ADMIN,Shanghai
-219,2012-01-09,ABIN,156614,0,11,98.4996,0,10000353,10005539,ADMIN,Shanghai
-220,2013-09-08,FP-GTC,156614,0,13,12.7473,0,10000063,10002408,ADMIN,Shanghai
-221,2013-09-23,Auction,41940,0,5,37.5254,0,10000383,10001341,ADMIN,Shanghai
-222,2012-11-11,Auction,145970,0,5,77.1361,0,10000420,10000232,MODELER,Hongkong
-223,2013-08-26,Others,11848,0,16,28.2799,0,10000323,10007637,MODELER,Hongkong
-224,2012-09-15,Others,11554,0,13,63.7232,0,10000729,10008500,ADMIN,Shanghai
-225,2012-01-09,FP-non GTC,25147,0,11,7.3009,0,10000745,10006070,ADMIN,Shanghai
-226,2012-08-16,Others,87118,0,15,47.8997,0,10000240,10002299,MODELER,Hongkong
-227,2013-12-26,FP-non GTC,16145,0,12,51.0442,0,10000050,10000493,MODELER,Hongkong
-228,2012-03-24,ABIN,48027,0,5,26.2083,0,10000600,10007513,ANALYST,Beijing
-229,2012-08-22,ABIN,216,0,5,79.9268,0,10000706,10002465,ANALYST,Beijing
-230,2013-11-03,FP-GTC,24541,0,12,83.0235,0,10000293,10007217,ADMIN,Shanghai
-231,2013-05-18,FP-non GTC,88750,0,-99,84.8856,0,10000147,10007044,ADMIN,Shanghai
-232,2013-01-15,ABIN,95672,0,15,86.0386,0,10000590,10009080,MODELER,Hongkong
-233,2012-12-03,ABIN,50508,0,11,67.9981,0,10000582,10007971,MODELER,Hongkong
-234,2012-06-19,FP-GTC,148324,15,15,53.4811,0,10000793,10009995,ANALYST,Beijing
-235,2013-10-06,Auction,6762,0,14,73.5338,0,10000544,10009125,MODELER,Hongkong
-236,2013-12-18,ABIN,63861,0,5,50.2892,0,10000769,10005451,ADMIN,Shanghai
-237,2013-12-04,Auction,25147,0,5,48.3303,0,10000590,10001560,MODELER,Hongkong
-238,2012-04-25,FP-non GTC,62179,0,14,72.3156,0,10000102,10008456,ADMIN,Shanghai
-239,2013-11-20,Auction,24760,0,14,91.4883,0,10000370,10003137,ADMIN,Shanghai
-240,2013-02-08,Others,20485,0,11,29.1083,0,10000261,10005174,MODELER,Hongkong
-241,2013-04-18,Others,2023,0,5,48.4321,0,10000355,10009360,ANALYST,Beijing
-242,2012-11-21,Others,32876,0,12,16.7365,0,10000352,10008131,ANALYST,Beijing
-243,2012-03-07,FP-GTC,61323,0,-99,32.5836,0,10000494,10001632,ADMIN,Shanghai
-244,2012-01-20,FP-non GTC,175750,0,16,52.8153,0,10000025,10007893,ADMIN,Shanghai
-245,2013-10-06,FP-GTC,26249,0,-99,1.9539,0,10000660,10008166,MODELER,Hongkong
-246,2012-01-10,Auction,57990,0,14,57.58,0,10000830,10004472,ANALYST,Beijing
-247,2013-09-13,Others,46575,0,15,60.3315,0,10000528,10008869,MODELER,Hongkong
-248,2012-11-19,FP-non GTC,44079,0,11,73.0457,0,10000482,10006453,ANALYST,Beijing
-249,2012-09-23,Auction,95672,0,13,84.6345,0,10000586,10000622,MODELER,Hongkong
-250,2013-08-31,Others,216,0,12,34.7429,0,10000676,10003143,ANALYST,Beijing
-251,2013-04-11,FP-GTC,963,0,-99,77.7226,0,10000367,10003468,ADMIN,Shanghai
-252,2013-03-28,ABIN,15687,0,-99,8.2384,0,10000542,10002480,ANALYST,Beijing
-253,2012-05-24,ABIN,152801,0,15,39.0204,0,10000277,10008011,ANALYST,Beijing
-254,2013-03-05,ABIN,67698,0,16,40.6091,0,10000886,10008753,ADMIN,Shanghai
-255,2013-08-06,Auction,13987,0,13,36.5088,0,10000628,10000546,MODELER,Hongkong
-256,2013-02-04,ABIN,20213,0,11,73.5301,0,10000134,10003648,MODELER,Hongkong
-257,2012-05-19,FP-non GTC,38238,0,12,59.3237,0,10000500,10008314,ADMIN,Shanghai
-258,2013-08-19,ABIN,57990,0,12,10.1688,0,10000715,10007648,ANALYST,Beijing
-259,2012-03-28,FP-non GTC,46575,0,16,77.7396,0,10000756,10003617,ADMIN,Shanghai
-260,2012-02-19,Auction,43479,0,-99,45.245,0,10000049,10001668,ANALYST,Beijing
-261,2012-04-02,FP-GTC,41940,0,15,58.8415,0,10000195,10000014,ADMIN,Shanghai
-262,2013-08-24,Others,16145,0,11,5.4971,0,10000119,10005635,MODELER,Hongkong
-263,2013-01-10,Auction,106340,15,12,24.1336,0,10000603,10003847,MODELER,Hongkong
-264,2012-09-23,Auction,152801,0,5,70.4024,0,10000384,10002592,ANALYST,Beijing
-265,2013-07-29,FP-GTC,73506,0,16,14.5932,0,10000580,10005977,ADMIN,Shanghai
-266,2012-02-16,FP-non GTC,67698,0,5,4.4981,0,10000524,10003718,MODELER,Hongkong
-267,2013-01-06,Others,65,0,13,64.8343,0,10000731,10002914,MODELER,Hongkong
-268,2013-03-02,Auction,43398,0,16,22.6621,0,10000734,10005774,ADMIN,Shanghai
-269,2013-10-25,Auction,57990,0,13,64.3384,0,10000961,10007907,ADMIN,Shanghai
-270,2012-07-07,FP-GTC,156356,0,16,47.3744,0,10000795,10004560,ADMIN,Shanghai
-271,2013-12-19,FP-GTC,67698,0,16,96.9535,0,10000207,10005626,MODELER,Hongkong
-272,2013-11-04,Auction,63861,0,5,34.8173,0,10000964,10001539,MODELER,Hongkong
-273,2013-01-18,FP-non GTC,174053,3,5,94.6036,0,10000393,10003930,ANALYST,Beijing
-274,2013-01-30,Auction,43479,0,16,28.9982,0,10000854,10004230,ANALYST,Beijing
-275,2012-05-16,ABIN,24541,0,15,91.6347,0,10000536,10004139,MODELER,Hongkong
-276,2013-06-03,FP-non GTC,159184,0,15,27.9722,0,10000644,10005293,ADMIN,Shanghai
-277,2013-08-24,FP-GTC,2635,0,14,95.7894,0,10000503,10009687,ADMIN,Shanghai
-278,2012-07-02,FP-non GTC,16509,0,5,76.9646,0,10000320,10009405,ADMIN,Shanghai
-279,2013-09-09,ABIN,80287,0,16,68.962,0,10000934,10006877,MODELER,Hongkong
-280,2012-04-15,FP-GTC,156356,0,12,95.364,0,10000269,10000216,ANALYST,Beijing
-281,2013-07-10,FP-GTC,1349,0,13,82.7969,0,10000033,10000809,MODELER,Hongkong
-282,2012-12-02,FP-GTC,963,0,13,31.4196,0,10000717,10002029,ADMIN,Shanghai
-283,2013-02-17,Auction,26249,0,14,9.5202,0,10000543,10009090,ANALYST,Beijing
-284,2013-02-17,Others,87118,0,16,77.3899,0,10000647,10009664,ADMIN,Shanghai
-285,2012-12-12,Others,6762,0,15,76.3422,0,10000076,10001968,ADMIN,Shanghai
-286,2013-07-08,Others,80053,0,5,95.2412,0,10000723,10007824,MODELER,Hongkong
-287,2012-08-19,ABIN,32876,0,12,40.9587,0,10000352,10000219,ANALYST,Beijing
-288,2013-07-24,Auction,314,0,11,78.8147,0,10000996,10004912,ANALYST,Beijing
-289,2013-10-10,Others,57990,0,12,89.8124,0,10000549,10001719,ANALYST,Beijing
-290,2013-11-13,Others,139973,0,16,85.5451,0,10000089,10006506,ADMIN,Shanghai
-291,2013-03-29,FP-non GTC,25147,0,16,16.229,0,10000017,10001175,MODELER,Hongkong
-292,2012-06-04,FP-non GTC,44079,0,-99,0.1588,0,10000758,10007273,MODELER,Hongkong
-293,2012-07-19,FP-non GTC,57784,0,12,84.9441,0,10000553,10005711,ANALYST,Beijing
-294,2012-01-08,FP-GTC,155226,0,15,31.9705,0,10000778,10006227,ADMIN,Shanghai
-295,2012-09-13,FP-non GTC,963,0,16,39.6392,0,10000799,10008569,ADMIN,Shanghai
-296,2012-12-24,Others,175750,0,15,77.3617,0,10000164,10000878,MODELER,Hongkong
-297,2013-08-31,Auction,57990,0,12,76.7885,0,10000580,10008645,ANALYST,Beijing
-298,2013-08-06,FP-non GTC,156356,0,13,86.0218,0,10000445,10003631,ANALYST,Beijing
-299,2013-05-29,FP-non GTC,11848,0,11,23.0269,0,10000372,10000789,MODELER,Hongkong
-300,2013-05-19,Others,53064,0,13,66.399,0,10000390,10009388,MODELER,Hongkong
-301,2012-11-01,FP-non GTC,51582,0,14,57.6157,0,10000061,10007487,ADMIN,Shanghai
-302,2013-01-01,Auction,1357,0,-99,18.2117,0,10000860,10006079,ANALYST,Beijing
-303,2013-05-17,Others,121153,0,14,79.4534,0,10000155,10004663,MODELER,Hongkong
-304,2013-07-04,FP-GTC,80053,0,5,43.3271,0,10000456,10003591,ANALYST,Beijing
-305,2013-12-17,Others,165888,0,-99,9.8402,0,10000357,10007752,MODELER,Hongkong
-306,2013-11-28,FP-GTC,26262,0,-99,1.4013,0,10000731,10008257,ADMIN,Shanghai
-307,2013-02-08,FP-GTC,65,0,11,61.9604,0,10000176,10004097,ADMIN,Shanghai
-308,2012-05-17,ABIN,9426,3,12,68.017,0,10000895,10000672,ANALYST,Beijing
-309,2012-08-29,Auction,159184,0,13,43.7406,0,10000723,10005989,MODELER,Hongkong
-310,2013-10-06,FP-GTC,61323,0,14,85.9325,0,10000692,10008834,ANALYST,Beijing
-311,2013-08-08,FP-GTC,20886,0,14,88.0137,0,10000063,10009103,ADMIN,Shanghai
-312,2013-04-06,FP-non GTC,156356,0,16,67.1201,0,10000441,10002323,ADMIN,Shanghai
-313,2012-03-09,Auction,1504,0,13,45.8878,0,10000862,10003955,ANALYST,Beijing
-314,2013-12-13,FP-GTC,139973,3,16,58.7879,0,10000947,10004467,ADMIN,Shanghai
-315,2013-02-03,ABIN,95173,0,-99,33.2714,0,10000634,10002647,ANALYST,Beijing
-316,2013-02-19,Others,41940,0,5,32.2179,0,10000257,10002668,ANALYST,Beijing
-317,2012-05-09,ABIN,65,0,11,96.0579,0,10000675,10008559,ANALYST,Beijing
-318,2012-10-30,Auction,80135,0,11,55.5335,0,10000288,10009113,ANALYST,Beijing
-319,2013-07-14,ABIN,11554,0,16,9.7012,0,10000210,10000034,MODELER,Hongkong
-320,2012-07-29,Others,11554,0,5,91.5938,0,10000736,10006505,ANALYST,Beijing
-321,2012-06-03,ABIN,80287,0,13,99.7431,0,10000403,10002322,ADMIN,Shanghai
-322,2012-10-10,Auction,36250,0,13,14.7056,0,10000561,10000446,ANALYST,Beijing
-323,2013-06-20,ABIN,20485,0,5,23.912,0,10000798,10002306,ANALYST,Beijing
-324,2013-12-26,Others,11554,0,15,27.2028,0,10000154,10001499,MODELER,Hongkong
-325,2013-05-07,Auction,11848,0,-99,73.7354,0,10000448,10000791,MODELER,Hongkong
-326,2013-12-14,ABIN,61323,0,16,44.6086,0,10000555,10003798,ADMIN,Shanghai
-327,2012-08-29,FP-GTC,139973,0,14,34.7404,0,10000686,10000704,ANALYST,Beijing
-328,2013-12-31,FP-GTC,100847,0,15,70.6027,0,10000534,10007718,MODELER,Hongkong
-329,2013-12-13,FP-non GTC,16145,3,12,76.5802,0,10000864,10002571,ADMIN,Shanghai
-330,2012-01-24,FP-non GTC,57990,0,-99,54.7904,0,10000325,10001260,ADMIN,Shanghai
-331,2013-07-26,Others,20485,0,11,9.313,0,10000991,10004211,MODELER,Hongkong
-332,2013-09-30,ABIN,31519,0,15,95.7797,0,10000701,10007592,ADMIN,Shanghai
-333,2012-03-30,FP-non GTC,88750,0,16,34.6777,0,10000494,10000223,ANALYST,Beijing
-334,2012-05-11,Others,20485,0,16,2.2782,0,10000436,10003082,MODELER,Hongkong
-335,2012-11-21,FP-non GTC,156356,0,13,3.4276,0,10000574,10007758,MODELER,Hongkong
-336,2012-11-19,FP-non GTC,36250,0,16,42.1665,0,10000885,10005830,ADMIN,Shanghai
-337,2013-01-07,Others,20485,0,13,29.1424,0,10000931,10007644,ANALYST,Beijing
-338,2013-01-16,ABIN,88750,0,13,87.3858,0,10000276,10004791,ADMIN,Shanghai
-339,2012-07-20,ABIN,25147,0,-99,41.9809,0,10000928,10001382,ADMIN,Shanghai
-340,2013-06-27,FP-non GTC,75665,0,11,65.3889,0,10000587,10001522,ADMIN,Shanghai
-341,2013-12-04,FP-non GTC,45333,0,15,41.7577,0,10000979,10006485,ADMIN,Shanghai
-342,2012-05-01,Auction,4943,0,11,76.3978,0,10000385,10006699,ANALYST,Beijing
-343,2013-05-03,Others,66767,15,-99,69.062,0,10000186,10009550,ANALYST,Beijing
-344,2013-02-14,FP-non GTC,95672,0,14,15.6152,0,10000356,10006265,ANALYST,Beijing
-345,2012-01-01,Others,1357,0,13,16.7271,0,10000263,10005410,ANALYST,Beijing
-346,2012-01-05,FP-non GTC,67703,3,5,32.2005,0,10000141,10008271,ADMIN,Shanghai
-347,2013-12-15,Others,2023,0,-99,55.3349,0,10000920,10002066,ADMIN,Shanghai
-348,2012-10-02,FP-GTC,314,0,14,95.9217,0,10000647,10009465,ANALYST,Beijing
-349,2012-10-27,Others,64076,0,5,3.4266,0,10000486,10005406,ADMIN,Shanghai
-350,2013-10-17,Others,41940,0,15,3.0363,0,10000299,10001071,ADMIN,Shanghai
-351,2013-02-09,Others,13836,0,15,37.1746,0,10000799,10005197,MODELER,Hongkong
-352,2013-04-03,FP-GTC,31519,0,15,83.3031,0,10000656,10000175,ADMIN,Shanghai
-353,2013-08-30,Others,63861,0,13,37.862,0,10000958,10007038,ADMIN,Shanghai
-354,2012-01-01,Others,1349,0,13,0.7227,0,10000774,10008113,ADMIN,Shanghai
-355,2013-03-24,FP-non GTC,87118,0,13,32.5615,0,10000661,10006407,ADMIN,Shanghai
-356,2012-12-03,FP-non GTC,216,0,5,40.1447,0,10000923,10008590,ANALYST,Beijing
-357,2012-07-12,FP-GTC,94847,0,5,74.4586,0,10000161,10009803,MODELER,Hongkong
-358,2013-03-18,Others,1161,3,-99,70.1521,0,10000123,10000399,ANALYST,Beijing
-359,2013-11-17,Auction,165888,0,5,16.1344,0,10000029,10008638,ANALYST,Beijing
-360,2013-11-25,FP-non GTC,44079,0,5,18.0996,0,10000461,10001260,MODELER,Hongkong
-361,2012-12-06,FP-GTC,159184,0,15,63.6868,0,10000399,10005791,ANALYST,Beijing
-362,2013-09-27,FP-non GTC,161567,15,15,35.8966,0,10000283,10005103,MODELER,Hongkong
-363,2013-11-02,ABIN,20865,0,5,70.2863,0,10000724,10008591,ADMIN,Shanghai
-364,2013-04-29,Auction,314,0,13,53.6862,0,10000676,10007462,MODELER,Hongkong
-365,2013-01-21,Auction,20886,0,-99,32.5548,0,10000052,10007849,MODELER,Hongkong
-366,2012-04-18,FP-GTC,67698,0,14,90.8206,0,10000221,10007877,ANALYST,Beijing
-367,2013-10-14,Others,46575,0,13,92.8392,0,10000187,10005505,ANALYST,Beijing
-368,2013-07-06,FP-GTC,26249,0,-99,54.6729,0,10000651,10008340,MODELER,Hongkong
-369,2012-09-15,Others,16145,0,15,70.0461,0,10000279,10007484,ADMIN,Shanghai
-370,2013-11-02,Others,38238,0,11,22.9595,0,10000451,10008446,MODELER,Hongkong
-371,2012-06-03,ABIN,57013,0,11,28.9005,0,10000028,10007103,ADMIN,Shanghai
-372,2013-10-08,FP-GTC,20886,0,14,24.6958,0,10000676,10008871,ANALYST,Beijing
-373,2013-04-11,Auction,100847,0,14,42.1878,0,10000645,10008666,MODELER,Hongkong
-374,2013-11-07,Auction,45333,0,13,58.1089,0,10000182,10002861,ANALYST,Beijing
-375,2012-10-18,FP-non GTC,57990,0,15,37.9049,0,10000696,10006240,ADMIN,Shanghai
-376,2012-11-28,FP-non GTC,63889,0,5,59.1268,0,10000116,10008447,ADMIN,Shanghai
-377,2012-03-02,Auction,45333,0,13,37.4799,0,10000298,10005848,ANALYST,Beijing
-378,2013-07-27,FP-non GTC,75665,0,12,19.423,0,10000333,10000499,ADMIN,Shanghai
-379,2012-12-01,Auction,44079,0,11,95.4596,0,10000163,10003317,ANALYST,Beijing
-380,2012-04-11,FP-non GTC,161567,15,13,47.4582,0,10000082,10003929,ADMIN,Shanghai
-381,2013-06-23,ABIN,175750,3,13,23.9511,0,10000560,10008767,ANALYST,Beijing
-382,2012-02-17,Others,46575,0,15,25.7613,0,10000308,10002423,ADMIN,Shanghai
-383,2013-05-19,ABIN,223,0,15,64.7127,0,10000850,10000542,ANALYST,Beijing
-384,2012-10-29,FP-non GTC,159184,0,5,15.4188,0,10000995,10003958,ADMIN,Shanghai
-385,2012-07-17,Auction,4943,0,11,32.1734,0,10000820,10000943,ANALYST,Beijing
-386,2013-01-03,ABIN,57784,0,5,47.7542,0,10000563,10007077,MODELER,Hongkong
-387,2013-07-17,FP-GTC,150047,3,16,35.0905,0,10000545,10004393,ADMIN,Shanghai
-388,2013-07-20,Others,87118,0,14,24.6574,0,10000985,10008103,MODELER,Hongkong
-389,2013-12-14,Others,11848,0,14,80.9504,0,10000318,10002671,MODELER,Hongkong
-390,2013-01-16,Auction,106246,0,12,94.2201,0,10000113,10007603,ADMIN,Shanghai
-391,2012-11-04,Others,63861,0,12,71.8354,0,10000985,10002264,MODELER,Hongkong
-392,2012-08-25,FP-non GTC,100847,0,5,68.7418,0,10000581,10002691,ADMIN,Shanghai
-393,2013-07-22,Auction,156356,0,15,63.6314,0,10000140,10002949,ANALYST,Beijing
-394,2013-11-04,Auction,67698,0,16,15.6061,0,10000128,10003790,MODELER,Hongkong
-395,2013-03-01,FP-GTC,20485,0,-99,49.2253,0,10000722,10000653,MODELER,Hongkong
-396,2013-12-03,FP-GTC,46575,0,-99,71.5261,0,10000796,10003451,ADMIN,Shanghai
-397,2013-03-25,FP-non GTC,60340,0,11,23.0796,0,10000605,10009790,ADMIN,Shanghai
-398,2012-11-16,FP-non GTC,99985,0,11,91.6223,0,10000619,10001315,ANALYST,Beijing
-399,2012-05-24,Auction,16509,0,16,97.7044,0,10000002,10004133,ADMIN,Shanghai
-400,2012-04-04,ABIN,139973,3,13,16.7393,0,10000923,10001413,MODELER,Hongkong
-401,2012-07-20,FP-GTC,174106,3,13,8.8874,0,10000134,10007563,MODELER,Hongkong
-402,2013-09-12,Auction,33038,15,13,46.7475,0,10000154,10001737,MODELER,Hongkong
-403,2012-06-12,ABIN,57990,0,11,33.1479,0,10000036,10005775,ADMIN,Shanghai
-404,2013-08-31,ABIN,46575,0,12,41.0635,0,10000310,10008468,ADMIN,Shanghai
-405,2012-12-18,Auction,1349,0,5,14.6328,0,10000296,10002985,MODELER,Hongkong
-406,2012-02-18,ABIN,16509,0,-99,76.3163,0,10000486,10007612,MODELER,Hongkong
-407,2013-07-02,Auction,3838,0,11,95.7402,0,10000963,10008136,ANALYST,Beijing
-408,2012-06-28,Auction,145970,0,16,41.1425,0,10000646,10001273,ADMIN,Shanghai
-409,2013-11-03,FP-GTC,12688,0,5,73.9106,0,10000357,10009931,MODELER,Hongkong
-410,2012-03-30,FP-non GTC,16509,0,5,39.5703,0,10000495,10005493,MODELER,Hongkong
-411,2012-08-18,FP-non GTC,75665,0,11,61.4928,0,10000522,10007332,MODELER,Hongkong
-412,2013-07-29,Others,156356,0,16,98.2405,0,10000417,10002579,ANALYST,Beijing
-413,2013-07-19,Others,32876,0,12,68.5148,0,10000306,10000694,ANALYST,Beijing
-414,2012-02-13,ABIN,175750,0,16,64.5444,0,10000274,10001204,ANALYST,Beijing
-415,2013-02-04,FP-non GTC,13836,0,-99,65.7663,0,10000511,10006170,ADMIN,Shanghai
-416,2013-07-24,FP-GTC,165888,0,16,91.7808,0,10000330,10000833,ADMIN,Shanghai
-417,2012-04-25,Others,41940,0,-99,6.5053,0,10000357,10003161,MODELER,Hongkong
-418,2013-11-28,ABIN,106340,15,12,71.4264,0,10000155,10004256,ANALYST,Beijing
-419,2012-09-16,FP-non GTC,170083,3,-99,80.2442,0,10000690,10007308,ANALYST,Beijing
-420,2012-05-10,FP-GTC,16145,0,14,11.0775,0,10000372,10008952,ANALYST,Beijing
-421,2013-10-28,FP-non GTC,15687,0,16,65.9436,0,10000032,10004950,ANALYST,Beijing
-422,2013-05-17,FP-non GTC,94847,0,13,37.352,0,10000044,10004245,ADMIN,Shanghai
-423,2013-05-31,FP-GTC,223,0,5,93.3181,0,10000365,10006098,MODELER,Hongkong
-424,2012-01-07,FP-non GTC,73506,0,13,13.1225,0,10000218,10000017,ANALYST,Beijing
-425,2013-07-01,FP-GTC,64076,0,15,78.1094,0,10000081,10009186,ADMIN,Shanghai
-426,2013-07-26,Others,67698,0,11,14.9245,0,10000775,10002428,ANALYST,Beijing
-427,2013-03-02,Auction,75665,0,5,84.2445,0,10000162,10007586,ANALYST,Beijing
-428,2012-01-13,FP-GTC,1120,3,16,43.8855,0,10000157,10009306,MODELER,Hongkong
-429,2012-06-02,ABIN,53064,0,12,17.36,0,10000091,10003461,ADMIN,Shanghai
-430,2012-02-23,ABIN,31387,3,14,43.1757,0,10000071,10009080,ADMIN,Shanghai
-431,2013-10-22,FP-non GTC,158798,0,15,10.9759,0,10000341,10006910,MODELER,Hongkong
-432,2013-05-23,FP-GTC,11848,0,-99,7.523,0,10000277,10004380,MODELER,Hongkong
-433,2013-07-25,FP-GTC,43479,0,14,13.262,0,10000500,10003341,MODELER,Hongkong
-434,2012-03-03,FP-non GTC,11848,0,14,95.0683,0,10000164,10009852,ADMIN,Shanghai
-435,2012-01-09,ABIN,11554,0,15,6.8777,0,10000785,10008587,ANALYST,Beijing
-436,2013-05-08,ABIN,80287,0,14,95.532,0,10000595,10003020,ANALYST,Beijing
-437,2013-07-26,FP-GTC,121153,0,12,28.7213,0,10000829,10008342,MODELER,Hongkong
-438,2012-11-27,FP-GTC,31673,0,14,7.8033,0,10000498,10003515,ANALYST,Beijing
-439,2013-06-04,ABIN,43398,0,15,76.0187,0,10000061,10009033,ANALYST,Beijing
-440,2012-10-05,ABIN,15687,0,13,87.1235,0,10000649,10006843,MODELER,Hongkong
-441,2013-06-29,Auction,61323,0,14,78.4,0,10000765,10000686,ANALYST,Beijing
-442,2012-07-25,ABIN,216,0,12,43.1556,0,10000597,10007703,ANALYST,Beijing
-443,2013-01-03,FP-non GTC,121153,0,12,70.1009,0,10000201,10008803,MODELER,Hongkong
-444,2013-10-22,FP-non GTC,170302,15,12,92.5959,0,10000151,10001464,MODELER,Hongkong
-445,2012-02-03,FP-non GTC,155226,0,12,68.6486,0,10000870,10007848,MODELER,Hongkong
-446,2013-03-30,FP-GTC,139973,0,11,93.3324,0,10000585,10004571,ANALYST,Beijing
-447,2012-11-21,Others,20865,0,11,68.0525,0,10000115,10004485,ADMIN,Shanghai
-448,2012-11-09,ABIN,11848,0,13,66.3745,0,10000652,10009670,ADMIN,Shanghai
-449,2012-10-22,Auction,2023,0,13,52.4918,0,10000836,10002112,ANALYST,Beijing
-450,2013-01-07,Auction,164261,0,-99,56.468,0,10000560,10000356,ANALYST,Beijing
-451,2012-01-05,Others,80053,0,-99,34.1305,0,10000905,10008693,ADMIN,Shanghai
-452,2012-08-27,FP-GTC,64076,0,14,3.0691,0,10000389,10008952,ADMIN,Shanghai
-453,2013-06-23,Others,139973,0,16,89.7673,0,10000636,10006901,ANALYST,Beijing
-454,2013-09-13,Auction,67698,0,15,62.9192,0,10000596,10000522,MODELER,Hongkong
-455,2012-08-18,FP-GTC,43398,0,12,45.3384,0,10000382,10003920,ADMIN,Shanghai
-456,2012-10-27,FP-non GTC,1120,3,5,84.6466,0,10000487,10001720,ANALYST,Beijing
-457,2012-05-24,Auction,24541,0,14,56.7041,0,10000366,10002296,ADMIN,Shanghai
-458,2013-02-12,Auction,64076,0,13,1.136,0,10000089,10004796,ADMIN,Shanghai
-459,2012-02-19,FP-non GTC,43398,0,14,7.2416,0,10000129,10005474,ANALYST,Beijing
-460,2012-07-30,ABIN,48027,0,11,41.6704,0,10000761,10009175,MODELER,Hongkong
-461,2013-10-04,FP-non GTC,37831,0,16,31.9567,0,10000610,10007936,MODELER,Hongkong
-462,2013-11-18,Auction,223,0,16,21.8155,0,10000079,10001417,ANALYST,Beijing
-463,2012-06-08,Others,80135,0,16,90.0166,0,10000478,10009633,MODELER,Hongkong
-464,2012-10-25,Others,159184,0,16,92.0436,0,10000166,10007086,ANALYST,Beijing
-465,2013-10-22,ABIN,13987,0,15,62.5546,0,10000592,10008119,ADMIN,Shanghai
-466,2013-11-01,ABIN,103324,15,16,14.9475,0,10000924,10006715,ADMIN,Shanghai
-467,2013-03-21,FP-GTC,1120,3,-99,29.5252,0,10000758,10000262,ADMIN,Shanghai
-468,2012-03-13,Others,759,0,16,62.4158,0,10000810,10000963,MODELER,Hongkong
-469,2012-05-14,Auction,95672,0,16,92.0781,0,10000609,10000260,ANALYST,Beijing
-470,2013-09-28,FP-GTC,64076,0,12,37.3599,0,10000487,10001678,ADMIN,Shanghai
-471,2013-10-09,ABIN,60606,3,13,48.9467,0,10000007,10004697,ANALYST,Beijing
-472,2013-04-21,ABIN,26249,0,14,67.152,0,10000257,10008616,MODELER,Hongkong
-473,2013-05-08,FP-GTC,20886,0,13,19.5685,0,10000122,10004150,ANALYST,Beijing
-474,2013-12-29,ABIN,1357,0,12,89.9069,0,10000900,10007759,MODELER,Hongkong
-475,2013-02-14,FP-GTC,63889,0,16,74.5657,0,10000676,10009640,ANALYST,Beijing
-476,2012-02-25,Auction,88750,0,-99,92.3939,0,10000769,10000718,MODELER,Hongkong
-477,2012-06-22,Others,13836,0,14,77.1277,0,10000795,10002041,MODELER,Hongkong
-478,2012-01-30,FP-GTC,43479,0,15,48.2346,0,10000013,10005919,ANALYST,Beijing
-479,2013-08-17,ABIN,57990,3,-99,87.0638,0,10000277,10006947,ADMIN,Shanghai
-480,2013-06-24,FP-GTC,1357,0,15,75.0985,0,10000056,10003367,ANALYST,Beijing
-481,2013-04-08,ABIN,57013,0,11,21.4485,0,10000195,10002888,ANALYST,Beijing
-482,2012-04-07,FP-non GTC,24541,0,-99,40.6197,0,10000313,10003959,ANALYST,Beijing
-483,2012-07-26,Others,963,0,16,99.8137,0,10000257,10009703,ANALYST,Beijing
-484,2012-07-18,Others,15115,0,-99,9.9153,0,10000937,10007448,MODELER,Hongkong
-485,2012-01-04,Auction,63889,0,13,84.5004,0,10000467,10002512,MODELER,Hongkong
-486,2013-05-25,Auction,1120,3,14,40.3618,0,10000076,10005541,MODELER,Hongkong
-487,2013-03-30,FP-GTC,38238,0,12,94.987,0,10000997,10005686,ANALYST,Beijing
-488,2012-05-12,Auction,32876,0,-99,38.58,0,10000943,10008237,ANALYST,Beijing
-489,2012-11-21,FP-GTC,100847,0,14,7.7495,0,10000588,10009021,MODELER,Hongkong
-490,2013-10-02,Auction,15868,0,5,92.2309,0,10000520,10000500,ADMIN,Shanghai
-491,2013-11-05,ABIN,31673,0,-99,75.9838,0,10000344,10003167,ANALYST,Beijing
-492,2013-03-23,FP-non GTC,48904,0,-99,15.426,0,10000090,10003494,MODELER,Hongkong
-493,2012-12-27,FP-non GTC,43479,0,5,35.7689,0,10000066,10002163,ANALYST,Beijing
-494,2013-01-11,FP-non GTC,223,0,12,5.9423,0,10000174,10005802,ANALYST,Beijing
-495,2012-05-25,Auction,50508,0,16,81.5853,0,10000337,10003087,MODELER,Hongkong
-496,2012-09-08,Others,6762,0,15,11.8473,0,10000718,10005439,ANALYST,Beijing
-497,2013-07-08,Auction,139973,0,5,11.7025,0,10000537,10004155,ANALYST,Beijing
-498,2013-04-23,ABIN,32876,0,12,40.4116,0,10000798,10004170,ADMIN,Shanghai
-499,2013-03-22,Auction,2023,0,15,89.8093,0,10000662,10004308,ANALYST,Beijing
-500,2012-11-01,Others,75708,3,11,95.9527,0,10000776,10000062,MODELER,Hongkong
-501,2012-01-29,ABIN,156356,0,16,64.6181,0,10000910,10009800,ADMIN,Shanghai
-502,2012-01-20,Others,2635,0,11,81.6196,0,10000893,10001706,ANALYST,Beijing
-503,2012-05-27,FP-non GTC,94847,0,15,87.1713,0,10000121,10005066,ANALYST,Beijing
-504,2013-12-17,Others,156614,0,14,87.7791,0,10000531,10000981,MODELER,Hongkong
-505,2013-03-05,ABIN,80135,0,16,64.8966,0,10000452,10009861,MODELER,Hongkong
-506,2013-01-26,ABIN,156356,0,16,59.0108,0,10000671,10008048,ADMIN,Shanghai
-507,2012-01-08,FP-non GTC,57990,0,15,56.347,0,10000028,10002420,ADMIN,Shanghai
-508,2013-08-04,ABIN,13836,0,11,98.5679,0,10000481,10007240,ADMIN,Shanghai
-509,2013-04-04,Others,63861,0,13,14.3911,0,10000944,10009613,ANALYST,Beijing
-510,2013-02-15,FP-GTC,2023,0,5,5.5878,0,10000816,10006549,MODELER,Hongkong
-511,2013-05-05,FP-non GTC,175750,0,-99,80.9471,0,10000727,10007640,MODELER,Hongkong
-512,2013-06-24,FP-GTC,38238,0,12,96.7825,0,10000202,10004824,ANALYST,Beijing
-513,2013-11-04,ABIN,45333,0,14,57.954,0,10000767,10002230,ANALYST,Beijing
-514,2012-02-23,FP-non GTC,57990,0,5,16.479,0,10000146,10002203,ADMIN,Shanghai
-515,2013-01-13,ABIN,158798,0,11,45.5371,0,10000133,10005287,ANALYST,Beijing
-516,2013-11-16,ABIN,73506,0,16,35.7207,0,10000304,10003454,ANALYST,Beijing
-517,2012-11-15,Auction,64076,0,15,66.0059,0,10000375,10009226,ANALYST,Beijing
-518,2012-01-03,ABIN,156614,0,15,84.8759,0,10000095,10004187,ADMIN,Shanghai
-519,2012-09-22,Auction,57990,0,14,42.7313,0,10000953,10005015,MODELER,Hongkong
-520,2013-11-03,FP-non GTC,67698,0,11,8.9322,0,10000913,10008723,MODELER,Hongkong
-521,2012-08-31,ABIN,1504,0,5,34.3706,0,10000748,10008788,ANALYST,Beijing
-522,2013-12-13,FP-GTC,15868,0,-99,18.3204,0,10000495,10005383,MODELER,Hongkong
-523,2012-07-20,ABIN,533,0,11,24.9451,0,10000237,10005440,MODELER,Hongkong
-524,2012-07-14,ABIN,41940,0,11,42.9796,0,10000328,10004188,ANALYST,Beijing
-525,2013-11-06,Others,32876,0,15,4.2693,0,10000383,10001156,ADMIN,Shanghai
-526,2012-07-02,Auction,41940,0,12,19.2987,0,10000196,10002110,ADMIN,Shanghai
-527,2012-06-21,FP-GTC,20865,0,-99,36.5029,0,10000098,10000543,ANALYST,Beijing
-528,2012-09-19,ABIN,159184,0,15,65.9768,0,10000517,10004195,ANALYST,Beijing
-529,2012-02-11,Others,16145,0,14,41.4762,0,10000733,10009727,ANALYST,Beijing
-530,2012-07-19,Auction,63861,0,11,30.7294,0,10000351,10001645,ANALYST,Beijing
-531,2013-05-24,ABIN,1349,0,12,61.0346,0,10000078,10006458,MODELER,Hongkong
-532,2013-06-26,Auction,139973,3,11,49.3849,0,10000537,10000240,ADMIN,Shanghai
-533,2013-11-30,Others,1357,0,16,62.0388,0,10000532,10003270,MODELER,Hongkong
-534,2012-11-01,Others,121153,0,5,64.8814,0,10000820,10007027,ANALYST,Beijing
-535,2013-10-20,Auction,20485,0,5,88.9093,0,10000467,10000657,MODELER,Hongkong
-536,2013-04-15,FP-GTC,43479,0,13,72.5969,0,10000352,10006790,MODELER,Hongkong
-537,2012-04-02,FP-GTC,33977,15,16,42.1845,0,10000614,10008912,ANALYST,Beijing
-538,2012-03-20,ABIN,2023,0,16,95.0254,0,10000014,10009645,ADMIN,Shanghai
-539,2013-02-21,FP-non GTC,57990,0,5,25.0404,0,10000386,10006749,ANALYST,Beijing
-540,2013-06-02,FP-non GTC,6762,0,12,60.3684,0,10000783,10002330,ADMIN,Shanghai
-541,2012-01-07,FP-GTC,759,0,11,80.172,0,10000952,10007127,MODELER,Hongkong
-542,2012-12-01,FP-non GTC,533,0,12,47.1653,0,10000550,10002624,ADMIN,Shanghai
-543,2012-11-03,ABIN,99985,0,16,40.828,0,10000629,10009256,MODELER,Hongkong
-544,2013-10-15,FP-GTC,64076,0,15,55.9541,0,10000531,10004447,ANALYST,Beijing
-545,2013-12-30,Auction,80135,0,13,13.2503,0,10000425,10001981,ADMIN,Shanghai
-546,2012-06-28,FP-GTC,15868,0,12,76.836,0,10000557,10006646,MODELER,Hongkong
-547,2013-04-07,FP-GTC,10866,0,16,37.754,0,10000810,10006760,ADMIN,Shanghai
-548,2013-08-03,FP-GTC,139255,15,13,70.8704,0,10000514,10004045,MODELER,Hongkong
-549,2012-07-23,ABIN,121153,0,5,65.6362,0,10000166,10009384,MODELER,Hongkong
-550,2013-05-04,FP-GTC,31519,0,5,38.9751,0,10000932,10001903,ANALYST,Beijing
-551,2013-01-27,ABIN,175750,3,-99,81.7736,0,10000529,10009081,ADMIN,Shanghai
-552,2013-03-30,ABIN,11554,0,15,37.8703,0,10000509,10009280,MODELER,Hongkong
-553,2012-06-03,FP-GTC,20485,0,-99,62.4729,0,10000404,10002401,MODELER,Hongkong
-554,2012-05-09,FP-non GTC,63861,0,11,35.1334,0,10000175,10008536,ADMIN,Shanghai
-555,2013-12-25,FP-non GTC,6762,0,12,95.0363,0,10000101,10005222,ANALYST,Beijing
-556,2012-11-18,FP-GTC,175750,0,5,95.8938,0,10000537,10003708,ADMIN,Shanghai
-557,2013-01-07,FP-non GTC,57784,0,14,0.2995,0,10000847,10001989,ADMIN,Shanghai
-558,2012-03-11,Auction,63861,0,12,58.1028,0,10000912,10006009,ANALYST,Beijing
-559,2012-03-23,Others,80287,0,14,24.79,0,10000967,10003972,ADMIN,Shanghai
-560,2012-03-19,ABIN,99985,0,16,5.9564,0,10000684,10007469,ADMIN,Shanghai
-561,2013-11-24,Auction,20886,0,15,28.1384,0,10000239,10009910,ADMIN,Shanghai
-562,2012-04-04,FP-GTC,100847,0,11,91.4446,0,10000743,10009881,ADMIN,Shanghai
-563,2013-12-15,ABIN,44079,0,16,39.686,0,10000476,10000011,MODELER,Hongkong
-564,2013-10-06,FP-GTC,132939,0,12,20.8855,0,10000854,10005709,ADMIN,Shanghai
-565,2013-12-07,Others,25147,0,14,16.4357,0,10000198,10007137,ADMIN,Shanghai
-566,2012-02-04,Others,158798,0,13,44.5451,0,10000543,10000252,MODELER,Hongkong
-567,2013-11-07,FP-GTC,10866,0,16,8.5249,0,10000105,10008626,ADMIN,Shanghai
-568,2012-08-04,FP-GTC,2635,0,12,11.7572,0,10000382,10009967,ADMIN,Shanghai
-569,2012-08-14,Auction,152801,0,15,46.6818,0,10000205,10008668,ADMIN,Shanghai
-570,2013-02-25,Auction,50508,0,15,37.8443,0,10000644,10000524,MODELER,Hongkong
-571,2013-10-09,ABIN,223,0,13,58.8049,0,10000768,10003846,ANALYST,Beijing
-572,2012-01-10,Auction,20485,0,11,92.2997,0,10000975,10007593,ADMIN,Shanghai
-573,2013-05-03,FP-non GTC,37831,0,12,15.9399,0,10000280,10006028,MODELER,Hongkong
-574,2013-11-12,ABIN,73506,0,12,13.9272,0,10000332,10004101,ANALYST,Beijing
-575,2013-08-16,Others,164,0,12,39.2371,0,10000887,10004134,MODELER,Hongkong
-576,2013-10-27,FP-non GTC,15868,0,16,51.4769,0,10000303,10004782,MODELER,Hongkong
-577,2013-04-27,FP-non GTC,80287,0,15,90.8971,0,10000207,10000144,MODELER,Hongkong
-578,2013-11-06,FP-non GTC,11554,0,12,26.0123,0,10000855,10007610,ADMIN,Shanghai
-579,2013-11-25,Others,6762,0,14,96.0198,0,10000418,10000846,ANALYST,Beijing
-580,2012-08-09,ABIN,80135,0,15,89.0544,0,10000861,10007561,MODELER,Hongkong
-581,2013-09-28,ABIN,16509,0,12,90.2997,0,10000520,10003310,MODELER,Hongkong
-582,2012-03-19,FP-GTC,43479,0,-99,93.8319,0,10000045,10002310,MODELER,Hongkong
-583,2012-05-25,FP-GTC,139973,0,16,8.9816,0,10000684,10006239,ADMIN,Shanghai
-584,2013-04-04,Auction,25147,0,11,43.4557,0,10000801,10002822,MODELER,Hongkong
-585,2013-07-07,Auction,67698,0,14,59.3634,0,10000305,10006424,ADMIN,Shanghai
-586,2013-10-27,ABIN,63861,0,14,48.1526,0,10000963,10001757,MODELER,Hongkong
-587,2012-05-31,Auction,130,0,5,1.056,0,10000357,10003239,MODELER,Hongkong
-588,2012-08-21,Auction,13836,0,16,14.4247,0,10000278,10003258,ADMIN,Shanghai
-589,2013-07-24,Auction,15568,15,14,14.5748,0,10000590,10006155,ADMIN,Shanghai
-590,2013-04-05,ABIN,12688,0,12,2.0636,0,10000951,10001457,ADMIN,Shanghai
-591,2013-02-27,FP-non GTC,73506,0,14,2.7139,0,10000134,10008130,ADMIN,Shanghai
-592,2013-06-16,FP-non GTC,164,0,5,59.4198,0,10000745,10008243,ANALYST,Beijing
-593,2012-04-01,Others,20213,0,11,77.849,0,10000929,10006004,ANALYST,Beijing
-594,2013-12-02,FP-non GTC,155226,0,16,71.11,0,10000433,10005008,ADMIN,Shanghai
-595,2013-06-07,Auction,158798,0,15,98.6517,0,10000320,10009500,ADMIN,Shanghai
-596,2012-11-25,ABIN,31519,0,12,68.8974,0,10000726,10000921,ADMIN,Shanghai
-597,2013-04-17,FP-GTC,57013,0,14,49.4405,0,10000050,10007518,ADMIN,Shanghai
-598,2013-03-18,FP-non GTC,175750,0,13,2.3784,0,10000559,10005674,ANALYST,Beijing
-599,2013-08-13,Others,62179,0,11,36.494,0,10000652,10003841,MODELER,Hongkong
-600,2012-10-15,ABIN,175750,0,14,42.7269,0,10000560,10007156,MODELER,Hongkong
-601,2012-01-08,Others,6762,0,5,58.5212,0,10000829,10005036,ADMIN,Shanghai
-602,2013-05-20,FP-non GTC,216,0,14,16.9644,0,10000104,10003344,ADMIN,Shanghai
-603,2012-12-07,Others,51582,0,5,6.0617,0,10000228,10008500,ANALYST,Beijing
-604,2013-12-04,ABIN,57013,0,16,48.2921,0,10000347,10008359,MODELER,Hongkong
-605,2013-12-17,ABIN,759,0,14,12.8117,0,10000161,10005247,ANALYST,Beijing
-606,2012-02-08,FP-non GTC,24760,0,13,11.319,0,10000938,10005304,MODELER,Hongkong
-607,2013-01-03,FP-non GTC,15568,15,14,0.3508,0,10000964,10003128,ADMIN,Shanghai
-608,2013-04-21,Others,87118,0,12,9.5385,0,10000949,10000218,ADMIN,Shanghai
-609,2013-11-13,Others,36250,0,12,0.9029,0,10000532,10000421,ANALYST,Beijing
-610,2013-03-18,FP-GTC,164,0,12,3.7727,0,10000237,10004397,ADMIN,Shanghai
-611,2013-06-29,Others,15868,0,11,23.3421,0,10000671,10003358,MODELER,Hongkong
-612,2013-12-25,Others,25147,0,5,99.4168,0,10000708,10008710,ANALYST,Beijing
-613,2012-10-06,FP-GTC,533,0,12,40.5069,0,10000295,10007759,MODELER,Hongkong
-614,2013-02-21,ABIN,216,0,14,84.3818,0,10000637,10004755,ADMIN,Shanghai
-615,2012-08-27,FP-non GTC,67698,0,16,7.0299,0,10000671,10009708,MODELER,Hongkong
-616,2013-04-13,Others,51582,0,13,67.973,0,10000778,10003242,ANALYST,Beijing
-617,2013-07-10,FP-GTC,139973,0,13,24.8423,0,10000253,10009806,ADMIN,Shanghai
-618,2013-11-02,ABIN,1349,0,14,56.631,0,10000922,10001129,ANALYST,Beijing
-619,2013-05-11,Others,26249,0,16,49.7279,0,10000610,10003206,ANALYST,Beijing
-620,2013-01-17,FP-non GTC,118687,3,11,34.6913,0,10000002,10003226,MODELER,Hongkong
-621,2012-07-02,Others,148324,15,-99,90.6687,0,10000613,10002537,ANALYST,Beijing
-622,2013-07-14,ABIN,41940,0,-99,60.2733,0,10000145,10004935,MODELER,Hongkong
-623,2013-06-13,Auction,63889,0,11,87.9644,0,10000920,10007836,MODELER,Hongkong
-624,2013-05-05,FP-non GTC,1120,3,16,99.4791,0,10000720,10004775,ADMIN,Shanghai
-625,2012-03-30,Others,4943,0,14,56.9643,0,10000105,10001645,ANALYST,Beijing
-626,2012-01-13,FP-non GTC,139973,3,-99,15.838,0,10000587,10009527,ANALYST,Beijing
-627,2012-09-02,ABIN,12688,0,13,69.4513,0,10000756,10008016,ANALYST,Beijing
-628,2013-08-08,Others,121153,0,11,29.2318,0,10000289,10000409,ADMIN,Shanghai
-629,2013-03-09,Others,99985,0,15,14.7661,0,10000453,10003921,MODELER,Hongkong
-630,2012-10-17,FP-non GTC,25147,0,11,96.1909,0,10000054,10004624,ADMIN,Shanghai
-631,2012-01-20,Others,73506,0,14,56.0157,0,10000909,10009618,ADMIN,Shanghai
-632,2012-04-15,FP-GTC,16145,0,12,94.6354,0,10000551,10008904,ANALYST,Beijing
-633,2013-08-08,Others,3838,0,13,85.7695,0,10000464,10003300,ANALYST,Beijing
-634,2013-01-21,Others,139973,0,13,64.5021,0,10000107,10004525,MODELER,Hongkong
-635,2012-06-17,Others,139973,0,-99,72.8065,0,10000495,10009258,MODELER,Hongkong
-636,2013-12-31,Others,20886,0,15,91.2326,0,10000746,10006331,MODELER,Hongkong
-637,2012-02-11,FP-non GTC,4943,0,14,28.3146,0,10000649,10003718,ANALYST,Beijing
-638,2013-10-01,Auction,155226,0,15,72.2196,0,10000576,10003692,MODELER,Hongkong
-639,2012-08-25,ABIN,223,0,15,17.6408,0,10000661,10004113,ANALYST,Beijing
-640,2012-07-10,Others,67698,0,15,70.9451,0,10000115,10000167,ANALYST,Beijing
-641,2013-12-27,FP-non GTC,26262,0,13,64.5694,0,10000740,10005696,MODELER,Hongkong
-642,2012-02-05,Auction,32996,15,12,9.9521,0,10000833,10009650,ANALYST,Beijing
-643,2012-11-07,FP-non GTC,25147,0,16,83.3886,0,10000494,10005692,MODELER,Hongkong
-644,2014-01-01,Others,45333,0,16,19.2606,0,10000869,10002121,ADMIN,Shanghai
-645,2013-10-05,Auction,50508,0,14,39.1465,0,10000904,10001424,ADMIN,Shanghai
-646,2012-02-25,FP-GTC,37831,0,11,12.4616,0,10000680,10003627,ANALYST,Beijing
-647,2012-05-16,Others,31519,0,12,66.9268,0,10000548,10008997,ADMIN,Shanghai
-648,2013-11-08,FP-GTC,100847,0,11,47.751,0,10000699,10006495,ADMIN,Shanghai
-649,2012-03-15,ABIN,106246,0,16,96.1592,0,10000922,10000661,ADMIN,Shanghai
-650,2013-05-15,Auction,64076,0,15,37.743,0,10000148,10009040,ADMIN,Shanghai
-651,2012-11-03,Auction,61323,0,15,14.3851,0,10000353,10002536,ADMIN,Shanghai
-652,2012-07-28,ABIN,175750,0,-99,45.5415,0,10000869,10009969,ANALYST,Beijing
-653,2013-06-26,FP-non GTC,95672,0,14,90.732,0,10000507,10005796,ANALYST,Beijing
-654,2012-09-11,Others,223,0,15,10.5185,0,10000705,10004209,ANALYST,Beijing
-655,2012-09-24,FP-GTC,43479,0,16,3.4611,0,10000991,10006760,ANALYST,Beijing
-656,2013-09-13,FP-GTC,100847,0,-99,51.0213,0,10000871,10003181,ANALYST,Beijing
-657,2013-07-01,Others,132939,0,16,14.6584,0,10000859,10006357,ADMIN,Shanghai
-658,2012-06-13,Others,26262,0,-99,46.0446,0,10000041,10006338,ANALYST,Beijing
-659,2013-10-26,Auction,3838,0,12,24.8196,0,10000561,10006508,ADMIN,Shanghai
-660,2012-03-05,Others,26262,0,13,66.7748,0,10000859,10005152,ANALYST,Beijing
-661,2012-12-10,FP-GTC,20485,0,12,52.3473,0,10000171,10002029,ANALYST,Beijing
-662,2013-11-15,Others,2023,0,5,84.6955,0,10000451,10006083,ADMIN,Shanghai
-663,2013-12-12,ABIN,80135,0,11,95.0112,0,10000718,10000889,ANALYST,Beijing
-664,2013-04-16,ABIN,106340,15,16,14.6367,0,10000822,10001648,ANALYST,Beijing
-665,2012-03-10,Auction,161567,15,16,5.28,0,10000081,10006504,ANALYST,Beijing
-666,2013-09-19,Auction,11848,0,11,27.6166,0,10000388,10008310,ADMIN,Shanghai
-667,2013-12-28,FP-GTC,216,0,15,90.1257,0,10000122,10004220,MODELER,Hongkong
-668,2012-01-26,ABIN,314,0,12,3.5634,0,10000912,10002566,ANALYST,Beijing
-669,2013-12-13,FP-GTC,158798,0,5,41.6652,0,10000485,10004610,MODELER,Hongkong
-670,2013-09-26,FP-non GTC,80135,0,11,20.5746,0,10000138,10008645,ANALYST,Beijing
-671,2012-01-20,Auction,62179,0,14,94.2904,0,10000310,10006436,ANALYST,Beijing
-672,2013-04-20,ABIN,963,0,5,68.3604,0,10000571,10003785,ADMIN,Shanghai
-673,2012-02-03,FP-GTC,164,0,11,94.0501,0,10000120,10004595,MODELER,Hongkong
-674,2013-10-14,Others,2023,0,12,84.9792,0,10000371,10008636,MODELER,Hongkong
-675,2013-08-19,FP-GTC,57990,3,14,57.0343,0,10000171,10008666,MODELER,Hongkong
-676,2012-06-18,Others,963,0,12,44.4186,0,10000343,10004277,MODELER,Hongkong
-677,2012-10-18,Others,31519,0,12,50.6753,0,10000875,10004037,MODELER,Hongkong
-678,2013-07-09,Others,36250,0,16,99.5251,0,10000446,10007730,MODELER,Hongkong
-679,2012-01-19,ABIN,38238,0,14,0.1259,0,10000242,10001967,ADMIN,Shanghai
-680,2013-05-19,FP-non GTC,13836,0,16,92.7668,0,10000895,10008811,MODELER,Hongkong
-681,2013-01-29,Others,95672,0,5,13.6012,0,10000843,10005299,ADMIN,Shanghai
-682,2013-01-30,ABIN,67698,0,5,13.9968,0,10000424,10008453,MODELER,Hongkong
-683,2012-09-02,FP-GTC,223,0,-99,38.0021,0,10000803,10006720,ADMIN,Shanghai
-684,2012-05-11,Auction,11848,0,13,7.191,0,10000964,10004802,ADMIN,Shanghai
-685,2013-05-14,FP-non GTC,759,0,16,43.172,0,10000123,10004560,ANALYST,Beijing
-686,2013-03-06,ABIN,3838,0,16,59.8866,0,10000653,10007059,ADMIN,Shanghai
-687,2012-08-23,FP-GTC,20213,0,16,91.3442,0,10000615,10001288,ANALYST,Beijing
-688,2013-03-24,FP-GTC,4943,0,-99,18.4365,0,10000718,10003403,MODELER,Hongkong
-689,2012-01-26,FP-non GTC,11554,0,14,41.6729,0,10000174,10006864,ADMIN,Shanghai
-690,2013-10-07,Others,108782,15,14,14.913,0,10000643,10005077,ADMIN,Shanghai
-691,2013-07-25,Others,13987,0,-99,28.7021,0,10000154,10008498,MODELER,Hongkong
-692,2013-11-08,FP-non GTC,31673,0,13,83.8884,0,10000059,10006354,MODELER,Hongkong
-693,2013-01-06,ABIN,82494,15,12,74.4582,0,10000023,10002631,MODELER,Hongkong
-694,2013-01-21,FP-non GTC,13987,0,11,72.4276,0,10000541,10008262,ANALYST,Beijing
-695,2013-11-22,Auction,67698,0,11,53.1908,0,10000553,10008665,ANALYST,Beijing
-696,2012-02-02,FP-non GTC,175750,0,12,79.116,0,10000121,10003008,ANALYST,Beijing
-697,2012-10-14,FP-GTC,24760,0,11,70.5894,0,10000677,10000508,ADMIN,Shanghai
-698,2012-10-20,ABIN,73506,0,11,3.7214,0,10000076,10001297,ANALYST,Beijing
-699,2013-12-01,Others,16145,0,15,54.8674,0,10000092,10003183,MODELER,Hongkong
-700,2012-11-07,FP-non GTC,53064,0,14,53.4385,0,10000267,10009566,MODELER,Hongkong
-701,2012-04-20,FP-non GTC,53064,0,15,76.5825,0,10000551,10002058,ANALYST,Beijing
-702,2012-11-18,Auction,50677,0,11,44.5149,0,10000924,10000200,MODELER,Hongkong
-703,2013-07-31,FP-GTC,51582,0,14,86.5325,0,10000652,10000550,MODELER,Hongkong
-704,2012-12-25,Others,31387,3,13,79.7624,0,10000495,10007108,ADMIN,Shanghai
-705,2012-08-10,ABIN,2635,0,12,54.5975,0,10000453,10000115,ADMIN,Shanghai
-706,2012-06-09,Others,31519,0,15,20.1828,0,10000226,10002561,ANALYST,Beijing
-707,2012-09-16,Others,44079,0,16,46.0626,0,10000193,10008238,ANALYST,Beijing
-708,2012-04-23,Others,16145,0,11,60.2064,0,10000058,10002015,ANALYST,Beijing
-709,2012-01-03,Others,164,0,15,94.1749,0,10000509,10000641,ADMIN,Shanghai
-710,2012-05-16,FP-GTC,13987,0,15,5.6834,0,10000052,10002235,ADMIN,Shanghai
-711,2012-07-24,FP-non GTC,16145,0,15,8.1625,0,10000036,10000988,ADMIN,Shanghai
-712,2013-01-01,FP-non GTC,32996,15,15,75.5057,0,10000393,10003712,ADMIN,Shanghai
-713,2013-01-28,FP-non GTC,155226,0,16,49.2395,0,10000966,10004827,ANALYST,Beijing
-714,2013-03-19,FP-GTC,20485,0,5,57.4433,0,10000138,10003150,ADMIN,Shanghai
-715,2013-02-03,FP-GTC,57784,0,5,11.8152,0,10000731,10007587,ADMIN,Shanghai
-716,2012-01-02,Others,53064,0,-99,45.8162,0,10000785,10006641,ANALYST,Beijing
-717,2013-02-23,ABIN,16145,0,14,25.1092,0,10000582,10000432,ANALYST,Beijing
-718,2013-04-19,Auction,159184,0,12,85.351,0,10000422,10005472,ANALYST,Beijing
-719,2012-10-12,FP-non GTC,139973,0,11,95.7331,0,10000791,10008481,ANALYST,Beijing
-720,2012-04-20,FP-non GTC,87118,0,16,15.2148,0,10000621,10001203,ADMIN,Shanghai
-721,2013-04-30,Auction,53064,0,5,89.5035,0,10000377,10006787,MODELER,Hongkong
-722,2013-10-24,FP-non GTC,73506,0,12,84.0199,0,10000503,10007044,ANALYST,Beijing
-723,2012-01-08,Auction,63861,0,13,74.886,0,10000589,10006905,ADMIN,Shanghai
-724,2012-05-21,Others,63861,3,16,63.7247,0,10000730,10007764,ANALYST,Beijing
-725,2013-01-29,Auction,24541,0,15,22.2189,0,10000167,10005939,ANALYST,Beijing
-726,2012-02-08,Auction,533,0,5,53.5867,0,10000564,10008398,MODELER,Hongkong
-727,2012-08-11,FP-non GTC,174053,3,13,0.5792,0,10000568,10009768,ANALYST,Beijing
-728,2012-07-05,FP-GTC,20886,0,12,31.1464,0,10000806,10007756,MODELER,Hongkong
-729,2012-01-08,Auction,156356,0,5,7.4293,0,10000638,10003893,MODELER,Hongkong
-730,2012-02-14,Others,65,0,14,62.6745,0,10000253,10000065,ADMIN,Shanghai
-731,2013-03-25,FP-non GTC,48027,0,-99,53.489,0,10000899,10004566,MODELER,Hongkong
-732,2012-09-30,Others,57013,0,12,75.9109,0,10000274,10004646,MODELER,Hongkong
-733,2013-09-01,ABIN,32876,0,12,75.0498,0,10000086,10004192,ADMIN,Shanghai
-734,2013-11-11,FP-GTC,164,0,12,36.6051,0,10000521,10001321,MODELER,Hongkong
-735,2012-02-18,FP-GTC,57013,0,12,14.3117,0,10000763,10008044,ANALYST,Beijing
-736,2013-11-23,ABIN,139973,0,15,58.3344,0,10000787,10008203,ANALYST,Beijing
-737,2012-03-21,FP-non GTC,41940,0,15,38.2502,0,10000342,10004965,MODELER,Hongkong
-738,2012-02-24,Auction,57990,0,5,73.5795,0,10000175,10007860,ANALYST,Beijing
-739,2013-07-20,Auction,88750,0,-99,58.2532,0,10000122,10002332,ANALYST,Beijing
-740,2012-10-17,Others,43479,0,13,67.9585,0,10000516,10003865,ANALYST,Beijing
-741,2012-11-03,FP-non GTC,118687,3,13,90.9173,0,10000957,10004057,ADMIN,Shanghai
-742,2012-10-22,FP-GTC,60340,0,14,68.5877,0,10000502,10003402,ANALYST,Beijing
-743,2013-04-25,FP-GTC,11554,0,5,60.828,0,10000074,10000561,ADMIN,Shanghai
-744,2012-10-23,FP-GTC,24760,0,13,6.0732,0,10000138,10001443,ANALYST,Beijing
-745,2013-04-29,ABIN,175750,0,11,8.0341,0,10000077,10006577,ADMIN,Shanghai
-746,2012-05-06,Auction,106340,15,14,8.0384,0,10000234,10008206,MODELER,Hongkong
-747,2012-07-15,ABIN,57784,0,14,27.1882,0,10000729,10004164,MODELER,Hongkong
-748,2012-10-15,Auction,67698,0,-99,63.179,0,10000226,10006851,MODELER,Hongkong
-749,2013-03-22,FP-non GTC,57013,0,14,27.7251,0,10000658,10009513,ADMIN,Shanghai
-750,2013-08-12,FP-non GTC,11848,0,13,38.5597,0,10000020,10004285,ANALYST,Beijing
-751,2012-07-14,FP-GTC,80053,0,-99,81.0002,0,10000850,10002887,ADMIN,Shanghai
-752,2012-08-12,FP-non GTC,38238,0,11,58.6987,0,10000377,10009099,ANALYST,Beijing
-753,2013-09-09,FP-GTC,106246,0,11,5.0352,0,10000594,10003969,MODELER,Hongkong
-754,2013-02-11,FP-non GTC,57784,0,12,80.842,0,10000657,10007844,ADMIN,Shanghai
-755,2013-03-09,Others,175750,0,14,87.9884,0,10000152,10002498,MODELER,Hongkong
-756,2013-12-18,FP-non GTC,20485,0,14,74.4627,0,10000272,10007976,MODELER,Hongkong
-757,2013-03-16,FP-non GTC,106246,0,5,64.1494,0,10000459,10002996,MODELER,Hongkong
-758,2012-09-06,FP-non GTC,80135,0,13,30.1506,0,10000269,10003262,MODELER,Hongkong
-759,2013-06-18,FP-non GTC,80135,0,-99,75.5826,0,10000706,10004201,ANALYST,Beijing
-760,2013-03-16,FP-GTC,156614,0,15,36.7157,0,10000761,10005335,MODELER,Hongkong
-761,2012-11-07,Others,1349,0,15,96.7575,0,10000612,10005478,ANALYST,Beijing
-762,2012-11-25,ABIN,95173,0,5,61.1186,0,10000364,10001034,MODELER,Hongkong
-763,2013-08-10,ABIN,31673,0,11,63.4095,0,10000421,10009510,ADMIN,Shanghai
-764,2012-02-17,ABIN,88750,0,14,87.8624,0,10000306,10006994,MODELER,Hongkong
-765,2013-03-29,ABIN,139973,0,16,9.1685,0,10000089,10003257,MODELER,Hongkong
-766,2012-12-27,ABIN,121153,0,12,88.7482,0,10000726,10003899,MODELER,Hongkong
-767,2013-10-13,FP-GTC,67698,0,16,20.8225,0,10000504,10008744,MODELER,Hongkong
-768,2012-09-26,ABIN,2023,0,11,83.6499,0,10000641,10007443,MODELER,Hongkong
-769,2012-08-01,Others,139973,0,-99,1.0542,0,10000202,10008156,MODELER,Hongkong
-770,2012-03-02,ABIN,113802,15,5,23.4439,0,10000068,10003049,MODELER,Hongkong
-771,2012-10-25,Others,10866,0,11,60.7453,0,10000770,10008471,ADMIN,Shanghai
-772,2013-01-07,FP-non GTC,20485,0,16,16.4177,0,10000698,10005385,ADMIN,Shanghai
-773,2012-03-18,FP-non GTC,46575,0,15,55.2768,0,10000962,10007105,MODELER,Hongkong
-774,2012-07-09,FP-GTC,106246,0,14,99.6346,0,10000563,10009652,MODELER,Hongkong
-775,2012-09-26,FP-GTC,314,0,13,45.8302,0,10000769,10002802,ANALYST,Beijing
-776,2012-01-14,FP-non GTC,15868,0,16,47.5496,0,10000869,10006392,ADMIN,Shanghai
-777,2012-01-01,ABIN,63861,0,13,76.1393,0,10000028,10003113,ADMIN,Shanghai
-778,2012-08-15,FP-GTC,26249,0,5,40.7448,0,10000419,10005625,ADMIN,Shanghai
-779,2012-09-24,FP-non GTC,43398,0,12,54.0864,0,10000688,10004711,MODELER,Hongkong
-780,2013-10-13,Others,533,0,-99,58.0312,0,10000658,10005979,MODELER,Hongkong
-781,2012-07-13,FP-GTC,148324,15,11,71.6067,0,10000006,10009327,ADMIN,Shanghai
-782,2012-01-21,FP-GTC,32996,15,11,86.8077,0,10000928,10006222,ADMIN,Shanghai
-783,2012-04-04,Others,63861,3,16,85.2671,0,10000648,10007439,MODELER,Hongkong
-784,2012-10-20,FP-non GTC,31519,0,13,99.193,0,10000469,10004261,ANALYST,Beijing
-785,2013-12-28,FP-GTC,1504,0,5,14.9821,0,10000342,10004429,ANALYST,Beijing
-786,2013-06-19,Others,25147,0,5,77.4671,0,10000212,10007671,ANALYST,Beijing
-787,2013-05-04,Others,13836,0,11,13.4655,0,10000331,10000004,MODELER,Hongkong
-788,2012-06-28,ABIN,145970,0,15,74.3304,0,10000900,10009524,ANALYST,Beijing
-789,2012-12-24,FP-non GTC,156614,0,12,79.4508,0,10000743,10003295,MODELER,Hongkong
-790,2013-07-17,FP-GTC,139973,0,13,30.0555,0,10000949,10007067,ADMIN,Shanghai
-791,2013-11-23,Others,31519,3,11,9.0845,0,10000883,10008607,MODELER,Hongkong
-792,2013-01-15,Others,164261,0,12,31.5563,0,10000029,10005351,ANALYST,Beijing
-793,2012-04-15,FP-non GTC,20213,0,-99,55.6917,0,10000513,10006690,ANALYST,Beijing
-794,2013-01-20,FP-non GTC,24760,0,15,71.7279,0,10000274,10003552,ANALYST,Beijing
-795,2012-11-13,FP-GTC,95173,0,5,95.7043,0,10000418,10004402,ANALYST,Beijing
-796,2012-09-03,ABIN,57990,0,12,1.5845,0,10000703,10007782,ANALYST,Beijing
-797,2012-01-02,FP-non GTC,57990,3,16,12.3706,0,10000090,10006504,ADMIN,Shanghai
-798,2013-12-06,Auction,67703,3,5,70.4134,0,10000753,10007346,MODELER,Hongkong
-799,2012-06-09,Others,50677,0,12,88.7297,0,10000990,10004575,MODELER,Hongkong
-800,2012-05-17,ABIN,31519,0,14,37.707,0,10000326,10001200,ADMIN,Shanghai
-801,2012-03-03,Others,156356,0,16,50.7844,0,10000693,10008188,MODELER,Hongkong
-802,2012-05-21,Auction,44079,0,5,86.759,0,10000795,10003443,ADMIN,Shanghai
-803,2013-10-15,Auction,26262,0,15,56.3327,0,10000077,10000771,MODELER,Hongkong
-804,2013-12-27,ABIN,20213,0,13,96.1815,0,10000195,10005442,ADMIN,Shanghai
-805,2012-02-15,ABIN,314,0,5,6.5869,0,10000753,10002892,ANALYST,Beijing
-806,2012-08-17,FP-non GTC,6762,0,15,41.0782,0,10000797,10005277,ADMIN,Shanghai
-807,2013-04-29,Auction,145970,0,15,11.0876,0,10000208,10001555,ADMIN,Shanghai
-808,2013-10-27,Auction,80053,0,14,81.7435,0,10000310,10001507,ADMIN,Shanghai
-809,2012-12-21,FP-GTC,63861,0,13,57.9491,0,10000971,10000647,MODELER,Hongkong
-810,2013-07-29,ABIN,45333,0,12,33.5405,0,10000475,10009395,ADMIN,Shanghai
-811,2012-04-18,Auction,64076,0,13,83.3621,0,10000697,10004193,ANALYST,Beijing
-812,2012-12-14,FP-non GTC,63889,0,16,0.964,0,10000607,10000685,ANALYST,Beijing
-813,2013-06-07,Auction,145970,0,13,6.1316,0,10000450,10000965,ANALYST,Beijing
-814,2012-11-27,Others,164,0,15,53.7718,0,10000305,10002326,ADMIN,Shanghai
-815,2012-08-03,ABIN,61323,0,15,62.2945,0,10000214,10006731,ANALYST,Beijing
-816,2012-07-07,ABIN,75665,0,15,77.0551,0,10000094,10002517,ANALYST,Beijing
-817,2012-08-06,FP-non GTC,99985,0,13,39.1253,0,10000609,10004893,MODELER,Hongkong
-818,2012-10-01,Auction,759,0,15,69.179,0,10000199,10003906,MODELER,Hongkong
-819,2012-03-22,Auction,20865,0,5,40.78,0,10000392,10003136,MODELER,Hongkong
-820,2012-02-06,FP-non GTC,132939,0,11,66.8705,0,10000207,10008892,ADMIN,Shanghai
-821,2012-10-17,FP-non GTC,63861,3,14,35.8018,0,10000096,10008663,ANALYST,Beijing
-822,2013-02-21,Others,41940,0,5,87.4721,0,10000858,10006568,MODELER,Hongkong
-823,2012-11-16,FP-GTC,57784,0,15,96.2799,0,10000537,10003039,ADMIN,Shanghai
-824,2012-01-10,Others,158666,15,11,7.4077,0,10000190,10005422,ADMIN,Shanghai
-825,2013-10-29,FP-non GTC,65,0,-99,21.3439,0,10000182,10003311,ADMIN,Shanghai
-826,2013-07-30,Auction,103178,15,-99,94.8679,0,10000943,10007911,ADMIN,Shanghai
-827,2012-08-05,FP-non GTC,175750,0,14,10.098,0,10000212,10006111,ADMIN,Shanghai
-828,2012-07-29,ABIN,24760,0,13,44.527,0,10000168,10008551,ANALYST,Beijing
-829,2013-04-27,FP-GTC,57990,3,-99,75.6664,0,10000878,10005106,ANALYST,Beijing
-830,2012-12-06,FP-GTC,37831,0,11,57.1191,0,10000090,10006324,ADMIN,Shanghai
-831,2013-08-29,FP-GTC,164,0,16,75.2345,0,10000689,10007653,ANALYST,Beijing
-832,2012-08-30,FP-GTC,63861,0,12,21.2099,0,10000643,10009845,MODELER,Hongkong
-833,2013-05-30,ABIN,57013,0,15,28.841,0,10000290,10009369,ADMIN,Shanghai
-834,2013-10-17,ABIN,50677,0,-99,16.3836,0,10000705,10006136,ANALYST,Beijing
-835,2012-04-25,FP-GTC,67698,0,11,52.3502,0,10000257,10002325,ANALYST,Beijing
-836,2013-08-18,ABIN,759,0,16,23.8258,0,10000911,10009193,ADMIN,Shanghai
-837,2013-02-24,Others,13836,0,-99,40.6826,0,10000399,10004991,ADMIN,Shanghai
-838,2012-07-22,FP-non GTC,158798,0,15,94.1886,0,10000705,10005096,ANALYST,Beijing
-839,2012-08-05,FP-GTC,100847,0,-99,74.15,0,10000797,10006264,ADMIN,Shanghai
-840,2012-12-30,Auction,31519,0,11,69.1499,0,10000576,10002471,ANALYST,Beijing
-841,2013-05-18,Auction,63864,3,14,44.5408,0,10000232,10006928,ANALYST,Beijing
-842,2012-11-01,ABIN,53064,0,15,19.9817,0,10000065,10004624,ADMIN,Shanghai
-843,2013-06-25,FP-GTC,156614,0,16,91.4652,0,10000326,10006437,ANALYST,Beijing
-844,2013-07-11,FP-non GTC,10058,3,5,92.6671,0,10000787,10002813,ADMIN,Shanghai
-845,2013-01-19,FP-non GTC,13987,0,12,49.0313,0,10000556,10005696,ADMIN,Shanghai
-846,2013-06-17,Auction,94847,0,13,95.5377,0,10000047,10001497,ANALYST,Beijing
-847,2012-12-23,Others,64076,0,-99,50.1173,0,10000063,10003567,MODELER,Hongkong
-848,2013-01-07,FP-non GTC,67698,0,5,6.5979,0,10000412,10003921,ADMIN,Shanghai
-849,2012-01-20,ABIN,175750,3,16,79.6931,0,10000270,10000997,ADMIN,Shanghai
-850,2013-01-14,Others,95672,3,12,31.7315,0,10000044,10002298,ANALYST,Beijing
-851,2013-02-22,Others,26249,0,16,43.7969,0,10000634,10003500,ADMIN,Shanghai
-852,2013-04-13,FP-non GTC,156614,0,16,79.3213,0,10000321,10003186,ADMIN,Shanghai
-853,2013-09-18,FP-non GTC,43398,0,5,17.3702,0,10000621,10002418,MODELER,Hongkong
-854,2013-09-05,ABIN,67703,3,11,55.6439,0,10000784,10004250,ANALYST,Beijing
-855,2013-05-25,FP-GTC,43398,0,15,73.0548,0,10000921,10007198,MODELER,Hongkong
-856,2012-10-06,Others,100847,0,16,77.0258,0,10000606,10009841,MODELER,Hongkong
-857,2013-11-08,Others,164261,0,11,16.6959,0,10000136,10006656,ANALYST,Beijing
-858,2012-04-13,FP-non GTC,3838,0,5,68.4347,0,10000451,10000115,ADMIN,Shanghai
-859,2012-07-03,FP-GTC,63861,0,-99,89.5962,0,10000893,10006463,ADMIN,Shanghai
-860,2012-02-21,Auction,40059,3,-99,84.7455,0,10000613,10009000,MODELER,Hongkong
-861,2012-02-25,FP-non GTC,759,0,13,23.9258,0,10000826,10005739,MODELER,Hongkong
-862,2013-10-30,FP-non GTC,57990,0,16,88.4859,0,10000782,10006594,MODELER,Hongkong
-863,2013-10-12,FP-non GTC,57784,0,14,85.8986,0,10000220,10004445,MODELER,Hongkong
-864,2012-08-15,FP-GTC,223,0,12,32.1974,0,10000316,10002696,ANALYST,Beijing
-865,2012-10-18,FP-GTC,15115,0,16,52.5243,0,10000353,10009319,ADMIN,Shanghai
-866,2013-11-09,FP-GTC,20886,0,13,94.0128,0,10000690,10004603,MODELER,Hongkong
-867,2012-09-27,ABIN,31673,0,12,24.1332,0,10000404,10009648,ANALYST,Beijing
-868,2012-05-08,FP-GTC,95672,0,16,88.8853,0,10000984,10002017,ADMIN,Shanghai
-869,2013-01-14,Others,95173,0,12,86.3987,0,10000501,10002846,ADMIN,Shanghai
-870,2012-12-02,Auction,152801,0,-99,71.8799,0,10000021,10009050,ADMIN,Shanghai
-871,2012-03-13,ABIN,31519,3,15,96.8998,0,10000188,10009876,ADMIN,Shanghai
-872,2013-06-15,Auction,121153,0,14,66.2729,0,10000101,10001578,ANALYST,Beijing
-873,2012-04-11,Others,63861,0,15,88.0097,0,10000536,10005703,ADMIN,Shanghai
-874,2013-01-25,FP-non GTC,1357,0,16,49.8991,0,10000637,10008932,ANALYST,Beijing
-875,2013-10-17,FP-GTC,106246,0,15,92.2507,0,10000046,10005384,MODELER,Hongkong
-876,2012-10-04,Others,4943,0,13,88.8498,0,10000215,10001232,ADMIN,Shanghai
-877,2013-05-14,Auction,314,0,13,77.3441,0,10000900,10008675,ANALYST,Beijing
-878,2013-03-17,FP-non GTC,759,0,16,17.9719,0,10000759,10000828,MODELER,Hongkong
-879,2013-01-20,FP-non GTC,51582,0,16,3.6644,0,10000961,10000593,ADMIN,Shanghai
-880,2013-03-20,FP-non GTC,57990,0,12,9.368,0,10000846,10009003,ANALYST,Beijing
-881,2012-06-06,FP-non GTC,165888,0,16,9.4584,0,10000312,10004071,ADMIN,Shanghai
-882,2012-12-13,Auction,43479,0,16,84.0632,0,10000349,10004913,MODELER,Hongkong
-883,2012-05-13,FP-non GTC,139973,3,5,37.351,0,10000790,10000443,MODELER,Hongkong
-884,2013-03-27,Auction,48027,0,5,75.7167,0,10000424,10006108,MODELER,Hongkong
-885,2013-08-08,FP-GTC,37831,0,-99,27.78,0,10000637,10000193,ANALYST,Beijing
-886,2012-04-06,ABIN,314,0,12,18.3274,0,10000749,10007076,MODELER,Hongkong
-887,2013-10-11,FP-non GTC,57013,0,14,39.1868,0,10000599,10004702,MODELER,Hongkong
-888,2012-01-18,FP-GTC,41940,0,12,94.5165,0,10000102,10003299,MODELER,Hongkong
-889,2013-05-20,ABIN,50677,0,16,25.6136,0,10000423,10001434,ANALYST,Beijing
-890,2012-06-24,FP-GTC,963,0,13,20.9442,0,10000069,10005352,ADMIN,Shanghai
-891,2013-04-01,ABIN,75665,0,12,65.3307,0,10000543,10006729,MODELER,Hongkong
-892,2012-02-05,ABIN,50508,0,-99,69.4516,0,10000623,10004814,ANALYST,Beijing
-893,2012-06-24,ABIN,314,0,5,13.8612,0,10000347,10003703,ANALYST,Beijing
-894,2013-12-07,Others,1349,0,16,10.8619,0,10000784,10000834,MODELER,Hongkong
-895,2013-03-01,ABIN,73506,0,11,7.7818,0,10000882,10001213,ADMIN,Shanghai
-896,2012-08-24,Auction,45333,0,16,94.9583,0,10000495,10002795,ANALYST,Beijing
-897,2012-08-04,FP-non GTC,20886,0,15,69.73,0,10000428,10004891,MODELER,Hongkong
-898,2012-11-08,Auction,31519,0,15,41.4423,0,10000431,10007344,MODELER,Hongkong
-899,2013-08-07,FP-GTC,156356,0,13,44.6853,0,10000305,10007928,ANALYST,Beijing
-900,2013-09-25,ABIN,43398,0,16,46.5536,0,10000387,10005678,ANALYST,Beijing
-901,2012-04-26,FP-non GTC,95672,0,13,30.9202,0,10000591,10009900,MODELER,Hongkong
-902,2012-03-30,Others,159184,0,12,58.3993,0,10000381,10004902,ADMIN,Shanghai
-903,2012-04-25,Auction,73506,0,14,46.1501,0,10000010,10005565,ANALYST,Beijing
-904,2012-09-13,FP-GTC,20485,0,5,2.4112,0,10000959,10005950,ADMIN,Shanghai
-905,2013-09-27,ABIN,80287,0,-99,64.4361,0,10000770,10006064,ANALYST,Beijing
-906,2013-01-17,Others,36250,0,11,73.5773,0,10000948,10009683,ADMIN,Shanghai
-907,2012-07-07,Auction,103324,15,16,4.8559,0,10000123,10000205,MODELER,Hongkong
-908,2013-08-15,ABIN,13836,0,16,25.8438,0,10000851,10008521,ANALYST,Beijing
-909,2012-07-07,Others,12688,0,5,3.4934,0,10000096,10007124,MODELER,Hongkong
-910,2012-06-03,Auction,80135,0,12,17.8258,0,10000144,10002338,ANALYST,Beijing
-911,2013-12-24,FP-GTC,60340,0,11,22.6409,0,10000341,10002484,MODELER,Hongkong
-912,2012-10-06,ABIN,36250,0,5,7.4322,0,10000384,10004712,ADMIN,Shanghai
-913,2013-01-03,Auction,175750,0,15,14.1976,0,10000366,10000904,ADMIN,Shanghai
-914,2013-04-15,Auction,88750,0,11,98.6057,0,10000584,10004925,MODELER,Hongkong
-915,2013-06-03,FP-non GTC,1349,0,15,11.0419,0,10000866,10004642,ANALYST,Beijing
-916,2013-01-30,FP-GTC,2635,0,14,54.5175,0,10000959,10002206,ANALYST,Beijing
-917,2013-08-27,FP-non GTC,26249,0,13,50.0365,0,10000289,10001579,MODELER,Hongkong
-918,2012-05-14,FP-non GTC,9426,3,5,60.0224,0,10000730,10008731,ADMIN,Shanghai
-919,2012-10-16,FP-non GTC,16145,0,-99,53.0896,0,10000195,10008596,MODELER,Hongkong
-920,2012-09-24,FP-GTC,57013,0,14,81.7929,0,10000498,10000826,MODELER,Hongkong
-921,2012-08-20,FP-non GTC,25147,0,-99,30.7605,0,10000242,10007503,ADMIN,Shanghai
-922,2012-02-19,Others,94847,0,13,20.6395,0,10000707,10009034,ANALYST,Beijing
-923,2013-11-14,ABIN,50508,0,15,91.9217,0,10000208,10003230,ADMIN,Shanghai
-924,2012-07-10,Others,95672,0,13,21.1886,0,10000762,10006779,ADMIN,Shanghai
-925,2012-03-14,ABIN,44079,0,13,89.6838,0,10000227,10003305,ADMIN,Shanghai
-926,2012-01-17,FP-non GTC,314,0,14,44.3066,0,10000251,10003512,ADMIN,Shanghai
-927,2012-02-27,FP-non GTC,15868,0,14,80.8427,0,10000741,10008344,ANALYST,Beijing
-928,2012-10-23,Others,1161,3,5,54.0104,0,10000289,10008643,ADMIN,Shanghai
-929,2013-06-12,Others,53064,0,12,87.1336,0,10000871,10007639,ANALYST,Beijing
-930,2013-03-06,Auction,34273,100,15,6.7044,0,10000269,10006027,MODELER,Hongkong
-931,2012-06-16,Others,533,0,14,43.7104,0,10000172,10005634,ADMIN,Shanghai
-932,2012-12-19,FP-GTC,15868,0,15,94.1727,0,10000366,10002612,ANALYST,Beijing
-933,2012-07-21,ABIN,121153,0,12,42.3437,0,10000688,10001422,ANALYST,Beijing
-934,2013-08-19,FP-non GTC,65,0,11,3.0275,0,10000055,10008305,MODELER,Hongkong
-935,2012-08-08,Auction,15687,0,14,72.9505,0,10000725,10008549,ADMIN,Shanghai
-936,2013-12-13,FP-GTC,4943,0,-99,37.7781,0,10000932,10002759,MODELER,Hongkong
-937,2013-05-18,FP-GTC,3838,0,14,45.1642,0,10000370,10000382,ANALYST,Beijing
-938,2012-01-31,Auction,25147,0,14,78.658,0,10000721,10005361,MODELER,Hongkong
-939,2013-12-05,FP-GTC,62179,0,12,76.4692,0,10000423,10007624,MODELER,Hongkong
-940,2012-03-07,Auction,175750,0,15,19.5357,0,10000053,10004204,MODELER,Hongkong
-941,2013-09-30,Others,100847,0,12,38.2299,0,10000772,10005476,ADMIN,Shanghai
-942,2013-01-07,Others,15115,0,12,82.5099,0,10000864,10003344,ADMIN,Shanghai
-943,2013-11-04,FP-non GTC,57013,0,-99,61.3731,0,10000335,10008636,MODELER,Hongkong
-944,2012-08-11,FP-non GTC,31673,0,12,12.7001,0,10000303,10001980,MODELER,Hongkong
-945,2012-12-30,ABIN,156356,0,14,99.3562,0,10000427,10001133,ANALYST,Beijing
-946,2013-05-19,Others,113593,15,11,31.1634,0,10000361,10006272,ADMIN,Shanghai
-947,2013-06-03,ABIN,314,0,13,32.6968,0,10000373,10008996,ANALYST,Beijing
-948,2013-06-29,Auction,15808,15,12,18.3967,0,10000484,10007850,ADMIN,Shanghai
-949,2012-06-10,FP-GTC,61323,0,13,10.8143,0,10000403,10006130,ADMIN,Shanghai
-950,2012-01-29,FP-GTC,1349,0,15,0.4358,0,10000639,10000947,ANALYST,Beijing
-951,2013-05-30,FP-non GTC,759,0,14,19.592,0,10000116,10002996,MODELER,Hongkong
-952,2013-07-28,ABIN,15868,0,14,59.7123,0,10000668,10009144,ADMIN,Shanghai
-953,2013-03-23,Auction,165888,0,12,67.1201,0,10000063,10003845,MODELER,Hongkong
-954,2012-05-29,FP-GTC,118687,3,-99,98.203,0,10000886,10004781,ADMIN,Shanghai
-955,2013-09-15,FP-non GTC,156614,0,5,43.1795,0,10000302,10007429,ANALYST,Beijing
-956,2012-05-21,FP-GTC,223,0,-99,66.285,0,10000118,10000885,ANALYST,Beijing
-957,2012-04-03,FP-non GTC,26262,0,12,17.5241,0,10000440,10008021,ADMIN,Shanghai
-958,2012-05-22,ABIN,44079,0,16,25.2405,0,10000183,10001255,ANALYST,Beijing
-959,2014-01-01,Others,50508,0,16,92.5843,0,10000911,10004209,MODELER,Hongkong
-960,2012-04-08,Others,16145,3,12,90.2242,0,10000825,10003876,ADMIN,Shanghai
-961,2013-11-27,ABIN,95672,0,16,90.938,0,10000829,10004065,ANALYST,Beijing
-962,2013-03-22,Auction,41940,0,15,72.0648,0,10000795,10008608,ANALYST,Beijing
-963,2012-04-06,Others,64076,0,14,86.9185,0,10000356,10008162,ADMIN,Shanghai
-964,2013-05-02,Others,156356,0,11,41.6871,0,10000122,10003384,ANALYST,Beijing
-965,2012-04-02,Others,132939,0,12,75.8372,0,10000395,10007751,ADMIN,Shanghai
-966,2012-06-29,ABIN,26262,0,13,83.6043,0,10000120,10009086,ADMIN,Shanghai
-967,2012-07-06,FP-GTC,11554,0,16,38.4329,0,10000352,10001074,MODELER,Hongkong
-968,2012-06-10,ABIN,156614,0,-99,94.1812,0,10000280,10003259,ANALYST,Beijing
-969,2012-04-05,Auction,164261,0,12,0.9691,0,10000480,10008760,ANALYST,Beijing
-970,2012-11-02,Auction,31519,0,11,98.0505,0,10000893,10008194,MODELER,Hongkong
-971,2012-05-20,Auction,50508,0,15,61.9439,0,10000363,10006222,ANALYST,Beijing
-972,2012-03-31,Auction,80135,0,16,11.1543,0,10000692,10009054,MODELER,Hongkong
-973,2012-07-01,FP-non GTC,63889,0,-99,86.6557,0,10000402,10009722,MODELER,Hongkong
-974,2013-01-04,ABIN,41940,0,-99,48.4505,0,10000296,10008506,ANALYST,Beijing
-975,2013-02-10,FP-non GTC,63861,0,13,60.3535,0,10000214,10006412,ADMIN,Shanghai
-976,2012-03-25,Others,73506,0,11,74.2113,0,10000801,10006619,ANALYST,Beijing
-977,2013-07-23,FP-non GTC,53064,0,16,69.2368,0,10000542,10003961,MODELER,Hongkong
-978,2012-04-09,Auction,11848,0,15,76.172,0,10000555,10009344,ADMIN,Shanghai
-979,2012-08-01,Others,12688,0,5,85.4589,0,10000130,10005142,ADMIN,Shanghai
-980,2012-07-15,FP-non GTC,31519,3,-99,17.8485,0,10000045,10007670,MODELER,Hongkong
-981,2013-10-06,ABIN,1349,0,11,52.278,0,10000622,10005595,MODELER,Hongkong
-982,2012-10-26,Auction,25147,0,13,60.9912,0,10000379,10009391,ADMIN,Shanghai
-983,2013-01-04,FP-non GTC,37831,0,15,61.0134,0,10000785,10003394,ADMIN,Shanghai
-984,2012-03-24,Auction,139973,0,-99,8.1533,0,10000487,10007074,ADMIN,Shanghai
-985,2012-12-30,Auction,6762,0,-99,77.8179,0,10000645,10001838,ADMIN,Shanghai
-986,2012-10-03,ABIN,103178,15,12,15.2201,0,10000824,10002677,MODELER,Hongkong
-987,2012-04-18,FP-non GTC,50508,0,14,11.9715,0,10000012,10006191,MODELER,Hongkong
-988,2012-09-30,FP-non GTC,31673,0,12,90.0864,0,10000281,10002287,MODELER,Hongkong
-989,2013-03-24,ABIN,4943,0,12,11.0294,0,10000785,10001441,ADMIN,Shanghai
-990,2012-05-02,Others,87118,0,14,52.6549,0,10000924,10004868,ADMIN,Shanghai
-991,2013-05-03,FP-GTC,24760,0,5,8.9077,0,10000758,10004142,MODELER,Hongkong
-992,2012-08-05,FP-non GTC,38238,0,11,0.8463,0,10000066,10003073,MODELER,Hongkong
-993,2012-08-15,Auction,67698,0,11,23.6278,0,10000085,10007036,ADMIN,Shanghai
-994,2013-04-04,Auction,164261,0,13,52.3718,0,10000965,10008307,ADMIN,Shanghai
-995,2013-04-18,ABIN,25147,0,14,71.7687,0,10000157,10001658,ANALYST,Beijing
-996,2013-11-09,ABIN,43479,0,12,39.9269,0,10000024,10007805,ANALYST,Beijing
-997,2013-05-02,FP-non GTC,175750,0,12,17.6149,0,10000862,10001964,ANALYST,Beijing
-998,2012-09-01,ABIN,50508,0,14,99.1518,0,10000746,10009362,ANALYST,Beijing
-999,2013-02-23,FP-non GTC,139255,15,-99,74.6182,0,10000447,10005318,ANALYST,Beijing
-1000,2012-01-19,FP-non GTC,6762,0,13,0.1297,0,10000189,10004748,MODELER,Hongkong
-1001,2012-11-10,Auction,113802,15,-99,11.9583,0,10000596,10006655,ANALYST,Beijing
-1002,2012-12-11,FP-non GTC,95173,0,5,67.9416,0,10000582,10003627,ADMIN,Shanghai
-1003,2012-01-05,ABIN,2635,0,11,72.3723,0,10000375,10002658,ANALYST,Beijing
-1004,2012-05-04,FP-non GTC,43479,0,13,79.1307,0,10000944,10007826,ADMIN,Shanghai
-1005,2013-07-20,ABIN,63861,0,13,86.7155,0,10000274,10005880,MODELER,Hongkong
-1006,2013-08-25,FP-GTC,80053,0,5,11.7108,0,10000059,10001987,MODELER,Hongkong
-1007,2012-10-24,ABIN,38238,0,5,79.2964,0,10000388,10000426,MODELER,Hongkong
-1008,2012-01-10,ABIN,73506,0,5,3.8878,0,10000895,10007256,ANALYST,Beijing
-1009,2013-07-11,Auction,174106,3,12,46.7087,0,10000683,10000752,ANALYST,Beijing
-1010,2013-06-16,FP-GTC,61323,0,15,64.0224,0,10000121,10001535,ADMIN,Shanghai
-1011,2013-06-23,ABIN,63889,0,5,59.2766,0,10000466,10003627,MODELER,Hongkong
-1012,2013-01-25,Others,156356,0,14,79.9992,0,10000611,10002293,ANALYST,Beijing
-1013,2013-10-09,Auction,50508,0,16,70.9682,0,10000009,10006058,MODELER,Hongkong
-1014,2013-05-19,Auction,25147,0,16,8.0973,0,10000731,10007700,ADMIN,Shanghai
-1015,2012-10-14,Auction,67698,0,5,49.0497,0,10000605,10009329,ADMIN,Shanghai
-1016,2012-06-08,Auction,65,0,14,76.5149,0,10000165,10008171,MODELER,Hongkong
-1017,2012-12-21,FP-GTC,15868,0,5,45.9224,0,10000215,10004400,ADMIN,Shanghai
-1018,2012-12-02,Auction,1357,0,15,8.9337,0,10000730,10003527,ANALYST,Beijing
-1019,2013-11-27,ABIN,24760,0,11,62.4675,0,10000932,10006658,ADMIN,Shanghai
-1020,2013-11-24,Auction,95173,0,16,16.7927,0,10000795,10004974,ANALYST,Beijing
-1021,2013-11-30,ABIN,164,0,11,10.8897,0,10000914,10001759,MODELER,Hongkong
-1022,2012-12-22,Auction,15868,0,15,7.4153,0,10000540,10006963,MODELER,Hongkong
-1023,2012-05-13,FP-non GTC,26249,0,5,58.8647,0,10000898,10002705,MODELER,Hongkong
-1024,2013-06-22,ABIN,95672,3,12,71.8628,0,10000026,10009984,ANALYST,Beijing
-1025,2013-05-22,ABIN,63861,0,11,71.8836,0,10000650,10003996,ADMIN,Shanghai
-1026,2012-11-17,FP-GTC,32876,0,12,35.6019,0,10000959,10000392,ADMIN,Shanghai
-1027,2013-04-13,Auction,38238,0,5,21.7443,0,10000036,10001425,ANALYST,Beijing
-1028,2012-09-09,FP-non GTC,87118,0,14,46.2858,0,10000512,10005447,MODELER,Hongkong
-1029,2013-08-23,FP-non GTC,41940,0,11,98.4779,0,10000844,10007297,MODELER,Hongkong
-1030,2012-09-02,FP-non GTC,43398,0,12,91.8543,0,10000917,10008944,ANALYST,Beijing
-1031,2012-11-23,Others,145970,0,11,43.8631,0,10000997,10007702,ANALYST,Beijing
-1032,2012-01-22,FP-non GTC,164261,0,16,74.9772,0,10000091,10002303,MODELER,Hongkong
-1033,2012-05-02,ABIN,152801,0,13,15.2794,0,10000649,10004568,ADMIN,Shanghai
-1034,2013-08-14,Others,50677,0,13,26.5329,0,10000275,10008334,MODELER,Hongkong
-1035,2012-06-04,Others,156614,0,14,42.781,0,10000347,10003401,ANALYST,Beijing
-1036,2012-08-04,FP-non GTC,145970,0,15,92.6953,0,10000622,10002124,ADMIN,Shanghai
-1037,2012-08-15,Auction,32876,0,16,35.2015,0,10000533,10001315,MODELER,Hongkong
-1038,2012-11-05,Auction,95672,0,11,0.5233,0,10000965,10004000,MODELER,Hongkong
-1039,2013-10-12,ABIN,165888,0,11,27.3564,0,10000036,10006840,ADMIN,Shanghai
-1040,2012-10-22,ABIN,20485,0,14,28.6742,0,10000448,10004840,ANALYST,Beijing
-1041,2012-07-02,Auction,20213,0,15,81.3332,0,10000484,10007894,ADMIN,Shanghai
-1042,2013-01-13,Others,152801,0,11,3.3406,0,10000124,10006403,MODELER,Hongkong
-1043,2012-05-17,Others,132939,0,15,55.3503,0,10000086,10007815,MODELER,Hongkong
-1044,2013-02-05,Others,48904,0,15,25.234,0,10000246,10002491,MODELER,Hongkong
-1045,2013-09-13,FP-GTC,32876,0,-99,10.7672,0,10000729,10002514,ANALYST,Beijing
-1046,2012-06-14,ABIN,20865,0,14,47.9841,0,10000406,10009589,ADMIN,Shanghai
-1047,2013-02-11,Auction,152801,0,5,86.4355,0,10000050,10001255,ANALYST,Beijing
-1048,2012-12-13,Auction,50508,0,5,72.5114,0,10000715,10009705,ANALYST,Beijing
-1049,2013-11-30,Auction,3838,0,-99,6.8517,0,10000706,10009022,ADMIN,Shanghai
-1050,2013-03-15,FP-non GTC,20865,0,14,93.304,0,10000926,10002565,MODELER,Hongkong
-1051,2012-08-12,FP-GTC,57784,0,-99,18.4045,0,10000231,10006705,ADMIN,Shanghai
-1052,2013-10-19,Others,1161,3,14,79.1353,0,10000561,10005173,ADMIN,Shanghai
-1053,2013-02-17,Others,31519,0,-99,49.0409,0,10000904,10006932,ADMIN,Shanghai
-1054,2012-03-30,FP-non GTC,314,0,11,93.6413,0,10000041,10002998,ANALYST,Beijing
-1055,2012-08-26,Others,175750,0,-99,46.6557,0,10000719,10004608,ANALYST,Beijing
-1056,2012-12-26,Auction,139973,0,15,6.9809,0,10000044,10003763,MODELER,Hongkong
-1057,2013-01-30,Others,159184,0,13,45.4599,0,10000131,10001765,ANALYST,Beijing
-1058,2012-04-27,Auction,48904,0,5,16.3472,0,10000157,10007395,MODELER,Hongkong
-1059,2012-11-10,ABIN,16145,0,13,0.4693,0,10000668,10004952,ANALYST,Beijing
-1060,2012-05-13,Auction,121153,0,5,24.642,0,10000314,10005354,MODELER,Hongkong
-1061,2012-02-01,FP-GTC,6762,0,13,5.9417,0,10000851,10003578,MODELER,Hongkong
-1062,2012-07-16,Auction,95672,0,-99,11.9934,0,10000828,10008797,MODELER,Hongkong
-1063,2012-09-21,Auction,106246,0,5,39.0057,0,10000826,10000054,ANALYST,Beijing
-1064,2012-10-20,FP-GTC,314,0,16,5.5339,0,10000011,10002088,ANALYST,Beijing
-1065,2013-04-29,FP-non GTC,100847,0,13,3.3872,0,10000840,10001720,ADMIN,Shanghai
-1066,2013-02-16,FP-non GTC,43398,0,13,3.4171,0,10000106,10006976,ANALYST,Beijing
-1067,2013-09-11,FP-GTC,63889,0,12,15.4662,0,10000004,10006486,MODELER,Hongkong
-1068,2012-12-18,Others,80287,0,15,71.309,0,10000722,10009936,MODELER,Hongkong
-1069,2013-09-14,FP-non GTC,20865,0,-99,98.0987,0,10000591,10004783,ADMIN,Shanghai
-1070,2013-01-27,FP-non GTC,57990,3,14,84.9483,0,10000304,10000164,ANALYST,Beijing
-1071,2013-02-07,Others,95672,0,14,9.7034,0,10000894,10000059,ANALYST,Beijing
-1072,2013-03-24,FP-non GTC,38238,0,-99,98.5328,0,10000286,10005535,MODELER,Hongkong
-1073,2012-01-05,Others,164,0,15,21.3336,0,10000151,10003519,ADMIN,Shanghai
-1074,2012-11-14,Auction,57990,0,12,91.1773,0,10000087,10002414,ADMIN,Shanghai
-1075,2013-03-24,Auction,13987,0,13,5.9953,0,10000551,10006991,ADMIN,Shanghai
-1076,2013-12-26,FP-GTC,32876,0,11,86.1904,0,10000012,10004509,MODELER,Hongkong
-1077,2013-06-23,FP-non GTC,1504,0,12,9.4689,0,10000720,10008088,ANALYST,Beijing
-1078,2012-01-11,FP-GTC,57013,0,13,96.6705,0,10000790,10001942,ADMIN,Shanghai
-1079,2012-08-30,FP-non GTC,62179,0,5,22.4244,0,10000403,10006119,MODELER,Hongkong
-1080,2012-09-01,ABIN,61323,0,12,30.1513,0,10000283,10003870,ANALYST,Beijing
-1081,2013-04-20,Auction,16145,3,12,77.9327,0,10000043,10000188,MODELER,Hongkong
-1082,2013-10-03,Others,963,0,14,82.3371,0,10000002,10006340,MODELER,Hongkong
-1083,2012-10-23,ABIN,45333,0,16,86.7017,0,10000180,10000817,MODELER,Hongkong
-1084,2013-10-09,ABIN,63889,0,-99,33.9942,0,10000485,10002344,ADMIN,Shanghai
-1085,2013-04-28,ABIN,73506,0,5,22.6563,0,10000070,10007027,ANALYST,Beijing
-1086,2013-07-07,ABIN,57990,0,11,11.7659,0,10000834,10000644,ADMIN,Shanghai
-1087,2012-01-04,FP-non GTC,165888,0,16,78.7873,0,10000743,10008093,ANALYST,Beijing
-1088,2012-05-05,ABIN,158666,15,16,55.1036,0,10000372,10009391,ANALYST,Beijing
-1089,2012-08-04,ABIN,43479,0,13,61.3469,0,10000425,10002587,MODELER,Hongkong
-1090,2013-02-18,Others,161567,15,5,43.5109,0,10000315,10009432,MODELER,Hongkong
-1091,2013-08-09,FP-GTC,161567,15,14,0.9756,0,10000420,10002373,MODELER,Hongkong
-1092,2012-04-30,Others,65,0,12,31.5741,0,10000858,10005147,ANALYST,Beijing
-1093,2013-10-16,FP-non GTC,16509,0,15,8.4323,0,10000376,10007194,MODELER,Hongkong
-1094,2013-11-06,FP-GTC,15115,0,12,91.7204,0,10000495,10003677,ADMIN,Shanghai
-1095,2012-04-13,ABIN,37831,0,16,86.915,0,10000947,10000539,MODELER,Hongkong
-1096,2013-04-21,FP-GTC,95672,0,-99,60.6048,0,10000052,10001595,ANALYST,Beijing
-1097,2013-12-04,FP-GTC,121153,0,13,9.3639,0,10000384,10009074,ADMIN,Shanghai
-1098,2012-08-03,Others,50508,0,13,74.2474,0,10000103,10004661,ANALYST,Beijing
-1099,2013-12-29,FP-GTC,1357,0,5,46.9382,0,10000987,10000797,ANALYST,Beijing
-1100,2012-11-01,Others,46575,0,13,54.452,0,10000247,10000707,ANALYST,Beijing
-1101,2012-06-30,FP-non GTC,26249,0,5,74.2755,0,10000641,10000127,MODELER,Hongkong
-1102,2012-06-20,Auction,26262,0,11,11.9173,0,10000418,10009843,MODELER,Hongkong
-1103,2012-07-24,Others,80287,0,14,49.7794,0,10000590,10006327,MODELER,Hongkong
-1104,2012-08-11,FP-GTC,26262,0,11,16.291,0,10000698,10002353,MODELER,Hongkong
-1105,2013-06-27,ABIN,63861,0,-99,5.4086,0,10000810,10007456,ANALYST,Beijing
-1106,2012-04-28,FP-GTC,15115,0,15,91.7724,0,10000414,10005702,MODELER,Hongkong
-1107,2013-04-19,ABIN,73506,0,13,14.8212,0,10000728,10000486,ADMIN,Shanghai
-1108,2012-01-25,Others,32876,0,13,23.4295,0,10000673,10003806,MODELER,Hongkong
-1109,2013-11-06,ABIN,139973,3,12,41.9681,0,10000902,10000143,MODELER,Hongkong
-1110,2013-03-09,Auction,61323,0,15,53.8594,0,10000323,10000572,ANALYST,Beijing
-1111,2012-03-01,ABIN,60340,0,11,78.1279,0,10000947,10001604,ANALYST,Beijing
-1112,2012-07-28,FP-non GTC,279,15,16,80.8961,0,10000691,10005343,MODELER,Hongkong
-1113,2013-02-04,Others,24541,0,11,40.4673,0,10000727,10004879,MODELER,Hongkong
-1114,2012-02-24,Auction,152801,0,13,9.8921,0,10000661,10003640,ADMIN,Shanghai
-1115,2013-08-13,ABIN,156614,0,15,61.6452,0,10000734,10000741,ANALYST,Beijing
-1116,2012-04-02,ABIN,63861,3,11,18.4644,0,10000446,10006619,ANALYST,Beijing
-1117,2013-09-28,Others,95173,0,-99,51.2096,0,10000049,10008305,ANALYST,Beijing
-1118,2013-08-13,FP-non GTC,64076,0,12,85.3952,0,10000581,10005375,ANALYST,Beijing
-1119,2012-09-09,ABIN,20485,0,15,0.5979,0,10000081,10008733,ADMIN,Shanghai
-1120,2012-04-26,Auction,145970,0,11,40.5874,0,10000401,10009751,ANALYST,Beijing
-1121,2013-05-10,Others,26262,0,11,83.8561,0,10000545,10005955,MODELER,Hongkong
-1122,2013-05-23,FP-GTC,80287,0,11,27.6123,0,10000529,10001324,ADMIN,Shanghai
-1123,2012-01-08,Others,175750,0,14,45.4268,0,10000408,10002363,ANALYST,Beijing
-1124,2012-07-20,ABIN,38238,0,16,44.1376,0,10000796,10009325,ADMIN,Shanghai
-1125,2012-09-15,Auction,65,0,-99,97.3146,0,10000792,10005350,MODELER,Hongkong
-1126,2012-10-17,ABIN,65,0,12,68.5363,0,10000569,10006467,MODELER,Hongkong
-1127,2013-08-03,FP-GTC,40059,3,15,87.3892,0,10000317,10004966,ADMIN,Shanghai
-1128,2013-08-28,FP-non GTC,80287,0,14,61.9829,0,10000015,10009708,ADMIN,Shanghai
-1129,2012-07-21,Auction,25147,0,11,24.7662,0,10000333,10003658,MODELER,Hongkong
-1130,2013-10-17,Auction,132939,0,11,11.6038,0,10000926,10004498,MODELER,Hongkong
-1131,2012-09-04,Auction,44079,0,15,75.1095,0,10000435,10008899,ANALYST,Beijing
-1132,2012-11-01,Auction,67698,0,5,8.8273,0,10000834,10008840,ADMIN,Shanghai
-1133,2013-12-02,Auction,24541,0,15,95.2195,0,10000259,10001544,ANALYST,Beijing
-1134,2012-02-06,Auction,533,0,12,66.4962,0,10000007,10001763,MODELER,Hongkong
-1135,2013-12-03,
<TRUNCATED>
[20/50] [abbrv] kylin git commit: KYLIN-2213 add test query
Posted by li...@apache.org.
KYLIN-2213 add test query
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3091f06a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3091f06a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3091f06a
Branch: refs/heads/master-cdh5.7
Commit: 3091f06a68e0b97b35639cbb65acbf532c114ca4
Parents: 9fc8f5e
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 2 13:21:54 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 2 13:21:54 2016 +0800
----------------------------------------------------------------------
.../test/resources/query/sql_like/query20.sql | 31 ++++++++++++++++++++
1 file changed, 31 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3091f06a/kylin-it/src/test/resources/query/sql_like/query20.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query20.sql b/kylin-it/src/test/resources/query/sql_like/query20.sql
new file mode 100644
index 0000000..6101cb0
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query20.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select lstg_format_name as lstg_format_name, count(*) as cnt
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where lstg_format_name not like '%BIN%'
+group by lstg_format_name
\ No newline at end of file
[22/50] [abbrv] kylin git commit: KYLIN-2192 More Robust Global
Dictionary
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
index a3948c3..0c163a6 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query00.sql
@@ -19,6 +19,6 @@
select lstg_format_name, cal_dt,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
+ count(distinct user_id) as user_count
from test_kylin_fact
group by lstg_format_name, cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
index e8579ef..62142a2 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query01.sql
@@ -19,7 +19,7 @@
select lstg_format_name,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
+ count(distinct user_id) as user_count
from test_kylin_fact
where lstg_format_name='FP-GTC'
group by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
index 48f49e9..88ce532 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query02.sql
@@ -19,7 +19,7 @@
select lstg_format_name,
sum(price) as GMV,
count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
+ count(distinct user_id) as user_count
from test_kylin_fact
where lstg_format_name='FP-GTC'
group by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
index dbc2fac..7f739e7 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query03.sql
@@ -17,7 +17,8 @@
--
select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
- , count(1) as TRANS_CNT, count(distinct seller_id) as seller_count
+ , count(1) as TRANS_CNT
+ , count(distinct user_id) as user_count
, count(distinct site_name) as site_count
from test_kylin_fact
inner JOIN edw.test_cal_dt as test_cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
index 69006ce..13de5a7 100644
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
+++ b/kylin-it/src/test/resources/query/sql_distinct_precisely/query04.sql
@@ -17,7 +17,8 @@
--
select test_cal_dt.week_beg_dt,sum(test_kylin_fact.price) as GMV
- , count(1) as TRANS_CNT, count(distinct seller_id) as seller_count
+ , count(1) as TRANS_CNT
+ , count(distinct user_id) as user_count
, count(distinct site_name) as site_count
from test_kylin_fact
inner JOIN edw.test_cal_dt as test_cal_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
deleted file mode 100644
index dea09f7..0000000
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query05.sql
+++ /dev/null
@@ -1,25 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select lstg_format_name,
- sum(price) as GMV,
- count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
- from test_kylin_fact
- group by lstg_format_name
- order by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query06.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query06.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query06.sql
deleted file mode 100644
index eb12620..0000000
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query06.sql
+++ /dev/null
@@ -1,26 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select lstg_format_name,
- sum(price) as GMV,
- count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
- from test_kylin_fact
- where lstg_format_name='FP-GTC'
- group by lstg_format_name
- order by lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/resources/query/sql_distinct_precisely/query07.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_distinct_precisely/query07.sql b/kylin-it/src/test/resources/query/sql_distinct_precisely/query07.sql
deleted file mode 100644
index 9bd2663..0000000
--- a/kylin-it/src/test/resources/query/sql_distinct_precisely/query07.sql
+++ /dev/null
@@ -1,24 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select lstg_format_name,
- sum(price) as GMV,
- count(1) as TRANS_CNT,
- count(distinct seller_id) as seller_count
- from test_kylin_fact
- group by lstg_format_name
[19/50] [abbrv] kylin git commit: KYLIN-2180 minor,
get owenr project in CubeDesc.init()
Posted by li...@apache.org.
KYLIN-2180 minor, get owenr project in CubeDesc.init()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9fc8f5e2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9fc8f5e2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9fc8f5e2
Branch: refs/heads/master-cdh5.7
Commit: 9fc8f5e207b5111fffaec3335efe0c8947353cfa
Parents: 1a5295b
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 2 11:39:34 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 2 11:39:34 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/model/CubeDesc.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9fc8f5e2/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 327ce57..853571c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -64,6 +64,9 @@ import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -519,11 +522,16 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
public void init(KylinConfig config) {
this.errors.clear();
- this.config = KylinConfigExt.createInstance(config, overrideKylinProps);
checkArgument(StringUtils.isNotBlank(name), "CubeDesc name is blank");
checkArgument(StringUtils.isNotBlank(modelName), "CubeDesc(%s) has blank modelName", name);
+ // note CubeDesc.name == CubeInstance.name
+ List<ProjectInstance> ownerPrj = ProjectManager.getInstance(config).findProjects(RealizationType.CUBE, name);
+ logger.info("CubeDesc '" + name + "' is owned by " + ownerPrj);
+
+ this.config = KylinConfigExt.createInstance(config, overrideKylinProps);
+
this.model = MetadataManager.getInstance(config).getDataModelDesc(modelName);
checkNotNull(this.model, "DateModelDesc(%s) not found", modelName);
@@ -823,6 +831,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return col;
}
+ @SuppressWarnings("deprecation")
private void initMeasureColumns() {
if (measures == null || measures.isEmpty()) {
return;
[42/50] [abbrv] kylin git commit: KYLIN-2246 redesign the way to
decide layer cubing reducer count
Posted by li...@apache.org.
KYLIN-2246 redesign the way to decide layer cubing reducer count
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/76999735
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/76999735
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/76999735
Branch: refs/heads/master-cdh5.7
Commit: 76999735f4157b6064eb099b59a7c79fbc9b6007
Parents: d2aaf27
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Dec 5 21:02:36 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 6 13:50:28 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/cuboid/CuboidScheduler.java | 31 +++++++-
.../kylin/engine/mr/common/CubeStatsReader.java | 26 ++++++-
.../apache/kylin/engine/mr/steps/CuboidJob.java | 52 +------------
.../engine/mr/steps/LayerReduerNumSizing.java | 82 ++++++++++++++++++++
.../kylin/engine/mr/steps/MergeCuboidJob.java | 2 +-
5 files changed, 138 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/76999735/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
index bd6a37a..733aded 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/CuboidScheduler.java
@@ -18,7 +18,7 @@
package org.apache.kylin.cube.cuboid;
-/**
+/**
*/
import java.util.Collections;
@@ -31,6 +31,7 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.kylin.cube.model.AggregationGroup;
import org.apache.kylin.cube.model.CubeDesc;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -39,6 +40,7 @@ public class CuboidScheduler {
private final CubeDesc cubeDesc;
private final long max;
private final Map<Long, List<Long>> cache;
+ private List<List<Long>> cuboidsByLayer;
public CuboidScheduler(CubeDesc cubeDesc) {
this.cubeDesc = cubeDesc;
@@ -232,4 +234,31 @@ public class CuboidScheduler {
getSubCuboidIds(cuboidId, result);
}
}
+
+ public List<List<Long>> getCuboidsByLayer() {
+ if (cuboidsByLayer != null) {
+ return cuboidsByLayer;
+ }
+
+ int totalNum = 0;
+ int layerNum = cubeDesc.getBuildLevel();
+ cuboidsByLayer = Lists.newArrayList();
+
+ cuboidsByLayer.add(Collections.singletonList(Cuboid.getBaseCuboidId(cubeDesc)));
+ totalNum++;
+
+ for (int i = 1; i <= layerNum; i++) {
+ List<Long> lastLayer = cuboidsByLayer.get(i - 1);
+ List<Long> newLayer = Lists.newArrayList();
+ for (Long parent : lastLayer) {
+ newLayer.addAll(getSpanningCuboid(parent));
+ }
+ cuboidsByLayer.add(newLayer);
+ totalNum += newLayer.size();
+ }
+
+ int size = getAllCuboidIds().size();
+ Preconditions.checkState(totalNum == size, "total Num: " + totalNum + " actual size: " + size);
+ return cuboidsByLayer;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/76999735/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index c917cfb..1cf5da6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -29,6 +29,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
@@ -75,9 +76,11 @@ public class CubeStatsReader {
final int mapperNumberOfFirstBuild; // becomes meaningless after merge
final double mapperOverlapRatioOfFirstBuild; // becomes meaningless after merge
final Map<Long, HyperLogLogPlusCounter> cuboidRowEstimatesHLL;
+ final CuboidScheduler cuboidScheduler;
public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) throws IOException {
ResourceStore store = ResourceStore.getStore(kylinConfig);
+ cuboidScheduler = new CuboidScheduler(cubeSegment.getCubeDesc());
String statsKey = cubeSegment.getStatisticsResourcePath();
File tmpSeqFile = writeTmpSeqFile(store.getResource(statsKey).inputStream);
Reader reader = null;
@@ -145,6 +148,10 @@ public class CubeStatsReader {
return getCuboidSizeMapFromRowCount(seg, getCuboidRowEstimatesHLL());
}
+ public double estimateCubeSize() {
+ return SumHelper.sumDouble(getCuboidSizeMap().values());
+ }
+
public int getMapperNumberOfFirstBuild() {
return mapperNumberOfFirstBuild;
}
@@ -248,12 +255,23 @@ public class CubeStatsReader {
out.println("----------------------------------------------------------------------------");
}
+ //return MB
+ public double estimateLayerSize(int level) {
+ List<List<Long>> layeredCuboids = cuboidScheduler.getCuboidsByLayer();
+ Map<Long, Double> cuboidSizeMap = getCuboidSizeMap();
+ double ret = 0;
+ for (Long cuboidId : layeredCuboids.get(level)) {
+ ret += cuboidSizeMap.get(cuboidId);
+ }
+
+ logger.info("Estimating size for layer {}, all cuboids are {}, total size is {}", level, StringUtils.join(layeredCuboids.get(level), ","), ret);
+ return ret;
+ }
+
private void printCuboidInfoTreeEntry(Map<Long, Long> cuboidRows, Map<Long, Double> cuboidSizes, PrintWriter out) {
- CubeDesc cubeDesc = seg.getCubeDesc();
- CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
- long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
+ long baseCuboid = Cuboid.getBaseCuboidId(seg.getCubeDesc());
int dimensionCount = Long.bitCount(baseCuboid);
- printCuboidInfoTree(-1L, baseCuboid, scheduler, cuboidRows, cuboidSizes, dimensionCount, 0, out);
+ printCuboidInfoTree(-1L, baseCuboid, cuboidScheduler, cuboidRows, cuboidSizes, dimensionCount, 0, out);
}
private void printKVInfo(PrintWriter writer) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/76999735/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
index ddd21b7..d3cb494 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer.Context;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
@@ -35,14 +34,11 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.CuboidCLI;
-import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.job.execution.ExecutableManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,6 +96,7 @@ public class CuboidJob extends AbstractHadoopJob {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cube = cubeMgr.getCube(cubeName);
+ CubeSegment segment = cube.getSegmentById(segmentID);
if (checkSkip(cubingJobId)) {
logger.info("Skip job " + getOptionValue(OPTION_JOB_NAME) + " for " + segmentID + "[" + segmentID + "]");
@@ -113,7 +110,7 @@ public class CuboidJob extends AbstractHadoopJob {
setJobClasspath(job, cube.getConfig());
// Mapper
- configureMapperInputFormat(cube.getSegmentById(segmentID));
+ configureMapperInputFormat(segment);
job.setMapperClass(this.mapperClass);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
@@ -134,7 +131,7 @@ public class CuboidJob extends AbstractHadoopJob {
// add metadata to distributed cache
attachKylinPropsAndMetadata(cube, job.getConfiguration());
- setReduceTaskNum(job, cube.getDescriptor(), nCuboidLevel);
+ LayerReduerNumSizing.setReduceTaskNum(job, segment, getTotalMapInputMB(), nCuboidLevel);
this.deletePath(job.getConfiguration(), output);
@@ -163,49 +160,6 @@ public class CuboidJob extends AbstractHadoopJob {
}
}
- protected void setReduceTaskNum(Job job, CubeDesc cubeDesc, int level) throws ClassNotFoundException, IOException, InterruptedException, JobException {
- KylinConfig kylinConfig = cubeDesc.getConfig();
-
- double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB();
- double reduceCountRatio = kylinConfig.getDefaultHadoopJobReducerCountRatio();
-
- // total map input MB
- double totalMapInputMB = this.getTotalMapInputMB();
-
- // output / input ratio
- int preLevelCuboids, thisLevelCuboids;
- if (level == 0) { // base cuboid
- preLevelCuboids = thisLevelCuboids = 1;
- } else { // n-cuboid
- int[] allLevelCount = CuboidCLI.calculateAllLevelCount(cubeDesc);
- preLevelCuboids = allLevelCount[level - 1];
- thisLevelCuboids = allLevelCount[level];
- }
-
- // total reduce input MB
- double totalReduceInputMB = totalMapInputMB * thisLevelCuboids / preLevelCuboids;
-
- // number of reduce tasks
- int numReduceTasks = (int) Math.round(totalReduceInputMB / perReduceInputMB * reduceCountRatio);
-
- // adjust reducer number for cube which has DISTINCT_COUNT measures for better performance
- if (cubeDesc.hasMemoryHungryMeasures()) {
- numReduceTasks = numReduceTasks * 4;
- }
-
- // at least 1 reducer by default
- numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks);
- // no more than 500 reducer by default
- numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
-
- job.setNumReduceTasks(numReduceTasks);
-
- logger.info("Having total map input MB " + Math.round(totalMapInputMB));
- logger.info("Having level " + level + ", pre-level cuboids " + preLevelCuboids + ", this level cuboids " + thisLevelCuboids);
- logger.info("Having per reduce MB " + perReduceInputMB + ", reduce count ratio " + reduceCountRatio);
- logger.info("Setting " + Context.NUM_REDUCES + "=" + numReduceTasks);
- }
-
/**
* @param mapperClass
* the mapperClass to set
http://git-wip-us.apache.org/repos/asf/kylin/blob/76999735/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReduerNumSizing.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReduerNumSizing.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReduerNumSizing.java
new file mode 100644
index 0000000..6bddcbd
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/LayerReduerNumSizing.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.engine.mr.steps;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.engine.mr.common.CubeStatsReader;
+import org.apache.kylin.job.exception.JobException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LayerReduerNumSizing {
+
+ private static final Logger logger = LoggerFactory.getLogger(LayerReduerNumSizing.class);
+
+ public static void setReduceTaskNum(Job job, CubeSegment cubeSegment, double totalMapInputMB, int level) throws ClassNotFoundException, IOException, InterruptedException, JobException {
+ CubeDesc cubeDesc = cubeSegment.getCubeDesc();
+ KylinConfig kylinConfig = cubeDesc.getConfig();
+
+ double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB();
+ double reduceCountRatio = kylinConfig.getDefaultHadoopJobReducerCountRatio();
+ logger.info("Having per reduce MB " + perReduceInputMB + ", reduce count ratio " + reduceCountRatio + ", level " + level);
+
+ CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, kylinConfig);
+
+ double parentLayerSizeEst, currentLayerSizeEst, adjustedCurrentLayerSizeEst;
+
+ if (level == -1) {
+ //merge case
+ adjustedCurrentLayerSizeEst = cubeStatsReader.estimateCubeSize();
+ logger.info("adjustedCurrentLayerSizeEst: {}", adjustedCurrentLayerSizeEst);
+ } else if (level == 0) {
+ //base cuboid case
+ adjustedCurrentLayerSizeEst = cubeStatsReader.estimateLayerSize(0);
+ logger.info("adjustedCurrentLayerSizeEst: {}", adjustedCurrentLayerSizeEst);
+ } else {
+ parentLayerSizeEst = cubeStatsReader.estimateLayerSize(level - 1);
+ currentLayerSizeEst = cubeStatsReader.estimateLayerSize(level);
+ adjustedCurrentLayerSizeEst = totalMapInputMB / parentLayerSizeEst * currentLayerSizeEst;
+ logger.info("totalMapInputMB: {}, parentLayerSizeEst: {}, currentLayerSizeEst: {}, adjustedCurrentLayerSizeEst: {}", totalMapInputMB, parentLayerSizeEst, currentLayerSizeEst, adjustedCurrentLayerSizeEst);
+ }
+
+ // number of reduce tasks
+ int numReduceTasks = (int) Math.round(adjustedCurrentLayerSizeEst / perReduceInputMB * reduceCountRatio);
+
+ // adjust reducer number for cube which has DISTINCT_COUNT measures for better performance
+ if (cubeDesc.hasMemoryHungryMeasures()) {
+ numReduceTasks = numReduceTasks * 4;
+ }
+
+ // at least 1 reducer by default
+ numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks);
+ // no more than 500 reducer by default
+ numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks);
+
+ job.setNumReduceTasks(numReduceTasks);
+
+ logger.info("Setting " + Reducer.Context.NUM_REDUCES + "=" + numReduceTasks);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/76999735/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
index 810da23..e805d25 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidJob.java
@@ -81,7 +81,7 @@ public class MergeCuboidJob extends CuboidJob {
// add metadata to distributed cache
attachKylinPropsAndMetadata(cube, job.getConfiguration());
- setReduceTaskNum(job, cube.getDescriptor(), 0);
+ LayerReduerNumSizing.setReduceTaskNum(job, cube.getSegmentById(segmentID), getTotalMapInputMB(), -1);
this.deletePath(job.getConfiguration(), output);
[28/50] [abbrv] kylin git commit: KYLIN-2212 'NOT' operator in filter
on derived column may get incorrect result
Posted by li...@apache.org.
KYLIN-2212 'NOT' operator in filter on derived column may get incorrect result
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fd1ed6f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fd1ed6f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fd1ed6f
Branch: refs/heads/master-cdh5.7
Commit: 0fd1ed6f14758bb38f1abf7a7ed9e24ed818f50f
Parents: 0eebd7d
Author: zhengdong <zh...@outlook.com>
Authored: Wed Nov 23 16:13:59 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 2 18:32:24 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/gridtable/GTUtil.java | 14 ++++++++++++
.../metadata/filter/CompareTupleFilter.java | 2 ++
.../metadata/filter/TupleFilterSerializer.java | 1 +
.../resources/query/sql_derived/query12.sql | 24 ++++++++++++++++++++
.../kylin/query/relnode/OLAPFilterRel.java | 3 +++
.../common/coprocessor/FilterDecorator.java | 14 ++++++++++++
6 files changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
old mode 100644
new mode 100755
index a4e574c..7496778
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -196,6 +196,20 @@ public class GTUtil {
result = newCompareFilter;
}
break;
+ case NOTIN:
+ Set notInValues = Sets.newHashSet();
+ for (Object value : constValues) {
+ code = translate(col, value, 0);
+ if (code != null)
+ notInValues.add(code);
+ }
+ if (notInValues.isEmpty()) {
+ result = ConstantTupleFilter.TRUE;
+ } else {
+ newCompareFilter.addChild(new ConstantTupleFilter(notInValues));
+ result = newCompareFilter;
+ }
+ break;
case NEQ:
code = translate(col, firstValue, 0);
if (code == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
old mode 100644
new mode 100755
index c7a3721..f2af735
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -59,6 +59,8 @@ public class CompareTupleFilter extends TupleFilter {
private CompareTupleFilter(CompareTupleFilter another) {
super(new ArrayList<TupleFilter>(another.children), another.operator);
this.column = another.column;
+ this.firstCondValue = another.getFirstValue();
+ this.function = another.getFunction();
this.conditionValues = new HashSet<Object>();
this.conditionValues.addAll(another.conditionValues);
this.dynamicVariables = new HashMap<String, Object>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
old mode 100644
new mode 100755
index 2df474e..63153ef
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -170,6 +170,7 @@ public class TupleFilterSerializer {
case GT:
case GTE:
case IN:
+ case NOTIN:
case ISNULL:
case ISNOTNULL:
filter = new CompareTupleFilter(op);
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/kylin-it/src/test/resources/query/sql_derived/query12.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_derived/query12.sql b/kylin-it/src/test/resources/query/sql_derived/query12.sql
new file mode 100755
index 0000000..959a59c
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_derived/query12.sql
@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select UPD_USER,count(1) as CNT
+from TEST_KYLIN_FACT as TEST_KYLIN_FACT
+JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+where UPD_USER not in ('USER_Y')
+group by UPD_USER
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
old mode 100644
new mode 100755
index 8b2035b..411142d
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -160,6 +160,9 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
if (inFilter != null) {
filter = inFilter;
}
+ } else if (op.getKind() == SqlKind.NOT) {
+ assert (filter.getChildren().size() == 1);
+ filter = filter.getChildren().get(0).reverse();
}
return filter;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fd1ed6f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
old mode 100644
new mode 100755
index 5ab4117..2b2e490
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
@@ -90,6 +90,20 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator {
result = newCompareFilter;
}
break;
+ case NOTIN:
+ Set<String> notInValues = Sets.newHashSet();
+ for (String value : constValues) {
+ v = translate(col, value, 0);
+ if (!isDictNull(v))
+ notInValues.add(v);
+ }
+ if (notInValues.isEmpty()) {
+ result = ConstantTupleFilter.TRUE;
+ } else {
+ newCompareFilter.addChild(new ConstantTupleFilter(notInValues));
+ result = newCompareFilter;
+ }
+ break;
case NEQ:
v = translate(col, firstValue, 0);
if (isDictNull(v)) {
[24/50] [abbrv] kylin git commit: KYLIN-2192 More Robust Global
Dictionary
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
index df64f3f..381e0b1 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
@@ -17,11 +17,12 @@
*/
package org.apache.kylin.dict;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.junit.After;
import org.junit.AfterClass;
-import org.junit.Before;
import org.junit.Test;
import java.io.*;
@@ -91,39 +92,26 @@ public class CachedTreeMapTest {
public static class CachedFileFilter implements FileFilter {
@Override
public boolean accept(File pathname) {
- return pathname.getName().startsWith("cached_");
+ return pathname.getName().startsWith(CachedTreeMap.CACHED_PREFIX);
}
}
- public static final String baseDir = "/tmp/kylin_cachedtreemap_test/";
- public static final String backupDir = "/tmp/kylin_cachedtreemap_test.bak/";
- public static final String tmpDir = "/tmp/kylin_cachedtreemap_test.tmp/";
-
- private static void cleanup() {
- File dir = new File(baseDir);
- if (dir.exists()) {
- for (File f : dir.listFiles()) {
- f.delete();
- }
- dir.delete();
- }
-
- dir = new File(tmpDir);
- if (dir.exists()) {
- for (File f : dir.listFiles()) {
- f.delete();
- }
- dir.delete();
+ public static class VersionFilter implements FileFilter {
+ @Override
+ public boolean accept(File pathname) {
+ return pathname.getName().startsWith(CachedTreeMap.VERSION_PREFIX);
}
+ }
- dir = new File(backupDir);
- if (dir.exists()) {
- for (File f : dir.listFiles()) {
- f.delete();
- }
- dir.delete();
- }
+ public static final String baseDir = "/tmp/kylin_cachedtreemap_test/";
+ public static final String workingDir = "/tmp/kylin_cachedtreemap_test/working";
+ private static void cleanup() {
+ Configuration conf = new Configuration();
+ Path basePath = new Path(baseDir);
+ try {
+ FileSystem.get(basePath.toUri(), conf).delete(basePath, true);
+ } catch (IOException e) {}
VALUE_WRITE_ERROR_TOGGLE = false;
}
@@ -139,154 +127,240 @@ public class CachedTreeMapTest {
@Test
public void testCachedTreeMap() throws IOException {
- CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
- .persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
+ CachedTreeMap map = createMutableMap();
map.put(Key.of(1), Value.of("a"));
map.put(Key.of(2), Value.of("b"));
map.put(Key.of(3), Value.of("c"));
map.put(Key.of(4), Value.of("d"));
map.put(Key.of(5), Value.of("e"));
- File dir = new File(tmpDir);
+ File dir = new File(workingDir);
assertEquals(3, dir.listFiles(new CachedFileFilter()).length);
- DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpDir+"/.index"));
- map.write(out);
- out.flush();
- out.close();
- map.commit(false);
+ flushAndCommit(map, true, true, false);
+ assertFalse(new File(workingDir).exists());
- dir = new File(baseDir);
+ dir = new File(map.getLatestVersion());
assertEquals(5, dir.listFiles(new CachedFileFilter()).length);
- DataInputStream in = new DataInputStream(new FileInputStream(baseDir+".index"));
- CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
- .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
- map2.readFields(in);
+ CachedTreeMap map2 = createImmutableMap();
assertEquals(5, map2.size());
assertEquals("b", ((Value)map2.get(Key.of(2))).valueStr);
try {
map2.put(Key.of(6), Value.of("f"));
fail("Should be error when put value into immutable map");
- } catch (AssertionError error) {
+ } catch (AssertionError error) {}
+ }
+
+ @Test
+ public void testMultiVersions() throws IOException, InterruptedException {
+ CachedTreeMap map = createMutableMap();
+ Thread.sleep(3000);
+ map.put(Key.of(1), Value.of("a"));
+ map.put(Key.of(2), Value.of("b"));
+ map.put(Key.of(3), Value.of("c"));
+ flushAndCommit(map, true, true, false);
+
+ CachedTreeMap map2 = createImmutableMap();
+ assertEquals("b", ((Value)map2.get(Key.of(2))).valueStr);
+
+ // re-open dict, append new data
+ map = createMutableMap();
+ map.put(Key.of(4), Value.of("d"));
+ flushAndCommit(map, true, true, true);
+
+ // new data is not visible for map2
+ assertNull(map2.get(Key.of(4)));
+
+ // append data, and be visible for new immutable map
+ map.put(Key.of(5), Value.of("e"));
+ flushAndCommit(map, true, true, true);
+
+ CachedTreeMap map3 = createImmutableMap();
+ assertEquals("d", ((Value)map3.get(Key.of(4))).valueStr);
+ assertEquals("e", ((Value)map3.get(Key.of(5))).valueStr);
+
+ // Check versions retention
+ File dir = new File(baseDir);
+ assertEquals(3, dir.listFiles(new VersionFilter()).length);
+ }
+
+ @Test
+ public void testKeepAppend() throws IOException {
+ CachedTreeMap map = createMutableMap();
+ map.put(Key.of(1), Value.of("a"));
+ map.put(Key.of(2), Value.of("b"));
+ map.put(Key.of(3), Value.of("c"));
+ map.put(Key.of(4), Value.of("d"));
+ map.put(Key.of(5), Value.of("e"));
+
+ // flush with keepAppend false, map can't be append
+ flushAndCommit(map, true, true, false);
+ // append into map has closed
+ try {
+ map.put(Key.of(6), Value.of("f"));
+ fail();
+ } catch (AssertionError e) {
+ assertEquals("Only support put method with immutable false and keepAppend true", e.getMessage());
}
- assertFalse(new File(tmpDir).exists());
- assertFalse(new File(backupDir).exists());
+ CachedTreeMap map2 = createImmutableMap();
+ assertEquals("a", ((Value)map2.get(Key.of(1))).valueStr);
+ assertEquals("d", ((Value)map2.get(Key.of(4))).valueStr);
+ assertEquals("e", ((Value)map2.get(Key.of(5))).valueStr);
+
+ map = createMutableMap();
+ map.put(Key.of(6), Value.of("f"));
+ map.put(Key.of(7), Value.of("g"));
+ map.put(Key.of(8), Value.of("h"));
+ // flush with keepAppend true
+ flushAndCommit(map, true, true, true);
+ map.put(Key.of(9), Value.of("i"));
+ // can still append data
+ flushAndCommit(map, true, true, false);
+
+ map2 = createImmutableMap();
+ assertEquals("a", ((Value)map2.get(Key.of(1))).valueStr);
+ assertEquals("d", ((Value)map2.get(Key.of(4))).valueStr);
+ assertEquals("f", ((Value)map2.get(Key.of(6))).valueStr);
+ assertEquals("i", ((Value)map2.get(Key.of(9))).valueStr);
+ }
+
+ @Test
+ public void testVersionRetention() throws IOException, InterruptedException {
+ File dir = new File(baseDir);
+ // TTL for 3s and keep 3 versions
+ CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
+ .immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class)
+ .maxVersions(3).versionTTL(1000 * 3).build();
+ map.put(Key.of(1), Value.of("a"));
+
+ // has version 0 when create map
+ assertEquals(1, dir.listFiles(new VersionFilter()).length);
+ Thread.sleep(2500);
+
+ // flush version 1
+ flushAndCommit(map, true, true, true);
+ assertEquals(2, dir.listFiles(new VersionFilter()).length);
+
+ // flush version 2
+ flushAndCommit(map, true, true, true);
+ assertEquals(3, dir.listFiles(new VersionFilter()).length);
+
+ // flush version 3
+ flushAndCommit(map, true, true, true);
+ // won't delete version since 3s TTL
+ assertEquals(4, dir.listFiles(new VersionFilter()).length);
+
+ // sleep to make version 0 expired
+ Thread.sleep(500);
+ // flush verion 4
+ flushAndCommit(map, true, true, false);
+ assertEquals(4, dir.listFiles(new VersionFilter()).length);
+
+ // TTL for 100ms and keep 2 versions
+ map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
+ .immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class)
+ .maxVersions(2).versionTTL(100).build();
+ flushAndCommit(map, true, true, false);
+ assertEquals(2, dir.listFiles(new VersionFilter()).length);
+ }
+
+ @Test
+ public void testWithOldFormat() throws IOException {
+ File dir = new File(baseDir);
+ CachedTreeMap map = createMutableMap();
+ map.put(Key.of(1), Value.of("a"));
+ map.put(Key.of(2), Value.of("b"));
+ map.put(Key.of(3), Value.of("c"));
+ map.put(Key.of(4), Value.of("d"));
+ map.put(Key.of(5), Value.of("e"));
+ flushAndCommit(map, true, true, true);
+
+ // move version dir to base dir, to simulate the older format
+ Path versionPath = new Path(map.getLatestVersion());
+ Path tmpVersionPath = new Path(versionPath.getParent().getParent(), versionPath.getName());
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(versionPath.toUri(), conf);
+ fs.rename(versionPath, tmpVersionPath);
+ fs.delete(new Path(baseDir), true);
+ fs.rename(tmpVersionPath, new Path(baseDir));
+ assertEquals(0, dir.listFiles(new VersionFilter()).length);
+ assertEquals(5, dir.listFiles(new CachedFileFilter()).length);
+
+ CachedTreeMap map2 = createImmutableMap();
+ assertEquals(5, map2.size());
+ assertEquals("a", ((Value)map2.get(Key.of(1))).valueStr);
+ assertEquals("e", ((Value)map2.get(Key.of(5))).valueStr);
+
+ assertEquals(1, dir.listFiles(new VersionFilter()).length);
+ assertEquals(0, dir.listFiles(new CachedFileFilter()).length);
}
@Test
public void testWriteFailed() throws IOException {
// normal case
- CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
- .persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
+ CachedTreeMap map = createMutableMap();
map.put(Key.of(1), Value.of("a"));
map.put(Key.of(2), Value.of("b"));
map.put(Key.of(3), Value.of("c"));
map.remove(Key.of(3));
map.put(Key.of(4), Value.of("d"));
- DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpDir+".index"));
- map.write(out);
- out.flush();
- out.close();
- map.commit(false);
+ flushAndCommit(map, true, true, false);
- DataInputStream in = new DataInputStream(new FileInputStream(baseDir+".index"));
- CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
- .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
- map2.readFields(in);
+ CachedTreeMap map2 = createImmutableMap();
assertEquals(3, map2.size());
assertEquals("a", ((Value)map2.get(Key.of(1))).valueStr);
// suppose write value failed and didn't commit data
- map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
- .persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
+ map = createMutableMap();
VALUE_WRITE_ERROR_TOGGLE = true;
map.put(Key.of(1), Value.of("aa"));
map.put(Key.of(2), Value.of("bb"));
VALUE_WRITE_ERROR_TOGGLE = false;
map.put(Key.of(3), Value.of("cc"));
map.put(Key.of(4), Value.of("dd"));
- out = new DataOutputStream(new FileOutputStream(tmpDir+".index"));
- map.write(out);
- out.flush();
- out.close();
// suppose write value failed and didn't commit data
- //map.commit(false);
+ flushAndCommit(map, true, false, false);
// read map data should not be modified
- in = new DataInputStream(new FileInputStream(baseDir+".index"));
- map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
- .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
- map2.readFields(in);
+ map2 = createImmutableMap();
assertEquals(3, map2.size());
assertEquals("a", ((Value)map2.get(Key.of(1))).valueStr);
- assertTrue(new File(tmpDir).exists());
- assertFalse(new File(backupDir).exists());
+ assertTrue(new File(workingDir).exists());
}
- @Test
- public void testCommit() throws IOException {
+ private CachedTreeMap createImmutableMap() throws IOException {
CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
- .persistent(true).immutable(false).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
- map.put(Key.of(1), Value.of("a"));
- map.put(Key.of(2), Value.of("b"));
- map.put(Key.of(3), Value.of("c"));
- map.put(Key.of(4), Value.of("d"));
-
- DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpDir+".index"));
- map.write(out);
- out.flush();
- out.close();
- map.commit(true);
-
- assertTrue(new File(tmpDir).exists());
- assertFalse(new File(backupDir).exists());
+ .immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
+ try (DataInputStream in = map.openIndexInput()) {
+ map.readFields(in);
+ }
+ return map;
+ }
- DataInputStream in = new DataInputStream(new FileInputStream(baseDir+".index"));
- CachedTreeMap map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
- .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
- map2.readFields(in);
- assertEquals(4, map2.size());
- assertEquals("a", ((Value)map2.get(Key.of(1))).valueStr);
+ private CachedTreeMap createMutableMap() throws IOException {
+ CachedTreeMap map = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
+ .immutable(false).maxSize(2).maxVersions(3).versionTTL(1000 * 3).keyClazz(Key.class).valueClazz(Value.class).build();
+ try (DataInputStream in = map.openIndexInput()) {
+ map.readFields(in);
+ } catch (IOException e) {}
+ return map;
+ }
- // continue modify map, but not commit
- map.put(Key.of(1), Value.of("aa"));
- map.put(Key.of(2), Value.of("bb"));
- map.put(Key.of(3), Value.of("cc"));
- map.put(Key.of(5), Value.of("e"));
- map.put(Key.of(6), Value.of("f"));
- out = new DataOutputStream(new FileOutputStream(tmpDir+".index"));
- map.write(out);
- out.flush();
- out.close();
-
- assertTrue(new File(tmpDir).exists());
- assertEquals(6, new File(tmpDir).listFiles(new CachedFileFilter()).length);
- assertEquals(4, new File(baseDir).listFiles(new CachedFileFilter()).length);
-
- in = new DataInputStream(new FileInputStream(baseDir+".index"));
- map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
- .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
- map2.readFields(in);
- assertEquals(4, map2.size());
- assertEquals("a", ((Value)map2.get(Key.of(1))).valueStr);
+ private void flushAndCommit(CachedTreeMap map, boolean doFlush, boolean doCommit, boolean keepAppend) throws IOException {
+ if (doFlush) {
+ try (DataOutputStream out = map.openIndexOutput()) {
+ map.write(out);
+ }
+ }
- // commit data
- map.commit(false);
- assertFalse(new File(tmpDir).exists());
- assertEquals(6, new File(baseDir).listFiles(new CachedFileFilter()).length);
-
- in = new DataInputStream(new FileInputStream(baseDir+".index"));
- map2 = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir)
- .persistent(true).immutable(true).maxSize(2).keyClazz(Key.class).valueClazz(Value.class).build();
- map2.readFields(in);
- assertEquals(6, map2.size());
- assertEquals("aa", ((Value)map2.get(Key.of(1))).valueStr);
- assertEquals("f", ((Value)map2.get(Key.of(6))).valueStr);
+ if (doCommit) {
+ map.commit(keepAppend);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
index f0e21cf..f7796c0 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/bitmap/BitmapCounterTest.java
@@ -44,10 +44,12 @@ public class BitmapCounterTest {
counter2.add(12273456);
counter2.add("4258");
counter2.add(123);
- assertEquals(4, counter2.getCount());
+ counter2.add(-2147483648);
+ counter2.add(-2);
+ assertEquals(6, counter2.getCount());
counter.merge(counter2);
- assertEquals(6, counter.getCount());
+ assertEquals(8, counter.getCount());
System.out.print("counter size: " + counter.getMemBytes() + ", counter2 size: " + counter2.getMemBytes());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index 0470dc6..9c26be9 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -116,12 +116,12 @@
},
"dependent_measure_ref" : null
}, {
- "name" : "SITE_NAME_BITMAP",
+ "name" : "USER_COUNT_BITMAP",
"function" : {
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "SITE_NAME",
+ "value" : "USER_ID",
"next_parameter" : null
},
"returntype" : "bitmap"
@@ -209,12 +209,10 @@
},
"dependent_measure_ref" : null
} ],
- "dictionaries" : [
- {
- "column" : "SITE_NAME",
- "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
- }
- ],
+ "dictionaries": [ {
+ "column": "USER_ID",
+ "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
+ } ],
"rowkey" : {
"rowkey_columns" : [ {
"column" : "cal_dt",
@@ -257,7 +255,7 @@
"name" : "f2",
"columns" : [ {
"qualifier" : "m",
- "measure_refs" : [ "seller_cnt_bitmap", "site_name_bitmap", "seller_format_cnt"]
+ "measure_refs" : [ "seller_cnt_bitmap", "user_count_bitmap", "seller_format_cnt"]
} ]
}, {
"name" : "f3",
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv b/examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv
index 2544d8a..736f9e8 100644
--- a/examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv
+++ b/examples/test_case_data/localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv
@@ -1,402 +1,402 @@
-10000000157,2013-03-31,Auction,48027,0,12,184.21,1,10000001
-10000000158,2013-11-12,Others,164261,0,5,172.03,1,10000002
-10000000161,2013-04-06,Auction,82494,15,14,66.6,1,10000003
-10000000137,2013-05-17,Auction,66767,15,12,92.98,1,10000004
-10000000147,2013-05-20,FP-GTC,152801,0,5,132.33,1,10000005
-10000000155,2013-06-16,FP-GTC,43398,0,13,7.12,1,10000006
-10000000170,2013-06-14,Auction,95173,0,14,204.28,1,10000007
-10000000173,2013-03-22,Auction,158666,15,13,35.72,1,10000008
-10000000178,2013-03-10,Auction,12688,0,12,4.13,1,10000009
-10000000163,2013-11-01,FP-GTC,103324,15,5,27.48,1,10000010
-10000000166,2013-06-16,FP-GTC,108782,15,14,9.26,1,10000011
-10000000167,2013-09-12,Auction,80287,0,12,3.18,1,10000012
-10000000086,2013-09-28,Others,140746,100,13,3.18,1,10000013
-10000000110,2013-06-15,ABIN,87118,0,14,377.94,1,10000014
-10000000113,2013-03-14,Auction,25147,0,12,146.33,1,10000015
-10000000069,2013-09-01,FP-GTC,170302,15,5,51.23,1,10000016
-10000000079,2013-05-29,FP-non GTC,53064,0,13,72.65,1,10000017
-10000000080,2013-05-31,Auction,132939,0,13,66.6,1,10000018
-10000000130,2013-03-18,Auction,113593,15,12,9.26,1,10000019
-10000000268,2013-07-19,Auction,34273,100,14,583.44,1,10000020
-10000000132,2013-06-23,FP-GTC,106340,15,14,638.72,1,10000021
-10000000115,2013-05-20,FP-GTC,150265,15,14,4.54,1,10000022
-10000000117,2013-05-17,FP-GTC,24760,0,12,319.79,1,10000023
-10000000118,2013-03-11,Auction,37831,0,12,20.35,1,10000024
-10000000223,2013-01-30,FP-non GTC,1120,3,5,223.63,1,10000025
-10000000224,2013-01-26,FP-GTC,43972,100,13,204.28,1,10000026
-10000000243,2013-03-22,Auction,166013,15,14,5.48,1,10000027
-10000000217,2013-07-23,Auction,15568,15,14,27.48,1,10000028
-10000000218,2013-07-27,FP-GTC,103178,15,5,21.72,1,10000029
-10000000221,2013-10-29,ABIN,2023,0,12,3.18,1,10000030
-10000000256,2013-10-08,FP-GTC,94847,0,11,491.32,1,10000031
-10000000257,2013-04-26,Auction,15868,0,14,448.8,1,10000032
-10000000263,2013-01-01,Auction,32876,0,13,415.73,1,10000033
-10000000245,2013-01-15,Auction,62179,0,13,377.94,1,10000034
-10000000248,2013-05-27,FP-GTC,33038,15,14,146.33,1,10000035
-10000000254,2013-11-11,FP-GTC,156614,0,5,7.12,1,10000036
-10000000192,2013-03-08,Auction,106246,0,13,42.99,1,10000037
-10000000196,2013-03-25,Auction,20865,0,13,12.85,1,10000038
-10000000203,2013-08-20,FP-GTC,15115,0,13,55.89,1,10000039
-10000000179,2013-05-17,FP-GTC,3838,0,14,73.26,1,10000040
-10000000183,2013-06-05,Auction,759,0,11,112.56,1,10000041
-10000000185,2013-10-08,FP-non GTC,61323,0,11,3.49,1,10000042
-10000000211,2013-08-14,Auction,121153,0,13,184.21,1,10000043
-10000000213,2013-08-14,Auction,88750,0,13,157.14,1,10000044
-10000000214,2013-05-17,FP-GTC,161567,15,14,72.65,1,10000045
-10000000204,2013-08-09,FP-GTC,113802,15,14,51.23,1,10000046
-10000000208,2013-06-30,FP-non GTC,15808,15,14,15.85,1,10000047
-10000000209,2013-06-03,FP-GTC,174053,3,13,7.12,1,10000048
-10000000042,2013-12-31,Auction,2635,0,14,12.04,1,10000049
-10000000044,2013-12-25,Auction,1161,3,13,73.26,1,10000050
-10000000022,2013-03-28,FP-non GTC,64076,0,5,184.21,1,10000051
-10000000023,2013-01-30,FP-GTC,33977,15,13,172.03,1,10000052
-10000000047,2013-12-05,FP-GTC,31673,0,14,122.78,1,10000053
-10000000056,2013-10-08,Auction,174106,3,14,92.98,1,10000054
-10000000062,2013-12-27,Auction,26249,0,13,12.19,1,10000055
-10000000053,2013-12-16,FP-GTC,159184,0,5,15.65,1,10000056
-10000000055,2013-10-17,FP-GTC,10058,3,11,101.79,1,10000057
-10000000020,2013-11-17,ABIN,48904,0,12,7.12,1,10000058
-10000000007,2013-09-18,FP-non GTC,145970,0,14,12.85,1,10000059
-10000000008,2013-06-30,FP-GTC,963,0,13,12.19,1,10000060
-10000000002,2013-10-12,FP-GTC,118687,3,13,92.98,1,10000061
-10000000003,2013-08-20,FP-GTC,20886,0,14,42.99,1,10000062
-10000000010,2013-08-29,Auction,148324,15,13,1.88,1,10000063
-10000000016,2013-07-17,Auction,139255,15,14,21.14,1,10000064
-10000000017,2013-07-23,FP-GTC,20213,0,5,21.14,1,10000065
-10000000012,2013-01-06,Auction,32996,15,13,132.33,1,10000066
-10000000013,2013-08-14,FP-GTC,99985,0,14,120.87,1,10000067
-10000000067,2013-08-10,Auction,67703,3,14,120.87,1,10000068
-10000000085,2013-09-28,FP-non GTC,65,0,11,9.26,1,10000069
-10000000141,2013-08-21,FP-non GTC,130,0,14,16.26,1,10000070
-10000000078,2013-03-11,FP-GTC,164,0,14,157.14,1,10000071
-10000000109,2013-05-06,FP-GTC,216,0,11,1.88,1,10000072
-10000000096,2013-05-17,FP-non GTC,223,0,5,12.04,1,10000073
-10000000095,2013-01-10,FP-non GTC,223,0,14,189.23,1,10000074
-10000000098,2013-05-05,FP-non GTC,223,0,5,73.26,1,10000075
-10000000097,2013-02-03,FP-non GTC,223,0,5,4.13,1,10000076
-10000000099,2013-11-26,FP-non GTC,223,0,5,290.72,1,10000077
-10000000100,2013-08-30,FP-non GTC,223,0,5,265.56,1,10000078
-10000000126,2013-04-26,FP-GTC,279,15,5,5.91,1,10000079
-10000000252,2013-06-30,Auction,314,0,5,319.79,1,10000080
-10000000052,2013-06-30,Auction,314,211,5,246,1,10000081
-10000000253,2013-12-16,Auction,314,211,5,20.35,1,10000082
-10000000051,2013-12-15,Auction,314,0,5,36.7,1,10000083
-10000000190,2013-08-17,Auction,533,0,13,101.79,1,10000084
-10000000251,2013-12-15,ABIN,1349,0,5,47.71,1,10000085
-10000000050,2013-02-04,ABIN,1349,0,5,3.49,1,10000086
-10000000049,2013-01-11,ABIN,1349,0,13,46.44,1,10000087
-10000000250,2013-02-04,ABIN,1349,0,13,4.54,1,10000088
-10000000131,2013-05-17,ABIN,1357,0,14,3.18,1,10000089
-10000000172,2013-11-12,FP-GTC,1504,0,14,86.58,1,10000090
-10000000142,2013-08-21,FP-GTC,4943,0,13,12.85,1,10000091
-10000000195,2013-05-09,ABIN,6762,0,13,16.26,1,10000092
-10000000070,2013-09-19,Auction,9426,3,13,21.14,1,10000093
-10000000165,2013-02-06,FP-non GTC,10866,0,14,20.6,1,10000094
-10000000187,2013-02-02,Auction,11554,0,13,246,1,10000095
-10000000189,2013-08-23,FP-GTC,11848,0,14,109,1,10000096
-10000000139,2013-08-03,Auction,13836,0,13,39.41,1,10000097
-10000000140,2013-05-17,Auction,13836,0,14,16.26,1,10000098
-10000000102,2013-06-06,FP-GTC,13987,0,13,112.56,1,10000099
-10000000076,2013-07-02,Auction,15687,0,14,184.21,1,10000100
-10000000082,2013-10-25,Auction,15687,0,11,27.48,1,10000001
-10000000129,2013-04-20,FP-non GTC,16145,3,12,26.45,1,10000002
-10000000128,2013-03-12,FP-non GTC,16145,0,13,415.73,1,10000003
-10000000222,2013-03-28,ABIN,16509,0,5,56.36,1,10000004
-10000000021,2013-10-29,ABIN,16509,0,5,2.44,1,10000005
-10000000134,2013-05-22,FP-GTC,20485,0,14,269.76,1,10000006
-10000000135,2013-01-25,FP-GTC,20485,101,12,109,1,10000007
-10000000136,2013-06-12,FP-GTC,20485,101,12,101.79,1,10000008
-10000000241,2013-12-26,Auction,23446,23,14,246,1,10000009
-10000000041,2013-12-26,Auction,23446,23,14,189.23,1,10000010
-10000000242,2013-12-31,Auction,23446,23,14,15.65,1,10000011
-10000000040,2013-10-04,Auction,23446,23,14,28.23,1,10000012
-10000000194,2013-03-16,FP-GTC,24541,0,5,16.26,1,10000013
-10000000101,2013-05-21,FP-GTC,26262,0,5,122.78,1,10000014
-10000000077,2013-01-28,FP-GTC,30059,3,14,172.03,1,10000015
-10000000057,2013-04-26,Auction,31387,3,14,42.99,1,10000016
-10000000258,2013-10-06,Auction,31387,3,14,207.5,1,10000017
-10000000261,2013-11-06,FP-GTC,31519,0,14,5.91,1,10000018
-10000000058,2013-10-06,FP-GTC,31519,3,14,39.41,1,10000019
-10000000059,2013-12-28,FP-GTC,31519,0,14,16.26,1,10000020
-10000000060,2013-11-06,FP-GTC,31519,0,14,16.26,1,10000021
-10000000260,2013-11-06,FP-GTC,31519,0,14,78.48,1,10000022
-10000000259,2013-12-28,FP-GTC,31519,3,14,190.22,1,10000023
-10000000156,2013-06-11,FP-GTC,35570,100,12,2.44,1,10000024
-10000000119,2013-01-10,Auction,36250,0,5,7.12,1,10000025
-10000000186,2013-09-17,FP-non GTC,38238,0,14,36.7,1,10000026
-10000000038,2013-08-14,FP-GTC,40059,3,14,35.72,1,10000027
-10000000239,2013-08-09,FP-GTC,40059,3,14,3.49,1,10000028
-10000000034,2013-12-02,FP-GTC,41940,0,13,223.63,1,10000029
-10000000235,2013-02-01,FP-GTC,41940,0,13,265.56,1,10000030
-10000000127,2013-07-28,FP-non GTC,43479,0,13,62.02,1,10000031
-10000000103,2013-06-16,FP-GTC,44079,0,12,46.44,1,10000032
-10000000201,2013-08-23,Auction,45238,101,14,132.33,1,10000033
-10000000122,2013-06-15,Auction,45333,0,13,448.8,1,10000034
-10000000123,2013-06-15,FP-non GTC,45333,0,14,207.5,1,10000035
-10000000124,2013-06-01,FP-non GTC,45333,0,14,190.22,1,10000036
-10000000072,2013-08-10,FP-GTC,46575,0,14,16.71,1,10000037
-10000000043,2013-03-22,FP-non GTC,50508,0,13,4.13,1,10000038
-10000000244,2013-12-25,FP-non GTC,50508,0,13,1.88,1,10000039
-10000000121,2013-07-22,FP-GTC,50677,0,13,491.32,1,10000040
-10000000120,2013-04-13,FP-GTC,50677,0,5,2.44,1,10000041
-10000000168,2013-04-16,Auction,51582,0,14,56.36,1,10000042
-10000000073,2013-08-21,FP-GTC,57013,0,13,15.85,1,10000043
-10000000075,2013-04-22,FP-non GTC,57013,0,14,2.44,1,10000044
-10000000074,2013-08-29,FP-GTC,57013,0,14,7.12,1,10000045
-10000000093,2013-05-16,Auction,57784,0,14,35.72,1,10000046
-10000000265,2013-08-23,Auction,57990,3,11,9.26,1,10000047
-10000000266,2013-07-10,Auction,57990,3,14,3.18,1,10000048
-10000000267,2013-08-10,Auction,57990,3,14,638.72,1,10000049
-10000000065,2013-08-23,Auction,57990,3,14,141.7,1,10000050
-10000000143,2013-04-18,ABIN,57990,0,13,12.19,1,10000051
-10000000066,2013-07-10,Auction,57990,3,14,132.33,1,10000052
-10000000144,2013-06-16,ABIN,57990,3,5,5.48,1,10000053
-10000000064,2013-07-15,Auction,57990,3,11,1.88,1,10000054
-10000000061,2013-11-06,FP-GTC,60340,0,14,12.85,1,10000055
-10000000262,2013-12-27,FP-GTC,60340,0,14,62.02,1,10000056
-10000000019,2013-07-29,FP-GTC,60606,3,12,15.85,1,10000057
-10000000220,2013-11-17,FP-GTC,60606,3,12,9.26,1,10000058
-10000000018,2013-07-27,FP-GTC,60606,3,12,16.71,1,10000059
-10000000219,2013-07-29,FP-GTC,60606,3,12,20.6,1,10000060
-10000000145,2013-01-09,Auction,63861,3,5,1.88,1,10000061
-10000000200,2013-06-11,ABIN,63861,0,5,141.7,1,10000062
-10000000199,2013-01-10,ABIN,63861,0,5,1.88,1,10000063
-10000000237,2013-09-16,Others,63861,0,11,112.56,1,10000064
-10000000036,2013-01-14,Others,63861,0,11,94.45,1,10000065
-10000000125,2013-05-17,Auction,63861,0,14,78.48,1,10000066
-10000000198,2013-06-05,ABIN,63861,0,13,5.48,1,10000067
-10000000094,2013-05-24,Auction,63864,3,14,28.23,1,10000068
-10000000104,2013-05-15,Others,63889,0,13,3.49,1,10000069
-10000000107,2013-03-25,FP-GTC,67698,2,11,15.65,1,10000070
-10000000108,2013-03-09,FP-GTC,67698,0,11,5.48,1,10000071
-10000000106,2013-12-05,FP-GTC,67698,0,11,246,1,10000072
-10000000182,2013-04-18,FP-non GTC,73506,0,13,122.78,1,10000073
-10000000169,2013-11-01,FP-GTC,75665,0,14,223.63,1,10000074
-10000000146,2013-05-03,ABIN,75708,3,5,141.7,1,10000075
-10000000151,2013-04-21,FP-non GTC,80053,0,11,21.14,1,10000076
-10000000149,2013-03-12,FP-non GTC,80053,0,11,55.89,1,10000077
-10000000150,2013-05-19,FP-non GTC,80053,0,11,51.23,1,10000078
-10000000083,2013-11-23,Auction,80135,0,14,21.72,1,10000079
-10000000089,2013-10-19,Auction,95672,3,14,204.28,1,10000080
-10000000152,2013-05-18,Others,95672,0,11,21.14,1,10000081
-10000000035,2013-02-01,Others,100847,0,5,204.28,1,10000082
-10000000236,2013-01-14,Others,100847,0,5,122.78,1,10000083
-10000000090,2013-08-05,ABIN,139973,3,14,94.45,1,10000084
-10000000091,2013-05-19,ABIN,139973,0,11,86.58,1,10000085
-10000000033,2013-12-01,Auction,150047,3,14,56.36,1,10000086
-10000000234,2013-12-02,Auction,150047,3,14,290.72,1,10000087
-10000000249,2013-01-11,FP-GTC,155226,0,13,60.37,1,10000088
-10000000048,2013-05-27,FP-GTC,155226,0,13,112.56,1,10000089
-10000000181,2013-09-01,FP-GTC,156356,0,13,265.56,1,10000090
-10000000092,2013-04-11,FP-GTC,158798,0,11,35.72,1,10000091
-10000000191,2013-05-05,FP-non GTC,165888,0,13,92.98,1,10000092
-10000000229,2013-11-21,Auction,170083,3,11,28.23,1,10000093
-10000000028,2013-10-07,Auction,170083,3,11,27.48,1,10000094
-10000000031,2013-07-12,Auction,175750,3,14,9.26,1,10000095
-10000000032,2013-06-07,Auction,175750,3,14,3.18,1,10000096
-10000000177,2013-05-22,FP-GTC,175750,0,14,12.04,1,10000097
-10000000030,2013-11-28,Auction,175750,3,13,20.6,1,10000098
-10000000231,2013-07-12,Auction,175750,3,13,12.04,1,10000099
-10000000232,2013-06-07,Auction,175750,3,14,4.13,1,10000100
-10000000233,2013-12-01,Auction,175750,3,14,73.26,1,10000201
-10000000002,2012-10-12,Auction,48027,0,12,184.21,1,10000001
-10000000003,2012-08-20,Others,164261,0,5,172.03,1,10000002
-10000000007,2012-09-18,Auction,82494,15,14,66.6,1,10000003
-10000000008,2012-06-30,Auction,66767,15,12,92.98,1,10000004
-10000000010,2012-08-29,FP-GTC,152801,0,5,132.33,1,10000005
-10000000012,2012-01-06,FP-GTC,43398,0,13,7.12,1,10000006
-10000000013,2012-08-14,Auction,95173,0,14,204.28,1,10000007
-10000000016,2012-07-17,Auction,158666,15,13,35.72,1,10000008
-10000000017,2012-07-23,Auction,12688,0,12,4.13,1,10000009
-10000000018,2012-07-27,FP-GTC,103324,15,5,27.48,1,10000010
-10000000019,2012-07-29,FP-GTC,108782,15,14,9.26,1,10000011
-10000000020,2012-11-17,Auction,80287,0,12,3.18,1,10000012
-10000000021,2012-10-29,Others,140746,100,13,3.18,1,10000013
-10000000022,2012-03-28,ABIN,87118,0,14,377.94,1,10000014
-10000000023,2012-01-30,Auction,25147,0,12,146.33,1,10000015
-10000000028,2012-10-07,FP-GTC,170302,15,5,51.23,1,10000016
-10000000030,2012-11-28,FP-non GTC,53064,0,13,72.65,1,10000017
-10000000031,2012-07-12,Auction,132939,0,13,66.6,1,10000018
-10000000032,2012-06-07,Auction,113593,15,12,9.26,1,10000019
-10000000033,2012-12-01,Auction,34273,100,14,583.44,1,10000020
-10000000034,2012-12-02,FP-GTC,106340,15,14,638.72,1,10000021
-10000000035,2012-02-01,FP-GTC,150265,15,14,4.54,1,10000022
-10000000036,2012-01-14,FP-GTC,24760,0,12,319.79,1,10000023
-10000000038,2012-08-14,Auction,37831,0,12,20.35,1,10000024
-10000000040,2012-10-04,FP-non GTC,1120,3,5,223.63,1,10000025
-10000000041,2012-12-26,FP-GTC,43972,100,13,204.28,1,10000026
-10000000042,2012-12-31,Auction,166013,15,14,5.48,1,10000027
-10000000043,2012-03-22,Auction,15568,15,14,27.48,1,10000028
-10000000044,2012-12-25,FP-GTC,103178,15,5,21.72,1,10000029
-10000000047,2012-12-05,ABIN,2023,0,12,3.18,1,10000030
-10000000048,2012-05-27,FP-GTC,94847,0,11,491.32,1,10000031
-10000000049,2012-01-11,Auction,15868,0,14,448.8,1,10000032
-10000000050,2012-02-04,Auction,32876,0,13,415.73,1,10000033
-10000000051,2012-12-15,Auction,62179,0,13,377.94,1,10000034
-10000000052,2012-06-30,FP-GTC,33038,15,14,146.33,1,10000035
-10000000053,2012-12-16,FP-GTC,156614,0,5,7.12,1,10000036
-10000000055,2012-10-17,Auction,106246,0,13,42.99,1,10000037
-10000000056,2012-10-08,Auction,20865,0,13,12.85,1,10000038
-10000000057,2012-04-26,FP-GTC,15115,0,13,55.89,1,10000039
-10000000058,2012-10-06,FP-GTC,3838,0,14,73.26,1,10000040
-10000000059,2012-12-28,Auction,759,0,11,112.56,1,10000041
-10000000060,2012-11-06,FP-non GTC,61323,0,11,3.49,1,10000042
-10000000061,2012-11-06,Auction,121153,0,13,184.21,1,10000043
-10000000062,2012-12-27,Auction,88750,0,13,157.14,1,10000044
-10000000064,2012-07-15,FP-GTC,161567,15,14,72.65,1,10000045
-10000000065,2012-08-23,FP-GTC,113802,15,14,51.23,1,10000046
-10000000066,2012-07-10,FP-non GTC,15808,15,14,15.85,1,10000047
-10000000067,2012-08-10,FP-GTC,174053,3,13,7.12,1,10000048
-10000000069,2012-09-01,Auction,2635,0,14,12.04,1,10000049
-10000000070,2012-09-19,Auction,1161,3,13,73.26,1,10000050
-10000000072,2012-08-10,FP-non GTC,64076,0,5,184.21,1,10000051
-10000000073,2012-08-21,FP-GTC,33977,15,13,172.03,1,10000052
-10000000074,2012-08-29,FP-GTC,31673,0,14,122.78,1,10000053
-10000000075,2012-04-22,Auction,174106,3,14,92.98,1,10000054
-10000000076,2012-07-02,Auction,26249,0,13,12.19,1,10000055
-10000000077,2012-01-28,FP-GTC,159184,0,5,15.65,1,10000056
-10000000078,2012-03-11,FP-GTC,10058,3,11,101.79,1,10000057
-10000000079,2012-05-29,ABIN,48904,0,12,7.12,1,10000058
-10000000080,2012-05-31,FP-non GTC,145970,0,14,12.85,1,10000059
-10000000082,2012-10-25,FP-GTC,963,0,13,12.19,1,10000060
-10000000083,2012-11-23,FP-GTC,118687,3,13,92.98,1,10000061
-10000000085,2012-09-28,FP-GTC,20886,0,14,42.99,1,10000062
-10000000086,2012-09-28,Auction,148324,15,13,1.88,1,10000063
-10000000089,2012-10-19,Auction,139255,15,14,21.14,1,10000064
-10000000090,2012-08-05,FP-GTC,20213,0,5,21.14,1,10000065
-10000000091,2012-05-19,Auction,32996,15,13,132.33,1,10000066
-10000000092,2012-04-11,FP-GTC,99985,0,14,120.87,1,10000067
-10000000093,2012-05-16,Auction,67703,3,14,120.87,1,10000068
-10000000094,2012-05-24,FP-non GTC,65,0,11,9.26,1,10000069
-10000000095,2012-01-10,FP-non GTC,130,0,14,16.26,1,10000070
-10000000096,2012-05-17,FP-GTC,164,0,14,157.14,1,10000071
-10000000097,2012-02-03,FP-GTC,216,0,11,1.88,1,10000072
-10000000098,2012-05-05,FP-non GTC,223,0,5,12.04,1,10000073
-10000000099,2012-11-26,FP-non GTC,223,0,14,189.23,1,10000074
-10000000100,2012-08-30,FP-non GTC,223,0,5,73.26,1,10000075
-10000000101,2012-05-21,FP-non GTC,223,0,5,4.13,1,10000076
-10000000102,2012-06-06,FP-non GTC,223,0,5,290.72,1,10000077
-10000000103,2012-06-16,FP-non GTC,223,0,5,265.56,1,10000078
-10000000104,2012-05-15,FP-GTC,279,15,5,5.91,1,10000079
-10000000106,2012-12-05,Auction,314,0,5,319.79,1,10000080
-10000000107,2012-03-25,Auction,314,211,5,246,1,10000081
-10000000108,2012-03-09,Auction,314,211,5,20.35,1,10000082
-10000000109,2012-05-06,Auction,314,0,5,36.7,1,10000083
-10000000110,2012-06-15,Auction,533,0,13,101.79,1,10000084
-10000000113,2012-03-14,ABIN,1349,0,5,47.71,1,10000085
-10000000115,2012-05-20,ABIN,1349,0,5,3.49,1,10000086
-10000000117,2012-05-17,ABIN,1349,0,13,46.44,1,10000087
-10000000118,2012-03-11,ABIN,1349,0,13,4.54,1,10000088
-10000000119,2012-01-10,ABIN,1357,0,14,3.18,1,10000089
-10000000120,2012-04-13,FP-GTC,1504,0,14,86.58,1,10000090
-10000000121,2012-07-22,FP-GTC,4943,0,13,12.85,1,10000091
-10000000122,2012-06-15,ABIN,6762,0,13,16.26,1,10000092
-10000000123,2012-06-15,Auction,9426,3,13,21.14,1,10000093
-10000000124,2012-06-01,FP-non GTC,10866,0,14,20.6,1,10000094
-10000000125,2012-05-17,Auction,11554,0,13,246,1,10000095
-10000000126,2012-04-26,FP-GTC,11848,0,14,109,1,10000096
-10000000127,2012-07-28,Auction,13836,0,13,39.41,1,10000097
-10000000128,2012-03-12,Auction,13836,0,14,16.26,1,10000098
-10000000129,2012-04-20,FP-GTC,13987,0,13,112.56,1,10000099
-10000000130,2012-03-18,Auction,15687,0,14,184.21,1,10000100
-10000000131,2012-05-17,Auction,15687,0,11,27.48,1,10000001
-10000000132,2012-06-23,FP-non GTC,16145,3,12,26.45,1,10000002
-10000000134,2012-05-22,FP-non GTC,16145,0,13,415.73,1,10000003
-10000000135,2012-01-25,ABIN,16509,0,5,56.36,1,10000004
-10000000136,2012-06-12,ABIN,16509,0,5,2.44,1,10000005
-10000000137,2012-05-17,FP-GTC,20485,0,14,269.76,1,10000006
-10000000139,2012-08-03,FP-GTC,20485,101,12,109,1,10000007
-10000000140,2012-05-17,FP-GTC,20485,101,12,101.79,1,10000008
-10000000141,2012-08-21,Auction,23446,23,14,246,1,10000009
-10000000142,2012-08-21,Auction,23446,23,14,189.23,1,10000010
-10000000143,2012-04-18,Auction,23446,23,14,15.65,1,10000011
-10000000144,2012-06-16,Auction,23446,23,14,28.23,1,10000012
-10000000145,2012-01-09,FP-GTC,24541,0,5,16.26,1,10000013
-10000000146,2012-05-03,FP-GTC,26262,0,5,122.78,1,10000014
-10000000147,2012-05-20,FP-GTC,30059,3,14,172.03,1,10000015
-10000000149,2012-03-12,Auction,31387,3,14,42.99,1,10000016
-10000000150,2012-05-19,Auction,31387,3,14,207.5,1,10000017
-10000000151,2012-04-21,FP-GTC,31519,0,14,5.91,1,10000018
-10000000152,2012-05-18,FP-GTC,31519,3,14,39.41,1,10000019
-10000000155,2012-06-16,FP-GTC,31519,0,14,16.26,1,10000020
-10000000156,2012-06-11,FP-GTC,31519,0,14,16.26,1,10000021
-10000000157,2012-03-31,FP-GTC,31519,0,14,78.48,1,10000022
-10000000158,2012-11-12,FP-GTC,31519,3,14,190.22,1,10000023
-10000000161,2012-04-06,FP-GTC,35570,100,12,2.44,1,10000024
-10000000163,2012-11-01,Auction,36250,0,5,7.12,1,10000025
-10000000165,2012-02-06,FP-non GTC,38238,0,14,36.7,1,10000026
-10000000166,2012-06-16,FP-GTC,40059,3,14,35.72,1,10000027
-10000000167,2012-09-12,FP-GTC,40059,3,14,3.49,1,10000028
-10000000168,2012-04-16,FP-GTC,41940,0,13,223.63,1,10000029
-10000000169,2012-11-01,FP-GTC,41940,0,13,265.56,1,10000030
-10000000170,2012-06-14,FP-non GTC,43479,0,13,62.02,1,10000031
-10000000172,2012-11-12,FP-GTC,44079,0,12,46.44,1,10000032
-10000000173,2012-03-22,Auction,45238,101,14,132.33,1,10000033
-10000000177,2012-05-22,Auction,45333,0,13,448.8,1,10000034
-10000000178,2012-03-10,FP-non GTC,45333,0,14,207.5,1,10000035
-10000000179,2012-05-17,FP-non GTC,45333,0,14,190.22,1,10000036
-10000000181,2012-09-01,FP-GTC,46575,0,14,16.71,1,10000037
-10000000182,2012-04-18,FP-non GTC,50508,0,13,4.13,1,10000038
-10000000183,2012-06-05,FP-non GTC,50508,0,13,1.88,1,10000039
-10000000185,2012-10-08,FP-GTC,50677,0,13,491.32,1,10000040
-10000000186,2012-09-17,FP-GTC,50677,0,5,2.44,1,10000041
-10000000187,2012-02-02,Auction,51582,0,14,56.36,1,10000042
-10000000189,2012-08-23,FP-GTC,57013,0,13,15.85,1,10000043
-10000000190,2012-08-17,FP-non GTC,57013,0,14,2.44,1,10000044
-10000000191,2012-05-05,FP-GTC,57013,0,14,7.12,1,10000045
-10000000192,2012-03-08,Auction,57784,0,14,35.72,1,10000046
-10000000194,2012-03-16,Auction,57990,3,11,9.26,1,10000047
-10000000195,2012-05-09,Auction,57990,3,14,3.18,1,10000048
-10000000196,2012-03-25,Auction,57990,3,14,638.72,1,10000049
-10000000198,2012-06-05,Auction,57990,3,14,141.7,1,10000050
-10000000199,2012-01-10,ABIN,57990,0,13,12.19,1,10000051
-10000000200,2012-06-11,Auction,57990,3,14,132.33,1,10000052
-10000000201,2012-08-23,ABIN,57990,3,5,5.48,1,10000053
-10000000203,2012-08-20,Auction,57990,3,11,1.88,1,10000054
-10000000204,2012-08-09,FP-GTC,60340,0,14,12.85,1,10000055
-10000000208,2012-06-30,FP-GTC,60340,0,14,62.02,1,10000056
-10000000209,2012-06-03,FP-GTC,60606,3,12,15.85,1,10000057
-10000000211,2012-08-14,FP-GTC,60606,3,12,9.26,1,10000058
-10000000213,2012-08-14,FP-GTC,60606,3,12,16.71,1,10000059
-10000000214,2012-05-17,FP-GTC,60606,3,12,20.6,1,10000060
-10000000217,2012-07-23,Auction,63861,3,5,1.88,1,10000061
-10000000218,2012-07-27,ABIN,63861,0,5,141.7,1,10000062
-10000000219,2012-07-29,ABIN,63861,0,5,1.88,1,10000063
-10000000220,2012-11-17,Others,63861,0,11,112.56,1,10000064
-10000000221,2012-10-29,Others,63861,0,11,94.45,1,10000065
-10000000222,2012-03-28,Auction,63861,0,14,78.48,1,10000066
-10000000223,2012-01-30,ABIN,63861,0,13,5.48,1,10000067
-10000000224,2012-01-26,Auction,63864,3,14,28.23,1,10000068
-10000000229,2012-11-21,Others,63889,0,13,3.49,1,10000069
-10000000231,2012-07-12,FP-GTC,67698,2,11,15.65,1,10000070
-10000000232,2012-06-07,FP-GTC,67698,0,11,5.48,1,10000071
-10000000233,2012-12-01,FP-GTC,67698,0,11,246,1,10000072
-10000000234,2012-12-02,FP-non GTC,73506,0,13,122.78,1,10000073
-10000000235,2012-02-01,FP-GTC,75665,0,14,223.63,1,10000074
-10000000236,2012-01-14,ABIN,75708,3,5,141.7,1,10000075
-10000000237,2012-09-16,FP-non GTC,80053,0,11,21.14,1,10000076
-10000000239,2012-08-09,FP-non GTC,80053,0,11,55.89,1,10000077
-10000000241,2012-12-26,FP-non GTC,80053,0,11,51.23,1,10000078
-10000000242,2012-12-31,Auction,80135,0,14,21.72,1,10000079
-10000000243,2012-03-22,Auction,95672,3,14,204.28,1,10000080
-10000000244,2012-12-25,Others,95672,0,11,21.14,1,10000081
-10000000245,2012-01-15,Others,100847,0,5,204.28,1,10000082
-10000000248,2012-05-27,Others,100847,0,5,122.78,1,10000083
-10000000249,2012-01-11,ABIN,139973,3,14,94.45,1,10000084
-10000000250,2012-02-04,ABIN,139973,0,11,86.58,1,10000085
-10000000251,2012-12-15,Auction,150047,3,14,56.36,1,10000086
-10000000252,2012-06-30,Auction,150047,3,14,290.72,1,10000087
-10000000253,2012-12-16,FP-GTC,155226,0,13,60.37,1,10000088
-10000000254,2012-11-11,FP-GTC,155226,0,13,112.56,1,10000089
-10000000256,2012-10-08,FP-GTC,156356,0,13,265.56,1,10000090
-10000000257,2012-04-26,FP-GTC,158798,0,11,35.72,1,10000091
-10000000258,2012-10-06,FP-non GTC,165888,0,13,92.98,1,10000092
-10000000259,2012-12-28,Auction,170083,3,11,28.23,1,10000093
-10000000260,2012-11-06,Auction,170083,3,11,27.48,1,10000094
-10000000261,2012-11-06,Auction,175750,3,14,9.26,1,10000095
-10000000262,2012-12-27,Auction,175750,3,14,3.18,1,10000096
-10000000263,2012-01-01,FP-GTC,175750,0,14,12.04,1,10000097
-10000000265,2012-08-23,Auction,175750,3,13,20.6,1,10000098
-10000000266,2012-07-10,Auction,175750,3,13,12.04,1,10000099
-10000000267,2012-08-10,Auction,175750,3,14,4.13,1,10000100
-10000000268,2012-07-19,Auction,175750,3,14,73.26,1,10000201
+10000000157,2013-03-31,Auction,48027,0,12,184.21,1,10000001,eef
+10000000158,2013-11-12,Others,164261,0,5,172.03,1,10000002,gji
+10000000161,2013-04-06,Auction,82494,15,14,66.6,1,10000003,jjc
+10000000137,2013-05-17,Auction,66767,15,12,92.98,1,10000004,add
+10000000147,2013-05-20,FP-GTC,152801,0,5,132.33,1,10000005,ife
+10000000155,2013-06-16,FP-GTC,43398,0,13,7.12,1,10000006,hce
+10000000170,2013-06-14,Auction,95173,0,14,204.28,1,10000007,bei
+10000000173,2013-03-22,Auction,158666,15,13,35.72,1,10000008,bjb
+10000000178,2013-03-10,Auction,12688,0,12,4.13,1,10000009,daf
+10000000163,2013-11-01,FP-GTC,103324,15,5,27.48,1,10000010,cji
+10000000166,2013-06-16,FP-GTC,108782,15,14,9.26,1,10000011,hch
+10000000167,2013-09-12,Auction,80287,0,12,3.18,1,10000012,edc
+10000000086,2013-09-28,Others,140746,100,13,3.18,1,10000013,jhi
+10000000110,2013-06-15,ABIN,87118,0,14,377.94,1,10000014,age
+10000000113,2013-03-14,Auction,25147,0,12,146.33,1,10000015,afc
+10000000069,2013-09-01,FP-GTC,170302,15,5,51.23,1,10000016,jib
+10000000079,2013-05-29,FP-non GTC,53064,0,13,72.65,1,10000017,bai
+10000000080,2013-05-31,Auction,132939,0,13,66.6,1,10000018,gii
+10000000130,2013-03-18,Auction,113593,15,12,9.26,1,10000019,fcj
+10000000268,2013-07-19,Auction,34273,100,14,583.44,1,10000020,ifc
+10000000132,2013-06-23,FP-GTC,106340,15,14,638.72,1,10000021,jfe
+10000000115,2013-05-20,FP-GTC,150265,15,14,4.54,1,10000022,ehd
+10000000117,2013-05-17,FP-GTC,24760,0,12,319.79,1,10000023,hbc
+10000000118,2013-03-11,Auction,37831,0,12,20.35,1,10000024,dbh
+10000000223,2013-01-30,FP-non GTC,1120,3,5,223.63,1,10000025,igh
+10000000224,2013-01-26,FP-GTC,43972,100,13,204.28,1,10000026,jbe
+10000000243,2013-03-22,Auction,166013,15,14,5.48,1,10000027,deb
+10000000217,2013-07-23,Auction,15568,15,14,27.48,1,10000028,hai
+10000000218,2013-07-27,FP-GTC,103178,15,5,21.72,1,10000029,cfj
+10000000221,2013-10-29,ABIN,2023,0,12,3.18,1,10000030,abc
+10000000256,2013-10-08,FP-GTC,94847,0,11,491.32,1,10000031,eha
+10000000257,2013-04-26,Auction,15868,0,14,448.8,1,10000032,fig
+10000000263,2013-01-01,Auction,32876,0,13,415.73,1,10000033,ghj
+10000000245,2013-01-15,Auction,62179,0,13,377.94,1,10000034,edj
+10000000248,2013-05-27,FP-GTC,33038,15,14,146.33,1,10000035,jii
+10000000254,2013-11-11,FP-GTC,156614,0,5,7.12,1,10000036,jbi
+10000000192,2013-03-08,Auction,106246,0,13,42.99,1,10000037,hce
+10000000196,2013-03-25,Auction,20865,0,13,12.85,1,10000038,jdc
+10000000203,2013-08-20,FP-GTC,15115,0,13,55.89,1,10000039,ggc
+10000000179,2013-05-17,FP-GTC,3838,0,14,73.26,1,10000040,bca
+10000000183,2013-06-05,Auction,759,0,11,112.56,1,10000041,hbh
+10000000185,2013-10-08,FP-non GTC,61323,0,11,3.49,1,10000042,hif
+10000000211,2013-08-14,Auction,121153,0,13,184.21,1,10000043,hhd
+10000000213,2013-08-14,Auction,88750,0,13,157.14,1,10000044,eif
+10000000214,2013-05-17,FP-GTC,161567,15,14,72.65,1,10000045,jjg
+10000000204,2013-08-09,FP-GTC,113802,15,14,51.23,1,10000046,igd
+10000000208,2013-06-30,FP-non GTC,15808,15,14,15.85,1,10000047,fii
+10000000209,2013-06-03,FP-GTC,174053,3,13,7.12,1,10000048,bdj
+10000000042,2013-12-31,Auction,2635,0,14,12.04,1,10000049,hfi
+10000000044,2013-12-25,Auction,1161,3,13,73.26,1,10000050,gaf
+10000000022,2013-03-28,FP-non GTC,64076,0,5,184.21,1,10000051,agf
+10000000023,2013-01-30,FP-GTC,33977,15,13,172.03,1,10000052,hic
+10000000047,2013-12-05,FP-GTC,31673,0,14,122.78,1,10000053,fae
+10000000056,2013-10-08,Auction,174106,3,14,92.98,1,10000054,ica
+10000000062,2013-12-27,Auction,26249,0,13,12.19,1,10000055,jfa
+10000000053,2013-12-16,FP-GTC,159184,0,5,15.65,1,10000056,hca
+10000000055,2013-10-17,FP-GTC,10058,3,11,101.79,1,10000057,ijc
+10000000020,2013-11-17,ABIN,48904,0,12,7.12,1,10000058,jfd
+10000000007,2013-09-18,FP-non GTC,145970,0,14,12.85,1,10000059,iab
+10000000008,2013-06-30,FP-GTC,963,0,13,12.19,1,10000060,abb
+10000000002,2013-10-12,FP-GTC,118687,3,13,92.98,1,10000061,gja
+10000000003,2013-08-20,FP-GTC,20886,0,14,42.99,1,10000062,iba
+10000000010,2013-08-29,Auction,148324,15,13,1.88,1,10000063,deh
+10000000016,2013-07-17,Auction,139255,15,14,21.14,1,10000064,afb
+10000000017,2013-07-23,FP-GTC,20213,0,5,21.14,1,10000065,ijb
+10000000012,2013-01-06,Auction,32996,15,13,132.33,1,10000066,ddi
+10000000013,2013-08-14,FP-GTC,99985,0,14,120.87,1,10000067,aej
+10000000067,2013-08-10,Auction,67703,3,14,120.87,1,10000068,jif
+10000000085,2013-09-28,FP-non GTC,65,0,11,9.26,1,10000069,fcb
+10000000141,2013-08-21,FP-non GTC,130,0,14,16.26,1,10000070,dib
+10000000078,2013-03-11,FP-GTC,164,0,14,157.14,1,10000071,ejc
+10000000109,2013-05-06,FP-GTC,216,0,11,1.88,1,10000072,jhb
+10000000096,2013-05-17,FP-non GTC,223,0,5,12.04,1,10000073,jgg
+10000000095,2013-01-10,FP-non GTC,223,0,14,189.23,1,10000074,bgb
+10000000098,2013-05-05,FP-non GTC,223,0,5,73.26,1,10000075,efh
+10000000097,2013-02-03,FP-non GTC,223,0,5,4.13,1,10000076,jja
+10000000099,2013-11-26,FP-non GTC,223,0,5,290.72,1,10000077,cbg
+10000000100,2013-08-30,FP-non GTC,223,0,5,265.56,1,10000078,gef
+10000000126,2013-04-26,FP-GTC,279,15,5,5.91,1,10000079,eji
+10000000252,2013-06-30,Auction,314,0,5,319.79,1,10000080,jhf
+10000000052,2013-06-30,Auction,314,211,5,246,1,10000081,hgg
+10000000253,2013-12-16,Auction,314,211,5,20.35,1,10000082,jjg
+10000000051,2013-12-15,Auction,314,0,5,36.7,1,10000083,ejf
+10000000190,2013-08-17,Auction,533,0,13,101.79,1,10000084,cbc
+10000000251,2013-12-15,ABIN,1349,0,5,47.71,1,10000085,aeh
+10000000050,2013-02-04,ABIN,1349,0,5,3.49,1,10000086,jdb
+10000000049,2013-01-11,ABIN,1349,0,13,46.44,1,10000087,eaj
+10000000250,2013-02-04,ABIN,1349,0,13,4.54,1,10000088,jjh
+10000000131,2013-05-17,ABIN,1357,0,14,3.18,1,10000089,cga
+10000000172,2013-11-12,FP-GTC,1504,0,14,86.58,1,10000090,jjj
+10000000142,2013-08-21,FP-GTC,4943,0,13,12.85,1,10000091,egj
+10000000195,2013-05-09,ABIN,6762,0,13,16.26,1,10000092,ehg
+10000000070,2013-09-19,Auction,9426,3,13,21.14,1,10000093,fff
+10000000165,2013-02-06,FP-non GTC,10866,0,14,20.6,1,10000094,bha
+10000000187,2013-02-02,Auction,11554,0,13,246,1,10000095,gfa
+10000000189,2013-08-23,FP-GTC,11848,0,14,109,1,10000096,jeb
+10000000139,2013-08-03,Auction,13836,0,13,39.41,1,10000097,baf
+10000000140,2013-05-17,Auction,13836,0,14,16.26,1,10000098,adh
+10000000102,2013-06-06,FP-GTC,13987,0,13,112.56,1,10000099,eii
+10000000076,2013-07-02,Auction,15687,0,14,184.21,1,10000100,ijj
+10000000082,2013-10-25,Auction,15687,0,11,27.48,1,10000001,jji
+10000000129,2013-04-20,FP-non GTC,16145,3,12,26.45,1,10000002,jfd
+10000000128,2013-03-12,FP-non GTC,16145,0,13,415.73,1,10000003,ibh
+10000000222,2013-03-28,ABIN,16509,0,5,56.36,1,10000004,gaa
+10000000021,2013-10-29,ABIN,16509,0,5,2.44,1,10000005,ece
+10000000134,2013-05-22,FP-GTC,20485,0,14,269.76,1,10000006,bjj
+10000000135,2013-01-25,FP-GTC,20485,101,12,109,1,10000007,acd
+10000000136,2013-06-12,FP-GTC,20485,101,12,101.79,1,10000008,ade
+10000000241,2013-12-26,Auction,23446,23,14,246,1,10000009,iai
+10000000041,2013-12-26,Auction,23446,23,14,189.23,1,10000010,jga
+10000000242,2013-12-31,Auction,23446,23,14,15.65,1,10000011,fed
+10000000040,2013-10-04,Auction,23446,23,14,28.23,1,10000012,gfh
+10000000194,2013-03-16,FP-GTC,24541,0,5,16.26,1,10000013,bjb
+10000000101,2013-05-21,FP-GTC,26262,0,5,122.78,1,10000014,jba
+10000000077,2013-01-28,FP-GTC,30059,3,14,172.03,1,10000015,fjb
+10000000057,2013-04-26,Auction,31387,3,14,42.99,1,10000016,bdg
+10000000258,2013-10-06,Auction,31387,3,14,207.5,1,10000017,jja
+10000000261,2013-11-06,FP-GTC,31519,0,14,5.91,1,10000018,ggd
+10000000058,2013-10-06,FP-GTC,31519,3,14,39.41,1,10000019,bdc
+10000000059,2013-12-28,FP-GTC,31519,0,14,16.26,1,10000020,cid
+10000000060,2013-11-06,FP-GTC,31519,0,14,16.26,1,10000021,jfg
+10000000260,2013-11-06,FP-GTC,31519,0,14,78.48,1,10000022,gha
+10000000259,2013-12-28,FP-GTC,31519,3,14,190.22,1,10000023,fdh
+10000000156,2013-06-11,FP-GTC,35570,100,12,2.44,1,10000024,fad
+10000000119,2013-01-10,Auction,36250,0,5,7.12,1,10000025,hjf
+10000000186,2013-09-17,FP-non GTC,38238,0,14,36.7,1,10000026,jdh
+10000000038,2013-08-14,FP-GTC,40059,3,14,35.72,1,10000027,bhh
+10000000239,2013-08-09,FP-GTC,40059,3,14,3.49,1,10000028,gjj
+10000000034,2013-12-02,FP-GTC,41940,0,13,223.63,1,10000029,cjf
+10000000235,2013-02-01,FP-GTC,41940,0,13,265.56,1,10000030,caj
+10000000127,2013-07-28,FP-non GTC,43479,0,13,62.02,1,10000031,fcd
+10000000103,2013-06-16,FP-GTC,44079,0,12,46.44,1,10000032,bhg
+10000000201,2013-08-23,Auction,45238,101,14,132.33,1,10000033,ddi
+10000000122,2013-06-15,Auction,45333,0,13,448.8,1,10000034,jce
+10000000123,2013-06-15,FP-non GTC,45333,0,14,207.5,1,10000035,daf
+10000000124,2013-06-01,FP-non GTC,45333,0,14,190.22,1,10000036,gfg
+10000000072,2013-08-10,FP-GTC,46575,0,14,16.71,1,10000037,cjj
+10000000043,2013-03-22,FP-non GTC,50508,0,13,4.13,1,10000038,agg
+10000000244,2013-12-25,FP-non GTC,50508,0,13,1.88,1,10000039,bad
+10000000121,2013-07-22,FP-GTC,50677,0,13,491.32,1,10000040,fhb
+10000000120,2013-04-13,FP-GTC,50677,0,5,2.44,1,10000041,cfc
+10000000168,2013-04-16,Auction,51582,0,14,56.36,1,10000042,eie
+10000000073,2013-08-21,FP-GTC,57013,0,13,15.85,1,10000043,cjc
+10000000075,2013-04-22,FP-non GTC,57013,0,14,2.44,1,10000044,hgc
+10000000074,2013-08-29,FP-GTC,57013,0,14,7.12,1,10000045,fdd
+10000000093,2013-05-16,Auction,57784,0,14,35.72,1,10000046,heg
+10000000265,2013-08-23,Auction,57990,3,11,9.26,1,10000047,cdf
+10000000266,2013-07-10,Auction,57990,3,14,3.18,1,10000048,adi
+10000000267,2013-08-10,Auction,57990,3,14,638.72,1,10000049,egh
+10000000065,2013-08-23,Auction,57990,3,14,141.7,1,10000050,hge
+10000000143,2013-04-18,ABIN,57990,0,13,12.19,1,10000051,egj
+10000000066,2013-07-10,Auction,57990,3,14,132.33,1,10000052,fea
+10000000144,2013-06-16,ABIN,57990,3,5,5.48,1,10000053,abf
+10000000064,2013-07-15,Auction,57990,3,11,1.88,1,10000054,ecc
+10000000061,2013-11-06,FP-GTC,60340,0,14,12.85,1,10000055,hda
+10000000262,2013-12-27,FP-GTC,60340,0,14,62.02,1,10000056,cgh
+10000000019,2013-07-29,FP-GTC,60606,3,12,15.85,1,10000057,gbf
+10000000220,2013-11-17,FP-GTC,60606,3,12,9.26,1,10000058,bae
+10000000018,2013-07-27,FP-GTC,60606,3,12,16.71,1,10000059,hag
+10000000219,2013-07-29,FP-GTC,60606,3,12,20.6,1,10000060,ici
+10000000145,2013-01-09,Auction,63861,3,5,1.88,1,10000061,bfa
+10000000200,2013-06-11,ABIN,63861,0,5,141.7,1,10000062,dea
+10000000199,2013-01-10,ABIN,63861,0,5,1.88,1,10000063,bce
+10000000237,2013-09-16,Others,63861,0,11,112.56,1,10000064,cfi
+10000000036,2013-01-14,Others,63861,0,11,94.45,1,10000065,ijh
+10000000125,2013-05-17,Auction,63861,0,14,78.48,1,10000066,gag
+10000000198,2013-06-05,ABIN,63861,0,13,5.48,1,10000067,bja
+10000000094,2013-05-24,Auction,63864,3,14,28.23,1,10000068,gca
+10000000104,2013-05-15,Others,63889,0,13,3.49,1,10000069,fcf
+10000000107,2013-03-25,FP-GTC,67698,2,11,15.65,1,10000070,fae
+10000000108,2013-03-09,FP-GTC,67698,0,11,5.48,1,10000071,cci
+10000000106,2013-12-05,FP-GTC,67698,0,11,246,1,10000072,efc
+10000000182,2013-04-18,FP-non GTC,73506,0,13,122.78,1,10000073,cig
+10000000169,2013-11-01,FP-GTC,75665,0,14,223.63,1,10000074,big
+10000000146,2013-05-03,ABIN,75708,3,5,141.7,1,10000075,bgi
+10000000151,2013-04-21,FP-non GTC,80053,0,11,21.14,1,10000076,bdg
+10000000149,2013-03-12,FP-non GTC,80053,0,11,55.89,1,10000077,ffh
+10000000150,2013-05-19,FP-non GTC,80053,0,11,51.23,1,10000078,chb
+10000000083,2013-11-23,Auction,80135,0,14,21.72,1,10000079,dhd
+10000000089,2013-10-19,Auction,95672,3,14,204.28,1,10000080,gid
+10000000152,2013-05-18,Others,95672,0,11,21.14,1,10000081,jij
+10000000035,2013-02-01,Others,100847,0,5,204.28,1,10000082,ddj
+10000000236,2013-01-14,Others,100847,0,5,122.78,1,10000083,hab
+10000000090,2013-08-05,ABIN,139973,3,14,94.45,1,10000084,fai
+10000000091,2013-05-19,ABIN,139973,0,11,86.58,1,10000085,bei
+10000000033,2013-12-01,Auction,150047,3,14,56.36,1,10000086,jhh
+10000000234,2013-12-02,Auction,150047,3,14,290.72,1,10000087,bdg
+10000000249,2013-01-11,FP-GTC,155226,0,13,60.37,1,10000088,heh
+10000000048,2013-05-27,FP-GTC,155226,0,13,112.56,1,10000089,hja
+10000000181,2013-09-01,FP-GTC,156356,0,13,265.56,1,10000090,egc
+10000000092,2013-04-11,FP-GTC,158798,0,11,35.72,1,10000091,dgb
+10000000191,2013-05-05,FP-non GTC,165888,0,13,92.98,1,10000092,idc
+10000000229,2013-11-21,Auction,170083,3,11,28.23,1,10000093,bgi
+10000000028,2013-10-07,Auction,170083,3,11,27.48,1,10000094,ihd
+10000000031,2013-07-12,Auction,175750,3,14,9.26,1,10000095,gci
+10000000032,2013-06-07,Auction,175750,3,14,3.18,1,10000096,fba
+10000000177,2013-05-22,FP-GTC,175750,0,14,12.04,1,10000097,adb
+10000000030,2013-11-28,Auction,175750,3,13,20.6,1,10000098,hbg
+10000000231,2013-07-12,Auction,175750,3,13,12.04,1,10000099,ige
+10000000232,2013-06-07,Auction,175750,3,14,4.13,1,10000100,jjj
+10000000233,2013-12-01,Auction,175750,3,14,73.26,1,10000201,ijb
+10000000002,2012-10-12,Auction,48027,0,12,184.21,1,10000001,ehb
+10000000003,2012-08-20,Others,164261,0,5,172.03,1,10000002,cai
+10000000007,2012-09-18,Auction,82494,15,14,66.6,1,10000003,bie
+10000000008,2012-06-30,Auction,66767,15,12,92.98,1,10000004,iba
+10000000010,2012-08-29,FP-GTC,152801,0,5,132.33,1,10000005,hjf
+10000000012,2012-01-06,FP-GTC,43398,0,13,7.12,1,10000006,dbg
+10000000013,2012-08-14,Auction,95173,0,14,204.28,1,10000007,fdb
+10000000016,2012-07-17,Auction,158666,15,13,35.72,1,10000008,icf
+10000000017,2012-07-23,Auction,12688,0,12,4.13,1,10000009,iag
+10000000018,2012-07-27,FP-GTC,103324,15,5,27.48,1,10000010,egj
+10000000019,2012-07-29,FP-GTC,108782,15,14,9.26,1,10000011,jda
+10000000020,2012-11-17,Auction,80287,0,12,3.18,1,10000012,cdc
+10000000021,2012-10-29,Others,140746,100,13,3.18,1,10000013,efa
+10000000022,2012-03-28,ABIN,87118,0,14,377.94,1,10000014,ibd
+10000000023,2012-01-30,Auction,25147,0,12,146.33,1,10000015,jbh
+10000000028,2012-10-07,FP-GTC,170302,15,5,51.23,1,10000016,bjj
+10000000030,2012-11-28,FP-non GTC,53064,0,13,72.65,1,10000017,djd
+10000000031,2012-07-12,Auction,132939,0,13,66.6,1,10000018,gae
+10000000032,2012-06-07,Auction,113593,15,12,9.26,1,10000019,adj
+10000000033,2012-12-01,Auction,34273,100,14,583.44,1,10000020,cdd
+10000000034,2012-12-02,FP-GTC,106340,15,14,638.72,1,10000021,cid
+10000000035,2012-02-01,FP-GTC,150265,15,14,4.54,1,10000022,jfj
+10000000036,2012-01-14,FP-GTC,24760,0,12,319.79,1,10000023,abi
+10000000038,2012-08-14,Auction,37831,0,12,20.35,1,10000024,fag
+10000000040,2012-10-04,FP-non GTC,1120,3,5,223.63,1,10000025,gci
+10000000041,2012-12-26,FP-GTC,43972,100,13,204.28,1,10000026,gch
+10000000042,2012-12-31,Auction,166013,15,14,5.48,1,10000027,ifh
+10000000043,2012-03-22,Auction,15568,15,14,27.48,1,10000028,hfh
+10000000044,2012-12-25,FP-GTC,103178,15,5,21.72,1,10000029,ggc
+10000000047,2012-12-05,ABIN,2023,0,12,3.18,1,10000030,gde
+10000000048,2012-05-27,FP-GTC,94847,0,11,491.32,1,10000031,hha
+10000000049,2012-01-11,Auction,15868,0,14,448.8,1,10000032,hgf
+10000000050,2012-02-04,Auction,32876,0,13,415.73,1,10000033,eia
+10000000051,2012-12-15,Auction,62179,0,13,377.94,1,10000034,cfh
+10000000052,2012-06-30,FP-GTC,33038,15,14,146.33,1,10000035,ide
+10000000053,2012-12-16,FP-GTC,156614,0,5,7.12,1,10000036,gac
+10000000055,2012-10-17,Auction,106246,0,13,42.99,1,10000037,afa
+10000000056,2012-10-08,Auction,20865,0,13,12.85,1,10000038,hed
+10000000057,2012-04-26,FP-GTC,15115,0,13,55.89,1,10000039,hif
+10000000058,2012-10-06,FP-GTC,3838,0,14,73.26,1,10000040,ggb
+10000000059,2012-12-28,Auction,759,0,11,112.56,1,10000041,ffi
+10000000060,2012-11-06,FP-non GTC,61323,0,11,3.49,1,10000042,bdh
+10000000061,2012-11-06,Auction,121153,0,13,184.21,1,10000043,bej
+10000000062,2012-12-27,Auction,88750,0,13,157.14,1,10000044,ihh
+10000000064,2012-07-15,FP-GTC,161567,15,14,72.65,1,10000045,eag
+10000000065,2012-08-23,FP-GTC,113802,15,14,51.23,1,10000046,fcc
+10000000066,2012-07-10,FP-non GTC,15808,15,14,15.85,1,10000047,gbb
+10000000067,2012-08-10,FP-GTC,174053,3,13,7.12,1,10000048,ihb
+10000000069,2012-09-01,Auction,2635,0,14,12.04,1,10000049,bie
+10000000070,2012-09-19,Auction,1161,3,13,73.26,1,10000050,eah
+10000000072,2012-08-10,FP-non GTC,64076,0,5,184.21,1,10000051,iag
+10000000073,2012-08-21,FP-GTC,33977,15,13,172.03,1,10000052,aga
+10000000074,2012-08-29,FP-GTC,31673,0,14,122.78,1,10000053,ede
+10000000075,2012-04-22,Auction,174106,3,14,92.98,1,10000054,gii
+10000000076,2012-07-02,Auction,26249,0,13,12.19,1,10000055,heb
+10000000077,2012-01-28,FP-GTC,159184,0,5,15.65,1,10000056,fab
+10000000078,2012-03-11,FP-GTC,10058,3,11,101.79,1,10000057,dbf
+10000000079,2012-05-29,ABIN,48904,0,12,7.12,1,10000058,fda
+10000000080,2012-05-31,FP-non GTC,145970,0,14,12.85,1,10000059,gaf
+10000000082,2012-10-25,FP-GTC,963,0,13,12.19,1,10000060,bfc
+10000000083,2012-11-23,FP-GTC,118687,3,13,92.98,1,10000061,eed
+10000000085,2012-09-28,FP-GTC,20886,0,14,42.99,1,10000062,deh
+10000000086,2012-09-28,Auction,148324,15,13,1.88,1,10000063,dii
+10000000089,2012-10-19,Auction,139255,15,14,21.14,1,10000064,ceh
+10000000090,2012-08-05,FP-GTC,20213,0,5,21.14,1,10000065,ejd
+10000000091,2012-05-19,Auction,32996,15,13,132.33,1,10000066,fab
+10000000092,2012-04-11,FP-GTC,99985,0,14,120.87,1,10000067,dab
+10000000093,2012-05-16,Auction,67703,3,14,120.87,1,10000068,big
+10000000094,2012-05-24,FP-non GTC,65,0,11,9.26,1,10000069,afj
+10000000095,2012-01-10,FP-non GTC,130,0,14,16.26,1,10000070,igh
+10000000096,2012-05-17,FP-GTC,164,0,14,157.14,1,10000071,aff
+10000000097,2012-02-03,FP-GTC,216,0,11,1.88,1,10000072,aif
+10000000098,2012-05-05,FP-non GTC,223,0,5,12.04,1,10000073,gcd
+10000000099,2012-11-26,FP-non GTC,223,0,14,189.23,1,10000074,hec
+10000000100,2012-08-30,FP-non GTC,223,0,5,73.26,1,10000075,cdh
+10000000101,2012-05-21,FP-non GTC,223,0,5,4.13,1,10000076,dei
+10000000102,2012-06-06,FP-non GTC,223,0,5,290.72,1,10000077,gib
+10000000103,2012-06-16,FP-non GTC,223,0,5,265.56,1,10000078,hbb
+10000000104,2012-05-15,FP-GTC,279,15,5,5.91,1,10000079,jdh
+10000000106,2012-12-05,Auction,314,0,5,319.79,1,10000080,fag
+10000000107,2012-03-25,Auction,314,211,5,246,1,10000081,egj
+10000000108,2012-03-09,Auction,314,211,5,20.35,1,10000082,ihi
+10000000109,2012-05-06,Auction,314,0,5,36.7,1,10000083,efi
+10000000110,2012-06-15,Auction,533,0,13,101.79,1,10000084,hed
+10000000113,2012-03-14,ABIN,1349,0,5,47.71,1,10000085,ajb
+10000000115,2012-05-20,ABIN,1349,0,5,3.49,1,10000086,gdf
+10000000117,2012-05-17,ABIN,1349,0,13,46.44,1,10000087,bch
+10000000118,2012-03-11,ABIN,1349,0,13,4.54,1,10000088,hdb
+10000000119,2012-01-10,ABIN,1357,0,14,3.18,1,10000089,dda
+10000000120,2012-04-13,FP-GTC,1504,0,14,86.58,1,10000090,gdd
+10000000121,2012-07-22,FP-GTC,4943,0,13,12.85,1,10000091,ahe
+10000000122,2012-06-15,ABIN,6762,0,13,16.26,1,10000092,cdj
+10000000123,2012-06-15,Auction,9426,3,13,21.14,1,10000093,jff
+10000000124,2012-06-01,FP-non GTC,10866,0,14,20.6,1,10000094,jfg
+10000000125,2012-05-17,Auction,11554,0,13,246,1,10000095,cjc
+10000000126,2012-04-26,FP-GTC,11848,0,14,109,1,10000096,baf
+10000000127,2012-07-28,Auction,13836,0,13,39.41,1,10000097,aha
+10000000128,2012-03-12,Auction,13836,0,14,16.26,1,10000098,gjc
+10000000129,2012-04-20,FP-GTC,13987,0,13,112.56,1,10000099,ige
+10000000130,2012-03-18,Auction,15687,0,14,184.21,1,10000100,dcg
+10000000131,2012-05-17,Auction,15687,0,11,27.48,1,10000001,ifb
+10000000132,2012-06-23,FP-non GTC,16145,3,12,26.45,1,10000002,dei
+10000000134,2012-05-22,FP-non GTC,16145,0,13,415.73,1,10000003,gih
+10000000135,2012-01-25,ABIN,16509,0,5,56.36,1,10000004,iha
+10000000136,2012-06-12,ABIN,16509,0,5,2.44,1,10000005,ige
+10000000137,2012-05-17,FP-GTC,20485,0,14,269.76,1,10000006,iab
+10000000139,2012-08-03,FP-GTC,20485,101,12,109,1,10000007,hhb
+10000000140,2012-05-17,FP-GTC,20485,101,12,101.79,1,10000008,ehc
+10000000141,2012-08-21,Auction,23446,23,14,246,1,10000009,dcc
+10000000142,2012-08-21,Auction,23446,23,14,189.23,1,10000010,hia
+10000000143,2012-04-18,Auction,23446,23,14,15.65,1,10000011,dja
+10000000144,2012-06-16,Auction,23446,23,14,28.23,1,10000012,baa
+10000000145,2012-01-09,FP-GTC,24541,0,5,16.26,1,10000013,aib
+10000000146,2012-05-03,FP-GTC,26262,0,5,122.78,1,10000014,dia
+10000000147,2012-05-20,FP-GTC,30059,3,14,172.03,1,10000015,ebd
+10000000149,2012-03-12,Auction,31387,3,14,42.99,1,10000016,gbe
+10000000150,2012-05-19,Auction,31387,3,14,207.5,1,10000017,fhg
+10000000151,2012-04-21,FP-GTC,31519,0,14,5.91,1,10000018,bhg
+10000000152,2012-05-18,FP-GTC,31519,3,14,39.41,1,10000019,gaf
+10000000155,2012-06-16,FP-GTC,31519,0,14,16.26,1,10000020,bad
+10000000156,2012-06-11,FP-GTC,31519,0,14,16.26,1,10000021,fic
+10000000157,2012-03-31,FP-GTC,31519,0,14,78.48,1,10000022,haj
+10000000158,2012-11-12,FP-GTC,31519,3,14,190.22,1,10000023,jcf
+10000000161,2012-04-06,FP-GTC,35570,100,12,2.44,1,10000024,bjc
+10000000163,2012-11-01,Auction,36250,0,5,7.12,1,10000025,def
+10000000165,2012-02-06,FP-non GTC,38238,0,14,36.7,1,10000026,gfj
+10000000166,2012-06-16,FP-GTC,40059,3,14,35.72,1,10000027,gbf
+10000000167,2012-09-12,FP-GTC,40059,3,14,3.49,1,10000028,dfi
+10000000168,2012-04-16,FP-GTC,41940,0,13,223.63,1,10000029,fhf
+10000000169,2012-11-01,FP-GTC,41940,0,13,265.56,1,10000030,ffc
+10000000170,2012-06-14,FP-non GTC,43479,0,13,62.02,1,10000031,iid
+10000000172,2012-11-12,FP-GTC,44079,0,12,46.44,1,10000032,dhg
+10000000173,2012-03-22,Auction,45238,101,14,132.33,1,10000033,dab
+10000000177,2012-05-22,Auction,45333,0,13,448.8,1,10000034,hci
+10000000178,2012-03-10,FP-non GTC,45333,0,14,207.5,1,10000035,hgh
+10000000179,2012-05-17,FP-non GTC,45333,0,14,190.22,1,10000036,ehc
+10000000181,2012-09-01,FP-GTC,46575,0,14,16.71,1,10000037,djg
+10000000182,2012-04-18,FP-non GTC,50508,0,13,4.13,1,10000038,gcg
+10000000183,2012-06-05,FP-non GTC,50508,0,13,1.88,1,10000039,bfa
+10000000185,2012-10-08,FP-GTC,50677,0,13,491.32,1,10000040,dah
+10000000186,2012-09-17,FP-GTC,50677,0,5,2.44,1,10000041,eeh
+10000000187,2012-02-02,Auction,51582,0,14,56.36,1,10000042,jaf
+10000000189,2012-08-23,FP-GTC,57013,0,13,15.85,1,10000043,hfc
+10000000190,2012-08-17,FP-non GTC,57013,0,14,2.44,1,10000044,ebg
+10000000191,2012-05-05,FP-GTC,57013,0,14,7.12,1,10000045,aff
+10000000192,2012-03-08,Auction,57784,0,14,35.72,1,10000046,hdc
+10000000194,2012-03-16,Auction,57990,3,11,9.26,1,10000047,hai
+10000000195,2012-05-09,Auction,57990,3,14,3.18,1,10000048,fdi
+10000000196,2012-03-25,Auction,57990,3,14,638.72,1,10000049,gii
+10000000198,2012-06-05,Auction,57990,3,14,141.7,1,10000050,cgg
+10000000199,2012-01-10,ABIN,57990,0,13,12.19,1,10000051,hed
+10000000200,2012-06-11,Auction,57990,3,14,132.33,1,10000052,jhf
+10000000201,2012-08-23,ABIN,57990,3,5,5.48,1,10000053,cbb
+10000000203,2012-08-20,Auction,57990,3,11,1.88,1,10000054,bed
+10000000204,2012-08-09,FP-GTC,60340,0,14,12.85,1,10000055,dei
+10000000208,2012-06-30,FP-GTC,60340,0,14,62.02,1,10000056,dbb
+10000000209,2012-06-03,FP-GTC,60606,3,12,15.85,1,10000057,fbf
+10000000211,2012-08-14,FP-GTC,60606,3,12,9.26,1,10000058,ijd
+10000000213,2012-08-14,FP-GTC,60606,3,12,16.71,1,10000059,hci
+10000000214,2012-05-17,FP-GTC,60606,3,12,20.6,1,10000060,hjh
+10000000217,2012-07-23,Auction,63861,3,5,1.88,1,10000061,cic
+10000000218,2012-07-27,ABIN,63861,0,5,141.7,1,10000062,cig
+10000000219,2012-07-29,ABIN,63861,0,5,1.88,1,10000063,gae
+10000000220,2012-11-17,Others,63861,0,11,112.56,1,10000064,bjd
+10000000221,2012-10-29,Others,63861,0,11,94.45,1,10000065,cih
+10000000222,2012-03-28,Auction,63861,0,14,78.48,1,10000066,hbe
+10000000223,2012-01-30,ABIN,63861,0,13,5.48,1,10000067,eec
+10000000224,2012-01-26,Auction,63864,3,14,28.23,1,10000068,deb
+10000000229,2012-11-21,Others,63889,0,13,3.49,1,10000069,bgd
+10000000231,2012-07-12,FP-GTC,67698,2,11,15.65,1,10000070,cgg
+10000000232,2012-06-07,FP-GTC,67698,0,11,5.48,1,10000071,heb
+10000000233,2012-12-01,FP-GTC,67698,0,11,246,1,10000072,jcf
+10000000234,2012-12-02,FP-non GTC,73506,0,13,122.78,1,10000073,djh
+10000000235,2012-02-01,FP-GTC,75665,0,14,223.63,1,10000074,ejc
+10000000236,2012-01-14,ABIN,75708,3,5,141.7,1,10000075,gjh
+10000000237,2012-09-16,FP-non GTC,80053,0,11,21.14,1,10000076,cid
+10000000239,2012-08-09,FP-non GTC,80053,0,11,55.89,1,10000077,ccd
+10000000241,2012-12-26,FP-non GTC,80053,0,11,51.23,1,10000078,bfh
+10000000242,2012-12-31,Auction,80135,0,14,21.72,1,10000079,gcg
+10000000243,2012-03-22,Auction,95672,3,14,204.28,1,10000080,bji
+10000000244,2012-12-25,Others,95672,0,11,21.14,1,10000081,dig
+10000000245,2012-01-15,Others,100847,0,5,204.28,1,10000082,eei
+10000000248,2012-05-27,Others,100847,0,5,122.78,1,10000083,icj
+10000000249,2012-01-11,ABIN,139973,3,14,94.45,1,10000084,jfe
+10000000250,2012-02-04,ABIN,139973,0,11,86.58,1,10000085,efj
+10000000251,2012-12-15,Auction,150047,3,14,56.36,1,10000086,icd
+10000000252,2012-06-30,Auction,150047,3,14,290.72,1,10000087,dci
+10000000253,2012-12-16,FP-GTC,155226,0,13,60.37,1,10000088,ghg
+10000000254,2012-11-11,FP-GTC,155226,0,13,112.56,1,10000089,ebc
+10000000256,2012-10-08,FP-GTC,156356,0,13,265.56,1,10000090,gdh
+10000000257,2012-04-26,FP-GTC,158798,0,11,35.72,1,10000091,cbc
+10000000258,2012-10-06,FP-non GTC,165888,0,13,92.98,1,10000092,gij
+10000000259,2012-12-28,Auction,170083,3,11,28.23,1,10000093,ife
+10000000260,2012-11-06,Auction,170083,3,11,27.48,1,10000094,hgb
+10000000261,2012-11-06,Auction,175750,3,14,9.26,1,10000095,dfa
+10000000262,2012-12-27,Auction,175750,3,14,3.18,1,10000096,jja
+10000000263,2012-01-01,FP-GTC,175750,0,14,12.04,1,10000097,gjf
+10000000265,2012-08-23,Auction,175750,3,13,20.6,1,10000098,bif
+10000000266,2012-07-10,Auction,175750,3,13,12.04,1,10000099,iad
+10000000267,2012-08-10,Auction,175750,3,14,4.13,1,10000100,bfg
+10000000268,2012-07-19,Auction,175750,3,14,73.26,1,10000201,dfg
[50/50] [abbrv] kylin git commit: KYLIN-1672 support kylin on cdh 5.7
Posted by li...@apache.org.
KYLIN-1672 support kylin on cdh 5.7
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/db1b826a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/db1b826a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/db1b826a
Branch: refs/heads/master-cdh5.7
Commit: db1b826ac6b6d086479dc044e88e58b7603c0a95
Parents: 469d9cc
Author: Lynne Jiang <ly...@hotmail.com>
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Dec 7 12:30:13 2016 +0800
----------------------------------------------------------------------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0
.../kylin/engine/mr/steps/MockupMapContext.java | 15 +-
examples/test_case_data/sandbox/core-site.xml | 146 +++---
examples/test_case_data/sandbox/hbase-site.xml | 162 ++----
examples/test_case_data/sandbox/hdfs-site.xml | 259 ++--------
examples/test_case_data/sandbox/mapred-site.xml | 398 ++++++---------
examples/test_case_data/sandbox/yarn-site.xml | 496 ++-----------------
pom.xml | 16 +-
server/pom.xml | 36 ++
.../storage/hbase/steps/MockupMapContext.java | 19 +-
tool/pom.xml | 12 +
11 files changed, 428 insertions(+), 1131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
outKV[0] = key;
outKV[1] = value;
}
+
}
@Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
throw new NotImplementedException();
}
+
@Override
public float getProgress() {
throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
}
@Override
- public RawComparator<?> getSortComparator() {
+ public boolean userClassesTakesPrecedence() {
throw new NotImplementedException();
}
@Override
- public String getJar() {
+ public RawComparator<?> getSortComparator() {
throw new NotImplementedException();
}
@Override
- public RawComparator<?> getGroupingComparator() {
+ public String getJar() {
throw new NotImplementedException();
}
@@ -221,7 +223,7 @@ public class MockupMapContext {
@Override
public boolean getProfileEnabled() {
- throw new NotImplementedException();
+ return false;
}
@Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
public RawComparator<?> getCombinerKeyGroupingComparator() {
throw new NotImplementedException();
}
+
+ @Override
+ public RawComparator<?> getGroupingComparator() {
+ return null;
+ }
});
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
+<!--Autogenerated by Cloudera Manager-->
<configuration>
-
<property>
<name>fs.defaultFS</name>
- <value>hdfs://sandbox.hortonworks.com:8020</value>
- <final>true</final>
+ <value>hdfs://quickstart.cloudera:8020</value>
</property>
-
<property>
<name>fs.trash.interval</name>
- <value>360</value>
+ <value>1</value>
</property>
-
<property>
- <name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
- <value>120</value>
+ <name>io.compression.codecs</name>
+ <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
</property>
-
<property>
- <name>hadoop.http.authentication.simple.anonymous.allowed</name>
- <value>true</value>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
</property>
-
<property>
- <name>hadoop.proxyuser.falcon.groups</name>
- <value>users</value>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value>authentication</value>
+ </property>
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>DEFAULT</value>
</property>
-
<property>
- <name>hadoop.proxyuser.falcon.hosts</name>
+ <name>hadoop.proxyuser.oozie.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hbase.groups</name>
- <value>users</value>
+ <name>hadoop.proxyuser.oozie.groups</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hbase.hosts</name>
+ <name>hadoop.proxyuser.mapred.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hcat.groups</name>
+ <name>hadoop.proxyuser.mapred.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hcat.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <name>hadoop.proxyuser.flume.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hive.groups</name>
- <value>users</value>
+ <name>hadoop.proxyuser.flume.groups</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.HTTP.hosts</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.HTTP.groups</name>
+ <value>*</value>
</property>
-
<property>
<name>hadoop.proxyuser.hive.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hue.groups</name>
+ <name>hadoop.proxyuser.hive.groups</name>
<value>*</value>
</property>
-
<property>
<name>hadoop.proxyuser.hue.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.oozie.groups</name>
+ <name>hadoop.proxyuser.hue.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.oozie.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <name>hadoop.proxyuser.httpfs.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.root.groups</name>
+ <name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.root.hosts</name>
+ <name>hadoop.proxyuser.hdfs.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.security.auth_to_local</name>
- <value>DEFAULT</value>
+ <name>hadoop.proxyuser.hdfs.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.security.authentication</name>
- <value>simple</value>
+ <name>hadoop.proxyuser.yarn.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.security.authorization</name>
- <value>false</value>
+ <name>hadoop.proxyuser.yarn.groups</name>
+ <value>*</value>
</property>
-
<property>
- <name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
+ <name>hadoop.security.group.mapping</name>
+ <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
</property>
-
<property>
- <name>io.file.buffer.size</name>
- <value>131072</value>
+ <name>hadoop.security.instrumentation.requires.admin</name>
+ <value>false</value>
</property>
-
<property>
- <name>io.serializations</name>
- <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
+ <name>net.topology.script.file.name</name>
+ <value>/etc/hadoop/conf.cloudera.yarn/topology.py</value>
</property>
-
<property>
- <name>ipc.client.connect.max.retries</name>
- <value>50</value>
+ <name>io.file.buffer.size</name>
+ <value>65536</value>
</property>
-
<property>
- <name>ipc.client.connection.maxidletime</name>
- <value>30000</value>
+ <name>hadoop.ssl.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.require.client.cert</name>
+ <value>false</value>
+ <final>true</final>
</property>
-
<property>
- <name>ipc.client.idlethreshold</name>
- <value>8000</value>
+ <name>hadoop.ssl.keystores.factory.class</name>
+ <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
+ <final>true</final>
</property>
-
<property>
- <name>ipc.server.tcpnodelay</name>
- <value>true</value>
+ <name>hadoop.ssl.server.conf</name>
+ <value>ssl-server.xml</value>
+ <final>true</final>
</property>
-
<property>
- <name>mapreduce.jobtracker.webinterface.trusted</name>
- <value>false</value>
+ <name>hadoop.ssl.client.conf</name>
+ <value>ssl-client.xml</value>
+ <final>true</final>
</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 734908e..58c6223 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -15,180 +15,104 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
<property>
- <name>hbase.client.keyvalue.maxsize</name>
- <value>10485760</value>
- </property>
-
- <property>
- <name>hbase.client.scanner.caching</name>
- <value>100</value>
+ <name>hbase.rootdir</name>
+ <value>hdfs://quickstart.cloudera:8020/hbase</value>
</property>
-
<property>
- <name>hbase.cluster.distributed</name>
+ <name>hbase.replication</name>
<value>true</value>
</property>
-
- <property>
- <name>hbase.coprocessor.master.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
- </property>
-
<property>
- <name>hbase.coprocessor.region.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
+ <name>hbase.client.write.buffer</name>
+ <value>2097152</value>
</property>
-
- <property>
- <name>hbase.defaults.for.version.skip</name>
- <value>true</value>
- </property>
-
<property>
- <name>hbase.hregion.majorcompaction</name>
- <value>604800000</value>
+ <name>hbase.client.pause</name>
+ <value>100</value>
</property>
-
<property>
- <name>hbase.hregion.majorcompaction.jitter</name>
- <value>0.50</value>
+ <name>hbase.client.retries.number</name>
+ <value>35</value>
</property>
-
<property>
- <name>hbase.hregion.max.filesize</name>
- <value>10737418240</value>
+ <name>hbase.client.scanner.caching</name>
+ <value>100</value>
</property>
-
<property>
- <name>hbase.hregion.memstore.block.multiplier</name>
- <value>4</value>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value>10485760</value>
</property>
-
<property>
- <name>hbase.hregion.memstore.flush.size</name>
- <value>134217728</value>
- </property>
-
- <property>
- <name>hbase.hregion.memstore.mslab.enabled</name>
+ <name>hbase.ipc.client.allowsInterrupt</name>
<value>true</value>
</property>
-
<property>
- <name>hbase.hstore.blockingStoreFiles</name>
+ <name>hbase.client.primaryCallTimeout.get</name>
<value>10</value>
</property>
-
- <property>
- <name>hbase.hstore.compactionThreshold</name>
- <value>3</value>
- </property>
-
<property>
- <name>hbase.local.dir</name>
- <value>${hbase.tmp.dir}/local</value>
+ <name>hbase.client.primaryCallTimeout.multiget</name>
+ <value>10</value>
</property>
-
<property>
- <name>hbase.master.info.bindAddress</name>
- <value>0.0.0.0</value>
+ <name>hbase.regionserver.thrift.http</name>
+ <value>false</value>
</property>
-
<property>
- <name>hbase.master.info.port</name>
- <value>60010</value>
+ <name>hbase.thrift.support.proxyuser</name>
+ <value>false</value>
</property>
-
<property>
- <name>hbase.master.port</name>
+ <name>hbase.rpc.timeout</name>
<value>60000</value>
</property>
-
- <property>
- <name>hbase.regionserver.global.memstore.lowerLimit</name>
- <value>0.38</value>
- </property>
-
<property>
- <name>hbase.regionserver.global.memstore.upperLimit</name>
- <value>0.4</value>
- </property>
-
- <property>
- <name>hbase.regionserver.handler.count</name>
- <value>60</value>
+ <name>hbase.snapshot.enabled</name>
+ <value>true</value>
</property>
-
<property>
- <name>hbase.regionserver.info.port</name>
- <value>60030</value>
+ <name>hbase.snapshot.master.timeoutMillis</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.rootdir</name>
- <value>hdfs://sandbox.hortonworks.com:8020/apps/hbase/data</value>
+ <name>hbase.snapshot.region.timeout</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.rpc.protection</name>
- <value>PRIVACY</value>
+ <name>hbase.snapshot.master.timeout.millis</name>
+ <value>60000</value>
</property>
-
<property>
<name>hbase.security.authentication</name>
<value>simple</value>
</property>
-
<property>
- <name>hbase.security.authorization</name>
- <value>true</value>
+ <name>hbase.rpc.protection</name>
+ <value>authentication</value>
</property>
-
<property>
- <name>hbase.superuser</name>
- <value>hbase</value>
+ <name>zookeeper.session.timeout</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.tmp.dir</name>
- <value>/hadoop/hbase</value>
+ <name>zookeeper.znode.parent</name>
+ <value>/hbase</value>
</property>
-
<property>
- <name>hbase.zookeeper.property.clientPort</name>
- <value>2181</value>
+ <name>zookeeper.znode.rootserver</name>
+ <value>root-region-server</value>
</property>
-
<property>
<name>hbase.zookeeper.quorum</name>
- <value>sandbox.hortonworks.com</value>
+ <value>quickstart.cloudera</value>
</property>
-
<property>
- <name>hbase.zookeeper.useMulti</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hfile.block.cache.size</name>
- <value>0.40</value>
- </property>
-
- <property>
- <name>zookeeper.session.timeout</name>
- <value>30000</value>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>2181</value>
</property>
-
<property>
- <name>zookeeper.znode.parent</name>
- <value>/hbase-unsecure</value>
+ <name>hbase.rest.ssl.enabled</name>
+ <value>false</value>
</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index 1175fff..05854bd 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -15,271 +15,68 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>dfs.block.access.token.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>dfs.block.size</name>
- <value>34217472</value>
- </property>
-
- <property>
- <name>dfs.blockreport.initialDelay</name>
- <value>120</value>
- </property>
-
- <property>
- <name>dfs.blocksize</name>
- <value>134217728</value>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit.streams.cache.size</name>
- <value>4096</value>
- </property>
-
- <property>
- <name>dfs.cluster.administrators</name>
- <value>hdfs</value>
- </property>
-
- <property>
- <name>dfs.datanode.address</name>
- <value>0.0.0.0:50010</value>
- </property>
-
- <property>
- <name>dfs.datanode.balance.bandwidthPerSec</name>
- <value>6250000</value>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir</name>
- <value>/hadoop/hdfs/data</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir.perm</name>
- <value>750</value>
- </property>
-
<property>
- <name>dfs.datanode.du.reserved</name>
- <value>1073741824</value>
- </property>
-
- <property>
- <name>dfs.datanode.failed.volumes.tolerated</name>
- <value>0</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:50075</value>
- </property>
-
- <property>
- <name>dfs.datanode.https.address</name>
- <value>0.0.0.0:50475</value>
- </property>
-
- <property>
- <name>dfs.datanode.ipc.address</name>
- <value>0.0.0.0:8010</value>
- </property>
-
- <property>
- <name>dfs.datanode.max.transfer.threads</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>dfs.datanode.max.xcievers</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
- <property>
- <name>dfs.heartbeat.interval</name>
- <value>3</value>
+ <name>dfs.namenode.name.dir</name>
+ <value>file:///var/lib/hadoop-hdfs/cache/hdfs/dfs/name</value>
</property>
-
<property>
- <name>dfs.hosts.exclude</name>
- <value>/etc/hadoop/conf/dfs.exclude</value>
+ <name>dfs.namenode.servicerpc-address</name>
+ <value>quickstart.cloudera:8022</value>
</property>
-
<property>
- <name>dfs.http.policy</name>
- <value>HTTP_ONLY</value>
+ <name>dfs.https.address</name>
+ <value>quickstart.cloudera:50470</value>
</property>
-
<property>
<name>dfs.https.port</name>
<value>50470</value>
</property>
-
- <property>
- <name>dfs.journalnode.edits.dir</name>
- <value>/hadoop/hdfs/journalnode</value>
- </property>
-
- <property>
- <name>dfs.journalnode.http-address</name>
- <value>0.0.0.0:8480</value>
- </property>
-
- <property>
- <name>dfs.journalnode.https-address</name>
- <value>0.0.0.0:8481</value>
- </property>
-
- <property>
- <name>dfs.namenode.accesstime.precision</name>
- <value>3600000</value>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.read.stale.datanode</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.write.stale.datanode</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.dir</name>
- <value>/hadoop/hdfs/namesecondary</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.edits.dir</name>
- <value>${dfs.namenode.checkpoint.dir}</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.period</name>
- <value>21600</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.txns</name>
- <value>1000000</value>
- </property>
-
- <property>
- <name>dfs.namenode.handler.count</name>
- <value>100</value>
- </property>
-
<property>
<name>dfs.namenode.http-address</name>
- <value>sandbox.hortonworks.com:50070</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.namenode.https-address</name>
- <value>sandbox.hortonworks.com:50470</value>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir</name>
- <value>/hadoop/hdfs/namenode</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir.restore</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.safemode.threshold-pct</name>
- <value>1.0f</value>
+ <value>quickstart.cloudera:50070</value>
</property>
-
<property>
- <name>dfs.namenode.secondary.http-address</name>
- <value>sandbox.hortonworks.com:50090</value>
+ <name>dfs.replication</name>
+ <value>1</value>
</property>
-
<property>
- <name>dfs.namenode.stale.datanode.interval</name>
- <value>30000</value>
+ <name>dfs.blocksize</name>
+ <value>134217728</value>
</property>
-
<property>
- <name>dfs.namenode.startup.delay.block.deletion.sec</name>
- <value>3600</value>
+ <name>dfs.client.use.datanode.hostname</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.namenode.write.stale.datanode.ratio</name>
- <value>1.0f</value>
+ <name>fs.permissions.umask-mode</name>
+ <value>022</value>
</property>
-
<property>
- <name>dfs.nfs.exports.allowed.hosts</name>
- <value>* rw</value>
+ <name>dfs.namenode.acls.enabled</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.nfs3.dump.dir</name>
- <value>/tmp/.hdfs-nfs</value>
+ <name>dfs.client.use.legacy.blockreader</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.permissions.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.permissions.superusergroup</name>
- <value>hdfs</value>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.replication</name>
- <value>1</value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/run/hdfs-sockets/dn</value>
</property>
-
<property>
- <name>dfs.replication.max</name>
- <value>50</value>
+ <name>dfs.client.read.shortcircuit.skip.checksum</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.support.append</name>
- <value>true</value>
- <final>true</final>
+ <name>dfs.client.domain.socket.data.traffic</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.webhdfs.enabled</name>
+ <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
<value>true</value>
- <final>true</final>
</property>
-
- <property>
- <name>fs.permissions.umask-mode</name>
- <value>022</value>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index e90f594..c9b1ca4 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -15,241 +15,165 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>io.sort.mb</name>
- <value>128</value>
- </property>
-
- <property>
- <name>mapred.child.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.admin.map.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>mapreduce.admin.reduce.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
- </property>
-
- <property>
- <name>mapreduce.am.max-attempts</name>
- <value>2</value>
- </property>
-
- <property>
- <name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
- </property>
-
- <property>
- <name>mapreduce.application.framework.path</name>
- <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
- </property>
-
- <property>
- <name>mapreduce.cluster.administrators</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>mapreduce.framework.name</name>
- <value>yarn</value>
- </property>
-
- <property>
- <name>mapreduce.job.emit-timeline-data</name>
- <value>false</value>
- </property>
-
- <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
- <property>
- <name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>1</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.address</name>
- <value>sandbox.hortonworks.com:10020</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.done-dir</name>
- <value>/mr-history/done</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.intermediate-done-dir</name>
- <value>/mr-history/tmp</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.webapp.address</name>
- <value>sandbox.hortonworks.com:19888</value>
- </property>
-
- <property>
- <name>mapreduce.map.java.opts</name>
- <value>-Xmx512m</value>
- </property>
-
- <property>
- <name>mapreduce.map.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.map.output.compress</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.map.sort.spill.percent</name>
- <value>0.7</value>
- </property>
-
- <property>
- <name>mapreduce.map.speculative</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress.type</name>
- <value>BLOCK</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.input.buffer.percent</name>
- <value>0.0</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
- <value>1</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
- <value>0.7</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.merge.percent</name>
- <value>0.66</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.parallelcopies</name>
- <value>30</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.speculative</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.shuffle.port</name>
- <value>13562</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.factor</name>
- <value>100</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.mb</name>
- <value>128</value>
- </property>
-
- <property>
- <name>mapreduce.task.timeout</name>
- <value>300000</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.admin-command-opts</name>
- <value>-Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx512m</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.resource.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.staging-dir</name>
- <value>/user</value>
- </property>
-
+<property>
+ <name>mapreduce.job.split.metainfo.maxsize</name>
+ <value>10000000</value>
+</property>
+<property>
+ <name>mapreduce.job.counters.max</name>
+ <value>120</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.DefaultCodec</value>
+</property>
+<property>
+ <name>mapreduce.map.output.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+</property>
+<property>
+ <name>mapreduce.map.output.compress</name>
+ <value>true</value>
+</property>
+<property>
+ <name>zlib.compress.level</name>
+ <value>DEFAULT_COMPRESSION</value>
+</property>
+<property>
+ <name>mapreduce.task.io.sort.factor</name>
+ <value>64</value>
+</property>
+<property>
+ <name>mapreduce.map.sort.spill.percent</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.reduce.shuffle.parallelcopies</name>
+ <value>10</value>
+</property>
+<property>
+ <name>mapreduce.task.timeout</name>
+ <value>600000</value>
+</property>
+<property>
+ <name>mapreduce.client.submit.file.replication</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.reduces</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.task.io.sort.mb</name>
+ <value>16</value>
+</property>
+<property>
+ <name>mapreduce.map.speculative</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.reduce.speculative</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.job.reduce.slowstart.completedmaps</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.address</name>
+ <value>quickstart.cloudera:10020</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>quickstart.cloudera:19888</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.webapp.https.address</name>
+ <value>quickstart.cloudera:19890</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.admin.address</name>
+ <value>quickstart.cloudera:10033</value>
+</property>
+<property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <value>/user</value>
+</property>
+<property>
+ <name>mapreduce.am.max-attempts</name>
+ <value>2</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.resource.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.ubertask.enable</name>
+ <value>false</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.command-opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>mapreduce.map.java.opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>mapreduce.reduce.java.opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
+</property>
+<property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>mapreduce.map.cpu.vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>mapreduce.reduce.cpu.vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.heap.memory-mb.ratio</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.application.classpath</name>
+ <value>/tmp/kylin/*,/usr/lib/hadoop-mapreduce/lib/*,/etc/hadoop/conf:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-mapreduce/.//*,/usr/lib/hbase/hbase-common.jar,/etc/hive/conf</value>
+</property>
+<property>
+ <name>mapreduce.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
+</property>
+<property>
+ <name>mapreduce.shuffle.max.connections</name>
+ <value>80</value>
+</property>
</configuration>
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index 8256158..8988d4a 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -15,520 +15,128 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>hadoop.registry.rm.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hadoop.registry.zk.quorum</name>
- <value>sandbox.hortonworks.com:2181</value>
- </property>
-
<property>
<name>yarn.acl.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.admin.acl</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.application.classpath</name>
- <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
- </property>
-
- <property>
- <name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.http.policy</name>
- <value>HTTP_ONLY</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation.retain-seconds</name>
- <value>2592000</value>
- </property>
-
- <property>
- <name>yarn.log.server.url</name>
- <value>http://sandbox.hortonworks.com:19888/jobhistory/logs</value>
- </property>
-
- <property>
- <name>yarn.node-labels.fs-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <name>yarn.node-labels.fs-store.root-dir</name>
- <value>/system/yarn/node-labels</value>
- </property>
-
- <property>
- <name>yarn.node-labels.manager-class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.address</name>
- <value>0.0.0.0:45454</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.admin-env</name>
- <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
- <value>org.apache.hadoop.mapred.ShuffleHandler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-executor.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-monitor.interval-ms</name>
- <value>3000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>0</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
- <value>90</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
- <value>0.25</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.interval-ms</name>
- <value>135000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
- <value>hadoop-yarn</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.group</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.local-dirs</name>
- <value>/hadoop/yarn/local</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.compression-type</name>
- <value>gz</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
- <value>30</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
- <value>-1</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-dirs</name>
- <value>/hadoop/yarn/log</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log.retain-second</name>
- <value>604800</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.pmem-check-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.recovery.dir</name>
- <value>/var/log/hadoop-yarn/nodemanager/recovery-state</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.recovery.enabled</name>
<value>true</value>
</property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>/app-logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
- <value>logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.cpu-vcores</name>
- <value>8</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>9216</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- </property>
-
<property>
- <name>yarn.nodemanager.vmem-pmem-ratio</name>
- <value>10</value>
+ <name>yarn.admin.acl</name>
+ <value>*</value>
</property>
-
<property>
<name>yarn.resourcemanager.address</name>
- <value>sandbox.hortonworks.com:8050</value>
+ <value>quickstart.cloudera:8032</value>
</property>
-
<property>
<name>yarn.resourcemanager.admin.address</name>
- <value>sandbox.hortonworks.com:8141</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.am.max-attempts</name>
- <value>2</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.connect.max-wait.ms</name>
- <value>900000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.connect.retry-interval.ms</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.fs.state-store.uri</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.ha.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.hostname</name>
- <value>sandbox.hortonworks.com</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.nodes.exclude-path</name>
- <value>/etc/hadoop/conf/yarn.exclude</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.recovery.enabled</name>
- <value>true</value>
+ <value>quickstart.cloudera:8033</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.resource-tracker.address</name>
- <value>sandbox.hortonworks.com:8025</value>
- </property>
-
<property>
<name>yarn.resourcemanager.scheduler.address</name>
- <value>sandbox.hortonworks.com:8030</value>
+ <value>quickstart.cloudera:8030</value>
</property>
-
<property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.state-store.max-completed-applications</name>
- <value>${yarn.resourcemanager.max-completed-applications}</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.store.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
- <value>10</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
- <value>true</value>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>quickstart.cloudera:8031</value>
</property>
-
<property>
<name>yarn.resourcemanager.webapp.address</name>
- <value>sandbox.hortonworks.com:8088</value>
+ <value>quickstart.cloudera:8088</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
- <value>false</value>
- </property>
-
<property>
<name>yarn.resourcemanager.webapp.https.address</name>
- <value>localhost:8090</value>
+ <value>quickstart.cloudera:8090</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.groups</name>
- <value>*</value>
+ <name>yarn.resourcemanager.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.hosts</name>
- <value>*</value>
+ <name>yarn.resourcemanager.scheduler.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.groups</name>
- <value>*</value>
+ <name>yarn.resourcemanager.admin.client.thread-count</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.hosts</name>
- <value>*</value>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
- <value>true</value>
+ <name>yarn.scheduler.increment-allocation-mb</name>
+ <value>512</value>
</property>
-
<property>
- <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
- <value>10000</value>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>2816</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-acl</name>
- <value>world:anyone:rwcda</value>
+ <name>yarn.scheduler.minimum-allocation-vcores</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-address</name>
- <value>localhost:2181</value>
+ <name>yarn.scheduler.increment-allocation-vcores</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-num-retries</name>
- <value>1000</value>
+ <name>yarn.scheduler.maximum-allocation-vcores</name>
+ <value>2</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-retry-interval-ms</name>
+ <name>yarn.resourcemanager.amliveliness-monitor.interval-ms</name>
<value>1000</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-state-store.parent-path</name>
- <value>/rmstore</value>
+ <name>yarn.am.liveness-monitor.expiry-interval-ms</name>
+ <value>600000</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.zk-timeout-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-mb</name>
- <value>9216</value>
- </property>
-
<property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1536</value>
+ <name>yarn.resourcemanager.am.max-attempts</name>
+ <value>2</value>
</property>
-
<property>
- <name>yarn.timeline-service.address</name>
- <value>sandbox.hortonworks.com:10200</value>
+ <name>yarn.resourcemanager.container.liveness-monitor.interval-ms</name>
+ <value>600000</value>
</property>
-
<property>
- <name>yarn.timeline-service.bind-host</name>
- <value>0.0.0.0</value>
+ <name>yarn.resourcemanager.nm.liveness-monitor.interval-ms</name>
+ <value>1000</value>
</property>
-
<property>
- <name>yarn.timeline-service.client.max-retries</name>
- <value>30</value>
+ <name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
+ <value>600000</value>
</property>
-
<property>
- <name>yarn.timeline-service.client.retry-interval-ms</name>
- <value>1000</value>
+ <name>yarn.resourcemanager.resource-tracker.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.timeline-service.enabled</name>
- <value>true</value>
+ <name>yarn.application.classpath</name>
+ <value>$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*</value>
</property>
-
<property>
- <name>yarn.timeline-service.generic-application-history.store-class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>
-
<property>
- <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+ <name>yarn.scheduler.fair.user-as-default-queue</name>
<value>true</value>
</property>
-
- <property>
- <name>yarn.timeline-service.http-authentication.type</name>
- <value>simple</value>
- </property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.path</name>
- <value>/hadoop/yarn/timeline</value>
+ <name>yarn.scheduler.fair.preemption</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
- <value>104857600</value>
+ <name>yarn.scheduler.fair.sizebasedweight</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
- <value>10000</value>
+ <name>yarn.scheduler.fair.assignmultiple</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
+ <name>yarn.resourcemanager.max-completed-applications</name>
<value>10000</value>
</property>
-
- <property>
- <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
- <value>300000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-ms</name>
- <value>2678400000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.address</name>
- <value>sandbox.hortonworks.com:8188</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.https.address</name>
- <value>sandbox.hortonworks.com:8190</value>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6d3425e..9b857b5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,19 +46,19 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.7.1</hadoop2.version>
- <yarn.version>2.7.1</yarn.version>
+ <hadoop2.version>2.6.0-cdh5.7.0</hadoop2.version>
+ <yarn.version>2.6.0-cdh5.7.0</yarn.version>
<!-- Hive versions -->
- <hive.version>1.2.1</hive.version>
- <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
+ <hive.version>1.1.0-cdh5.7.0</hive.version>
+ <hive-hcatalog.version>1.1.0-cdh5.7.0</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.2.0-cdh5.7.0</hbase-hadoop2.version>
<kafka.version>0.10.0.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
- <zookeeper.version>3.4.6</zookeeper.version>
+ <zookeeper.version>3.4.5-cdh5.7.0</zookeeper.version>
<curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
@@ -814,6 +814,10 @@
<id>conjars</id>
<url>http://conjars.org/repo/</url>
</repository>
+ <repository>
+ <id>cloudera</id>
+ <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+ </repository>
</repositories>
<build>
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index cf92fb1..20f4483 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -112,6 +112,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
@@ -131,6 +135,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -146,6 +154,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -161,6 +173,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -176,6 +192,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -199,6 +219,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -214,6 +238,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -229,6 +257,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -262,6 +294,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
index d5c3f60..5adf327 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
@@ -100,11 +100,6 @@ public class MockupMapContext {
}
@Override
- public float getProgress() {
- throw new NotImplementedException();
- }
-
- @Override
public Counter getCounter(Enum<?> counterName) {
throw new NotImplementedException();
}
@@ -165,6 +160,11 @@ public class MockupMapContext {
}
@Override
+ public boolean userClassesTakesPrecedence() {
+ return false;
+ }
+
+ @Override
public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
throw new NotImplementedException();
}
@@ -214,10 +214,6 @@ public class MockupMapContext {
throw new NotImplementedException();
}
- @Override
- public boolean getTaskCleanupNeeded() {
- throw new NotImplementedException();
- }
@Override
public boolean getProfileEnabled() {
@@ -230,11 +226,6 @@ public class MockupMapContext {
}
@Override
- public IntegerRanges getProfileTaskRange(boolean isMap) {
- throw new NotImplementedException();
- }
-
- @Override
public String getUser() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/db1b826a/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 8eddec4..3d466f0 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -49,6 +49,18 @@
<!--Env-->
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${yarn.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${yarn.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<scope>provided</scope>
[21/50] [abbrv] kylin git commit: KYLIN-2240 Add a toggle to ignore
all cube signature inconsistency temporally
Posted by li...@apache.org.
KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e562aafe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e562aafe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e562aafe
Branch: refs/heads/master-cdh5.7
Commit: e562aafee10c8bbd125937503d42b958c1843aad
Parents: 3091f06
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Dec 2 13:33:02 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Dec 2 13:33:18 2016 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 18 ++++++++++++++++++
.../java/org/apache/kylin/jdbc/IRemoteClient.java | 3 ++-
.../java/org/apache/kylin/jdbc/KylinClient.java | 7 ++++---
.../org/apache/kylin/jdbc/KylinResultSet.java | 9 ++++++++-
.../org/apache/kylin/jdbc/json/QueryRequest.java | 12 ++++++++++++
.../java/org/apache/kylin/jdbc/DummyClient.java | 3 ++-
.../apache/kylin/rest/service/QueryService.java | 11 ++++++++++-
7 files changed, 56 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index f7c90aa..28f7697 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -79,6 +79,14 @@ public class BackdoorToggles {
}
}
+ public static Integer getStatementMaxRows() {
+ String v = getString(ATTR_STATEMENT_MAX_ROWS);
+ if (v == null)
+ return null;
+ else
+ return Integer.valueOf(v);
+ }
+
private static String getString(String key) {
Map<String, String> toggles = _backdoorToggles.get();
if (toggles == null) {
@@ -183,4 +191,14 @@ public class BackdoorToggles {
*/
public final static String DEBUG_TOGGLE_SHARD_ASSIGNMENT = "DEBUG_TOGGLE_SHARD_ASSIGNMENT";
+ // properties on statement may go with this "channel" too
+ /**
+ * set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "ATTR_STATEMENT_MAX_ROWS": "10"
+ }
+ */
+ public final static String ATTR_STATEMENT_MAX_ROWS = "ATTR_STATEMENT_MAX_ROWS";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
index b6a13e5..dfd8d76 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
@@ -21,6 +21,7 @@ package org.apache.kylin.jdbc;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import org.apache.calcite.avatica.AvaticaParameter;
import org.apache.calcite.avatica.ColumnMetaData;
@@ -51,6 +52,6 @@ public interface IRemoteClient extends Closeable {
/**
* Execute query remotely and get back result.
*/
- public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues) throws IOException;
+ public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues, Map<String, String> queryToggles) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
index 2d06a92..86c3a5b 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -321,9 +321,9 @@ public class KylinClient implements IRemoteClient {
}
@Override
- public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues) throws IOException {
+ public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues, Map<String, String> queryToggles) throws IOException {
- SQLResponseStub queryResp = executeKylinQuery(sql, convertParameters(params, paramValues));
+ SQLResponseStub queryResp = executeKylinQuery(sql, convertParameters(params, paramValues), queryToggles);
if (queryResp.getIsException())
throw new IOException(queryResp.getExceptionMessage());
@@ -346,7 +346,7 @@ public class KylinClient implements IRemoteClient {
return result;
}
- private SQLResponseStub executeKylinQuery(String sql, List<StatementParameter> params) throws IOException {
+ private SQLResponseStub executeKylinQuery(String sql, List<StatementParameter> params, Map<String, String> queryToggles) throws IOException {
String url = baseUrl() + "/kylin/api/query";
String project = conn.getProject();
@@ -360,6 +360,7 @@ public class KylinClient implements IRemoteClient {
}
request.setSql(sql);
request.setProject(project);
+ request.setBackdoorToggles(queryToggles);
HttpPost post = new HttpPost(url);
addHttpHeaders(post);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
index 1bf2555..1c1157a 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
@@ -21,7 +21,9 @@ package org.apache.kylin.jdbc;
import java.io.IOException;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.TimeZone;
import org.apache.calcite.avatica.AvaticaParameter;
@@ -57,9 +59,14 @@ public class KylinResultSet extends AvaticaResultSet {
}
IRemoteClient client = ((KylinConnection) statement.connection).getRemoteClient();
+
+ Map<String, String> queryToggles = new HashMap<>();
+ int maxRows = statement.getMaxRows();
+ queryToggles.put("ATTR_STATEMENT_MAX_ROWS", String.valueOf(maxRows));
+
QueryResult result;
try {
- result = client.executeQuery(sql, params, paramValues);
+ result = client.executeQuery(sql, params, paramValues, queryToggles);
} catch (IOException e) {
throw new SQLException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java b/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java
index 0e21e1f..b66ef8a 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/json/QueryRequest.java
@@ -18,11 +18,15 @@
package org.apache.kylin.jdbc.json;
+import java.util.Map;
+
public class QueryRequest {
private String sql;
private String project;
private boolean acceptPartial = false;
+ private Map<String, String> backdoorToggles;
+
public String getSql() {
return sql;
}
@@ -46,4 +50,12 @@ public class QueryRequest {
public void setAcceptPartial(boolean acceptPartial) {
this.acceptPartial = acceptPartial;
}
+
+ public Map<String, String> getBackdoorToggles() {
+ return backdoorToggles;
+ }
+
+ public void setBackdoorToggles(Map<String, String> backdoorToggles) {
+ this.backdoorToggles = backdoorToggles;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
index d85ccae..6578825 100644
--- a/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
+++ b/jdbc/src/test/java/org/apache/kylin/jdbc/DummyClient.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.sql.Types;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import org.apache.calcite.avatica.AvaticaParameter;
import org.apache.calcite.avatica.ColumnMetaData;
@@ -63,7 +64,7 @@ public class DummyClient implements IRemoteClient {
}
@Override
- public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues) throws IOException {
+ public QueryResult executeQuery(String sql, List<AvaticaParameter> params, List<Object> paramValues, Map<String, String> queryToggles) throws IOException {
List<Object> data = new ArrayList<Object>();
Object[] row = new Object[] { "foo", "bar", "tool" };
data.add(row);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e562aafe/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 4389a5d..e1787d7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -324,12 +324,12 @@ public class QueryService extends BasicService {
}
final String queryId = UUID.randomUUID().toString();
-
Map<String, String> toggles = new HashMap<>();
toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId);
if (sqlRequest.getBackdoorToggles() != null) {
toggles.putAll(sqlRequest.getBackdoorToggles());
}
+ sqlRequest.setBackdoorToggles(toggles);
BackdoorToggles.setToggles(toggles);
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
@@ -510,6 +510,13 @@ public class QueryService extends BasicService {
return tableMetas;
}
+ private void processStatementAttr(Statement s, SQLRequest sqlRequest) throws SQLException {
+ Integer statementMaxRows = BackdoorToggles.getStatementMaxRows();
+ if (statementMaxRows != null) {
+ s.setMaxRows(statementMaxRows);
+ }
+ }
+
/**
* @param sql
* @param sqlRequest
@@ -529,6 +536,7 @@ public class QueryService extends BasicService {
if (sqlRequest instanceof PrepareSqlRequest) {
PreparedStatement preparedState = conn.prepareStatement(sql);
+ processStatementAttr(preparedState, sqlRequest);
for (int i = 0; i < ((PrepareSqlRequest) sqlRequest).getParams().length; i++) {
setParam(preparedState, i + 1, ((PrepareSqlRequest) sqlRequest).getParams()[i]);
@@ -537,6 +545,7 @@ public class QueryService extends BasicService {
resultSet = preparedState.executeQuery();
} else {
stat = conn.createStatement();
+ processStatementAttr(stat, sqlRequest);
resultSet = stat.executeQuery(sql);
}
[25/50] [abbrv] kylin git commit: KYLIN-2192 More Robust Global
Dictionary
Posted by li...@apache.org.
KYLIN-2192 More Robust Global Dictionary
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a0ee798
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a0ee798
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a0ee798
Branch: refs/heads/master-cdh5.7
Commit: 4a0ee7989d5f8272592b980fce3f5716ca40d4c1
Parents: e562aaf
Author: sunyerui <su...@gmail.com>
Authored: Mon Nov 21 21:26:34 2016 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Fri Dec 2 13:33:59 2016 +0800
----------------------------------------------------------------------
.../kylin/job/dataGen/FactTableGenerator.java | 12 +-
.../apache/kylin/common/KylinConfigBase.java | 8 +
.../apache/kylin/common/util/Dictionary.java | 2 +-
.../model/validation/rule/DictionaryRule.java | 78 +-
.../validation/rule/DictionaryRuleTest.java | 28 +-
.../apache/kylin/dict/AppendTrieDictionary.java | 285 +++++--
.../kylin/dict/AppendTrieDictionaryChecker.java | 102 +++
.../org/apache/kylin/dict/CachedTreeMap.java | 260 +++---
.../kylin/dict/GlobalDictionaryBuilder.java | 36 +-
.../kylin/dict/AppendTrieDictionaryTest.java | 150 +++-
.../apache/kylin/dict/CachedTreeMapTest.java | 320 +++++---
.../kylin/measure/bitmap/BitmapCounterTest.java | 6 +-
...t_kylin_cube_without_slr_left_join_desc.json | 16 +-
.../localmeta/data/DEFAULT.TEST_KYLIN_FACT.csv | 804 +++++++++----------
.../flatten_data_for_without_slr_left_join.csv | 804 +++++++++----------
.../test_kylin_inner_join_model_desc.json | 3 +-
.../test_kylin_inner_join_view_model_desc.json | 3 +-
.../test_kylin_left_join_model_desc.json | 3 +-
.../test_kylin_left_join_view_model_desc.json | 3 +-
.../table/DEFAULT.TEST_KYLIN_FACT.json | 8 +-
.../source/hive/ITHiveTableReaderTest.java | 2 +-
.../query/sql_distinct_precisely/query00.sql | 2 +-
.../query/sql_distinct_precisely/query01.sql | 2 +-
.../query/sql_distinct_precisely/query02.sql | 2 +-
.../query/sql_distinct_precisely/query03.sql | 3 +-
.../query/sql_distinct_precisely/query04.sql | 3 +-
.../query/sql_distinct_precisely/query05.sql | 25 -
.../query/sql_distinct_precisely/query06.sql | 26 -
.../query/sql_distinct_precisely/query07.sql | 24 -
29 files changed, 1737 insertions(+), 1283 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 8068fd1..677b713 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -403,13 +403,13 @@ public class FactTableGenerator {
}
private String createRandomCell(ColumnDesc cDesc) {
- String type = cDesc.getTypeName();
- String s = type.toLowerCase();
- if (s.equals("string") || s.equals("char") || s.equals("varchar")) {
+ DataType type =cDesc.getType();
+ String s = type.getName();
+ if (s.equals("char") || s.equals("varchar")) {
StringBuilder sb = new StringBuilder();
- for (int i = 0; i < 2; i++) {
- sb.append((char) ('a' + r.nextInt(10)));// there are 10*10
- // possible strings
+ int len = Math.min(type.getPrecision(), 3);
+ for (int i = 0; i < len; i++) {
+ sb.append((char) ('a' + r.nextInt(10))); // cardinality at most 10x10x10
}
return sb.toString();
} else if (s.equals("bigint") || s.equals("int") || s.equals("tinyint") || s.equals("smallint")) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 3c10826..f46c185 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -232,6 +232,14 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "10000000"));
}
+ public int getAppendDictMaxVersions() {
+ return Integer.parseInt(getOptional("kylin.dictionary.append-max-versions", "3"));
+ }
+
+ public int getAppendDictVersionTTL() {
+ return Integer.parseInt(getOptional("kylin.dictionary.append-version-ttl", "259200000"));
+ }
+
// for test
public void setAppendDictEntrySize(int entrySize) {
setProperty("kylin.dictionary.append-entry-size", String.valueOf(entrySize));
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index 0fb299c..1e172bc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -158,7 +158,7 @@ abstract public class Dictionary<T> implements Serializable {
return nullId();
else {
int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
- if (id < 0)
+ if (id == -1)
throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!");
return id;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
index d06c816..37889c2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
@@ -18,9 +18,12 @@
package org.apache.kylin.cube.model.validation.rule;
-import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DictionaryDesc;
import org.apache.kylin.cube.model.validation.IValidatorRule;
@@ -29,9 +32,19 @@ import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.metadata.model.TblColRef;
/**
- * Created by sunyerui on 16/6/1.
+ * Validate Dictionary Settings:
+ *
+ * <ul>
+ * <li> no duplicated dictionary for one column
+ * <li> dictionary can't set both `reuse` and `builder`
+ * <li> transitive `reuse` like "a <- b <- c" is not allowed, force "a <- b, a <- c"
+ * </ul>
*/
public class DictionaryRule implements IValidatorRule<CubeDesc> {
+ static final String ERROR_DUPLICATE_DICTIONARY_COLUMN = "Duplicated dictionary specification for column: ";
+ static final String ERROR_REUSE_BUILDER_BOTH_SET = "REUSE and BUILDER both set on dictionary for column: ";
+ static final String ERROR_REUSE_BUILDER_BOTH_EMPTY = "REUSE and BUILDER both empty on dictionary for column: ";
+ static final String ERROR_TRANSITIVE_REUSE = "Transitive REUSE is not allowed for dictionary: ";
@Override
public void validate(CubeDesc cubeDesc, ValidateContext context) {
@@ -40,40 +53,43 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
return;
}
- HashMap<TblColRef, String> colToBuilderMap = new HashMap<>();
- HashMap<TblColRef, TblColRef> colToReuseColMap = new HashMap<>();
+ Set<TblColRef> allDictCols = new HashSet<>();
+ Set<TblColRef> baseCols = new HashSet<>(); // col with builder
+ List<DictionaryDesc> reuseDictionaries = new ArrayList<>();
+
+ // first pass
for (DictionaryDesc dictDesc : dictDescs) {
TblColRef dictCol = dictDesc.getColumnRef();
- if (dictCol == null) {
- context.addResult(ResultLevel.ERROR, "Some column in dictionaries not found");
+ TblColRef reuseCol = dictDesc.getResuseColumnRef();
+ String builderClass = dictDesc.getBuilderClass();
+
+ if (!allDictCols.add(dictCol)) {
+ context.addResult(ResultLevel.ERROR, ERROR_DUPLICATE_DICTIONARY_COLUMN + dictCol);
return;
}
- String builder = dictDesc.getBuilderClass();
- TblColRef reuseCol = dictDesc.getResuseColumnRef();
- if (reuseCol == null) {
- if (builder == null || builder.isEmpty()) {
- context.addResult(ResultLevel.ERROR, "Column " + dictCol + " cannot have builder and reuse column both empty");
- return;
- }
-
- // Make sure the same column associate with same builder class
- String oldBuilder = colToBuilderMap.put(dictCol, builder);
- if (oldBuilder != null && !oldBuilder.equals(builder)) {
- context.addResult(ResultLevel.ERROR, "Column " + dictCol + " has inconsistent builders " + builder + " and " + oldBuilder);
- return;
- }
+
+ if (reuseCol != null && StringUtils.isNotEmpty(builderClass)) {
+ context.addResult(ResultLevel.ERROR, ERROR_REUSE_BUILDER_BOTH_SET + dictCol);
+ return;
+ }
+
+ if (reuseCol == null && StringUtils.isEmpty(builderClass)) {
+ context.addResult(ResultLevel.ERROR, ERROR_REUSE_BUILDER_BOTH_EMPTY + dictCol);
+ return;
+ }
+
+ if (reuseCol != null) {
+ reuseDictionaries.add(dictDesc);
} else {
- if (builder != null && !builder.isEmpty()) {
- context.addResult(ResultLevel.ERROR, "Column " + dictCol + " cannot have builder and reuse column both");
- return;
- }
-
- // Make sure one column only reuse another one column
- TblColRef oldReuseCol = colToReuseColMap.put(dictCol, reuseCol);
- if (oldReuseCol != null && !reuseCol.equals(oldReuseCol)) {
- context.addResult(ResultLevel.ERROR, "Column " + dictCol + " reuse inconsistent column " + reuseCol + " and " + oldReuseCol);
- return;
- }
+ baseCols.add(dictCol);
+ }
+ }
+
+ // second pass: check no transitive reuse
+ for (DictionaryDesc dictDesc : reuseDictionaries) {
+ if (!baseCols.contains(dictDesc.getResuseColumnRef())) {
+ context.addResult(ResultLevel.ERROR, ERROR_TRANSITIVE_REUSE + dictDesc.getColumnRef());
+ return;
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index 9b37507..b6e0bcb 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -18,6 +18,10 @@
package org.apache.kylin.cube.model.validation.rule;
+import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_DUPLICATE_DICTIONARY_COLUMN;
+import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_EMPTY;
+import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_SET;
+import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_TRANSITIVE_REUSE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -36,9 +40,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-/**
- * Created by sunyerui on 16/6/1.
- */
public class DictionaryRuleTest extends LocalFileMetadataTestCase {
private static KylinConfig config;
@@ -65,24 +66,31 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase {
desc.init(config);
ValidateContext vContext = new ValidateContext();
rule.validate(desc, vContext);
- vContext.print(System.out);
assertTrue(vContext.getResults().length == 0);
}
}
@Test
public void testBadDesc() throws IOException {
- testDictionaryDesc("Column EDW.TEST_SITES.SITE_NAME has inconsistent builders " + "FakeBuilderClass and org.apache.kylin.dict.GlobalDictionaryBuilder", DictionaryDesc.create("SITE_NAME", null, "FakeBuilderClass"));
+ testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("USER_ID", null, "FakeBuilderClass"));
+ testDictionaryDesc(ERROR_DUPLICATE_DICTIONARY_COLUMN, DictionaryDesc.create("USER_ID", null, GlobalDictionaryBuilder.class.getName()));
}
@Test
public void testBadDesc2() throws IOException {
- testDictionaryDesc("Column DEFAULT.TEST_KYLIN_FACT.LSTG_SITE_ID cannot have builder and reuse column both", DictionaryDesc.create("lstg_site_id", "SITE_NAME", "FakeBuilderClass"));
+ testDictionaryDesc(ERROR_REUSE_BUILDER_BOTH_SET, DictionaryDesc.create("lstg_site_id", "SITE_NAME", "FakeBuilderClass"));
}
@Test
public void testBadDesc3() throws IOException {
- testDictionaryDesc("Column DEFAULT.TEST_KYLIN_FACT.LSTG_SITE_ID cannot have builder and reuse column both empty", DictionaryDesc.create("lstg_site_id", null, null));
+ testDictionaryDesc(ERROR_REUSE_BUILDER_BOTH_EMPTY, DictionaryDesc.create("lstg_site_id", null, null));
+ }
+
+ @Test
+ public void testBadDesc4() throws IOException {
+ testDictionaryDesc(ERROR_TRANSITIVE_REUSE,
+ DictionaryDesc.create("lstg_site_id", "USER_ID", null),
+ DictionaryDesc.create("price", "lstg_site_id", null));
}
@Test
@@ -102,13 +110,13 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase {
desc.init(config);
ValidateContext vContext = new ValidateContext();
rule.validate(desc, vContext);
- vContext.print(System.out);
if (expectMessage == null) {
assertTrue(vContext.getResults().length == 0);
} else {
- assertTrue(vContext.getResults().length >= 1);
- assertEquals(expectMessage, vContext.getResults()[0].getMessage());
+ assertTrue(vContext.getResults().length == 1);
+ String actualMessage = vContext.getResults()[0].getMessage();
+ assertTrue(actualMessage.startsWith(expectMessage));
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 14980bf..84060a7 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -31,10 +31,13 @@ import java.lang.ref.SoftReference;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.LinkedList;
import java.util.List;
+import java.util.NavigableSet;
import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -49,6 +52,8 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.MetadataManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,16 +62,16 @@ import org.slf4j.LoggerFactory;
* int IDs, used for global dictionary.
*
* Trie data is split into sub trees, called {@link DictSlice}, and stored in a {@link CachedTreeMap} with a configurable cache size.
- *
+ *
* With Trie the memory footprint of the mapping is kinda minimized at the cost
* CPU, if compared to HashMap of ID Arrays. Performance test shows Trie is
* roughly 10 times slower, so there's a cache layer overlays on top of Trie and
* gracefully fall back to Trie using a weak reference.
- *
+ *
* The implementation is NOT thread-safe for now.
*
* TODO making it thread-safe
- *
+ *
* @author sunyerui
*/
@SuppressWarnings({ "rawtypes", "unchecked", "serial" })
@@ -87,7 +92,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
transient private int nValues;
transient private BytesConverter<T> bytesConverter;
- private TreeMap<DictSliceKey, DictSlice> dictSliceMap;
+ volatile private TreeMap<DictSliceKey, DictSlice> dictSliceMap;
transient private boolean enableValueCache = true;
transient private SoftReference<HashMap> valueToIdCache;
@@ -99,17 +104,23 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
}
}
- public void update(String baseDir, int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter, CachedTreeMap dictMap) throws IOException {
+ public void initParams(String baseDir, int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter) throws IOException {
this.baseDir = baseDir;
this.baseId = baseId;
this.maxId = maxId;
this.maxValueLength = maxValueLength;
this.nValues = nValues;
this.bytesConverter = bytesConverter;
+ }
+ public void initDictSliceMap(CachedTreeMap dictMap) throws IOException {
int cacheSize = KylinConfig.getInstanceFromEnv().getAppendDictCacheSize();
- dictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir).persistent(true).immutable(true).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build();
- ((CachedTreeMap)dictSliceMap).loadEntry(dictMap);
+ int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions();
+ long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL();
+ CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir)
+ .immutable(true).maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build();
+ newDictSliceMap.loadEntry(dictMap);
+ this.dictSliceMap = newDictSliceMap;
}
public byte[] writeDictMap() throws IOException {
@@ -123,6 +134,13 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
return dictMapBytes;
}
+ // The dict id starts from 1 to 2147483647 and 2147483648 to -2, leave 0 and -1 used for uninitialized state
+ public static void checkValidId(int id) {
+ if (id == 0 || id == -1) {
+ throw new IllegalArgumentException("AppendTrieDictionary Id Overflow Unsigned Integer Size 4294967294");
+ }
+ }
+
public static class DictSliceKey implements WritableComparable {
byte[] key;
@@ -181,7 +199,8 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
transient private int nValues;
transient private int sizeOfId;
- transient private int childOffsetMask;
+ // mask MUST be long, since childOffset maybe 5 bytes at most
+ transient private long childOffsetMask;
transient private int firstByteOffset;
private void init(byte[] trieBytes) {
@@ -197,7 +216,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
this.sizeChildOffset = headIn.read();
this.sizeOfId = headIn.read();
- this.childOffsetMask = ~((BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
+ this.childOffsetMask = ~(((long)(BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE)) << ((sizeChildOffset - 1) * 8));
this.firstByteOffset = sizeChildOffset + 1; // the offset from begin of node to its first value byte
} catch (Exception e) {
if (e instanceof RuntimeException)
@@ -216,7 +235,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
if (checkFlag(nodeOffset, BIT_IS_END_OF_VALUE)) {
break;
}
- nodeOffset = headSize + (BytesUtil.readUnsigned(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
+ nodeOffset = headSize + (int)(BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
if (nodeOffset == headSize) {
break;
}
@@ -258,7 +277,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
}
// find a child to continue
- int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ int c = headSize + (int)(BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
if (c == headSize) // has no children
return -1;
byte inpByte = inp[o];
@@ -297,7 +316,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
DictNode root = null;
while (true) {
int p = n + firstByteOffset;
- int childOffset = BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask;
+ int childOffset = (int)(BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
@@ -329,6 +348,53 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
return root;
}
+ public boolean doCheck() {
+ int offset = headSize;
+ HashSet<Integer> parentSet = new HashSet<>();
+ boolean lastChild = false;
+
+ while (offset < trieBytes.length) {
+ if (lastChild) {
+ boolean contained = parentSet.remove(offset - headSize);
+ // Can't find parent, the data is corrupted
+ if (!contained) {
+ return false;
+ }
+ lastChild = false;
+ }
+ int p = offset + firstByteOffset;
+ int childOffset = (int)(BytesUtil.readLong(trieBytes, offset, sizeChildOffset) & childOffsetMask);
+ int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
+ boolean isEndOfValue = checkFlag(offset, BIT_IS_END_OF_VALUE);
+
+ // Copy value overflow, the data is corrupted
+ if (trieBytes.length < p + parLen) {
+ return false;
+ }
+
+ // Check id is fine
+ if (isEndOfValue) {
+ BytesUtil.readUnsigned(trieBytes, p + parLen, sizeOfId);
+ }
+
+ // Record it if has children
+ if (childOffset != 0) {
+ parentSet.add(childOffset);
+ }
+
+ // brothers done, move to next parent
+ if (checkFlag(offset, BIT_IS_LAST_CHILD)) {
+ lastChild = true;
+ }
+
+ // move to next node
+ offset += firstByteOffset + parLen + (isEndOfValue ? sizeOfId : 0);
+ }
+
+ // ParentMap is empty, meaning all nodes has parent, the data is correct
+ return parentSet.isEmpty();
+ }
+
public void write(DataOutput out) throws IOException {
out.write(trieBytes);
}
@@ -341,7 +407,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
throw new IllegalArgumentException("Wrong file type (magic does not match)");
DataInputStream headIn = new DataInputStream(//
- new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
+ new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
int headSize = headIn.readShort();
int bodyLen = headIn.readInt();
headIn.close();
@@ -398,6 +464,9 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
this.id = o.id;
this.isEndOfValue = o.isEndOfValue;
this.children = o.children;
+ for (DictNode child : o.children) {
+ child.parent = this;
+ }
this.nValuesBeneath = o.nValuesBeneath;
this.parent = o.parent;
this.childrenCount = o.childrenCount;
@@ -602,7 +671,8 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
// nValueBytes
if (n.part.length > 255)
- throw new RuntimeException();
+ throw new RuntimeException("Value length is " + n.part.length
+ + " and larger than 255: " + Bytes.toStringBinary(n.part));
BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
o++;
@@ -611,7 +681,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
o += n.part.length;
if (n.isEndOfValue) {
- assert n.id > 0;
+ checkValidId(n.id);
BytesUtil.writeUnsigned(n.id, trieBytes, o, sizeId);
o += sizeId;
}
@@ -715,12 +785,13 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
s.mbpn_sizeId = 4;
s.mbpn_sizeValueTotal = s.nValueBytesCompressed + s.nValues * s.mbpn_sizeId;
s.mbpn_sizeNoValueBytes = 1;
- s.mbpn_sizeChildOffset = 4;
+ s.mbpn_sizeChildOffset = 5;
s.mbpn_footprint = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeChildOffset);
while (true) { // minimize the offset size to match the footprint
int t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeChildOffset - 1);
// *4 because 2 MSB of offset is used for isEndOfValue & isEndChild flag
- if (BytesUtil.sizeForValue(t * 4) <= s.mbpn_sizeChildOffset - 1) {
+ // expand t to long before *4, avoiding exceed Integer.MAX_VALUE
+ if (BytesUtil.sizeForValue((long)t * 4) <= s.mbpn_sizeChildOffset - 1) {
s.mbpn_sizeChildOffset--;
s.mbpn_footprint = t;
} else
@@ -760,31 +831,97 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
}
public static class Builder<T> {
- private String baseDir;
+ private static ConcurrentHashMap<String, Pair<Integer, Builder>> builderInstanceAndCountMap = new ConcurrentHashMap();
+
+ public static Builder getInstance(String resourcePath) throws IOException {
+ return getInstance(resourcePath, null);
+ }
+
+ public synchronized static Builder getInstance(String resourcePath, AppendTrieDictionary dict) throws IOException {
+ Pair<Integer, Builder> entry = builderInstanceAndCountMap.get(resourcePath);
+ if (entry == null) {
+ entry = new Pair<>(0, createNewBuilder(resourcePath, dict));
+ builderInstanceAndCountMap.put(resourcePath, entry);
+ }
+ entry.setFirst(entry.getFirst() + 1);
+ return entry.getSecond();
+ }
+
+ // return true if entry still in map
+ private synchronized static boolean releaseInstance(String resourcePath) {
+ Pair<Integer, Builder> entry = builderInstanceAndCountMap.get(resourcePath);
+ if (entry != null) {
+ entry.setFirst(entry.getFirst() - 1);
+ if (entry.getFirst() <= 0) {
+ builderInstanceAndCountMap.remove(resourcePath);
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ public static Builder createNewBuilder(String resourcePath, AppendTrieDictionary existDict) throws IOException {
+ String dictDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict" + resourcePath + "/";
+
+ AppendTrieDictionary dictToUse = existDict;
+ if (dictToUse == null) {
+ // Try to load the existing dict from cache, making sure there's only the same one object in memory
+ NavigableSet<String> dicts = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getStore().listResources(resourcePath);
+ ArrayList<String> appendDicts = new ArrayList<>();
+ if (dicts != null && !dicts.isEmpty()) {
+ for (String dict : dicts) {
+ DictionaryInfo info = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getStore().getResource(dict, DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
+ if (info.getDictionaryClass().equals(AppendTrieDictionary.class.getName())) {
+ appendDicts.add(dict);
+ }
+ }
+ }
+ if (appendDicts.isEmpty()) {
+ dictToUse = null;
+ } else if (appendDicts.size() == 1) {
+ dictToUse = (AppendTrieDictionary) DictionaryManager.getInstance(KylinConfig.getInstanceFromEnv()).getDictionary(appendDicts.get(0));
+ } else {
+ throw new IllegalStateException(String.format("GlobalDict %s should have 0 or 1 append dict but %d", resourcePath, appendDicts.size()));
+ }
+ }
+
+ AppendTrieDictionary.Builder<String> builder;
+ if (dictToUse == null) {
+ logger.info("GlobalDict {} is empty, create new one", resourcePath);
+ builder = new Builder<>(resourcePath, null, dictDir, 0, 0, 0, new StringBytesConverter(), null);
+ } else {
+ logger.info("GlobalDict {} exist, append value", dictToUse);
+ builder = new Builder<>(resourcePath, dictToUse, dictToUse.baseDir, dictToUse.maxId, dictToUse.maxValueLength,
+ dictToUse.nValues, dictToUse.bytesConverter, dictToUse.writeDictMap());
+ }
+
+ return builder;
+ }
+
+ private final String resourcePath;
+ private final String baseDir;
private int maxId;
private int maxValueLength;
private int nValues;
- private BytesConverter<T> bytesConverter;
+ private final BytesConverter<T> bytesConverter;
- private AppendTrieDictionary dict;
+ private final AppendTrieDictionary dict;
- private TreeMap<DictSliceKey, DictNode> mutableDictSliceMap;
- private static int MAX_ENTRY_IN_SLICE = 10_000_000;
+ private final TreeMap<DictSliceKey, DictNode> mutableDictSliceMap;
+ private int MAX_ENTRY_IN_SLICE = 10_000_000;
private static final double MAX_ENTRY_OVERHEAD_FACTOR = 1.0;
private int processedCount = 0;
- public static Builder create(String baseDir) throws IOException {
- return new Builder<>(null, baseDir, 0, 0, 0, new StringBytesConverter(), null);
- }
-
- public static Builder create(AppendTrieDictionary dict) throws IOException {
- return new Builder<>(dict, dict.baseDir, dict.maxId, dict.maxValueLength, dict.nValues, dict.bytesConverter, dict.writeDictMap());
- }
-
// Constructor for a new Dict
- private Builder(AppendTrieDictionary dict, String baseDir, int maxId, int maxValueLength, int nValues, BytesConverter<T> bytesConverter, byte[] dictMapBytes) throws IOException {
- this.dict = dict;
+ private Builder(String resourcePath, AppendTrieDictionary dict, String baseDir, int maxId, int maxValueLength, int nValues, BytesConverter<T> bytesConverter, byte[] dictMapBytes) throws IOException {
+ this.resourcePath = resourcePath;
+ if (dict == null) {
+ this.dict = new AppendTrieDictionary<T>();
+ } else {
+ this.dict = dict;
+ }
this.baseDir = baseDir;
this.maxId = maxId;
this.maxValueLength = maxValueLength;
@@ -793,8 +930,11 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
MAX_ENTRY_IN_SLICE = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
int cacheSize = KylinConfig.getInstanceFromEnv().getAppendDictCacheSize();
+ int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions();
+ long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL();
// create a new cached map with baseDir
- mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir).keyClazz(DictSliceKey.class).valueClazz(DictNode.class).persistent(true).immutable(false).build();
+ mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir)
+ .maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictNode.class).immutable(false).build();
if (dictMapBytes != null) {
((Writable) mutableDictSliceMap).readFields(new DataInputStream(new ByteArrayInputStream(dictMapBytes)));
}
@@ -804,7 +944,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
addValue(bytesConverter.convertToBytes(value));
}
- public void addValue(byte[] value) {
+ private synchronized void addValue(byte[] value) {
if (++processedCount % 1_000_000 == 0) {
logger.debug("add value count " + processedCount);
}
@@ -859,15 +999,41 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
private int createNextId() {
int id = ++maxId;
- if (maxId < 0) {
- throw new IllegalArgumentException("AppendTrieDictionary Id overflow Integer.MAX_VALUE");
- }
+ checkValidId(id);
nValues++;
return id;
}
+ // Only used for test
+ public void setMaxId(int id) {
+ this.maxId = id;
+ }
+
+ // When add a new node, the value part maybe over 255 bytes, need split it into a sub tree
+ private DictNode addNodeMaybeOverflow(byte[] value, int start, int end) {
+ DictNode head = null;
+ DictNode current = null;
+ for (; start + 255 < end; start += 255) {
+ DictNode c = new DictNode(BytesUtil.subarray(value, start, start + 255), false);
+ if (head == null) {
+ head = c;
+ current = c;
+ } else {
+ current.addChild(c);
+ current = c;
+ }
+ }
+ DictNode last = new DictNode(BytesUtil.subarray(value, start, end), true);
+ last.id = createNextId();
+ if (head == null) {
+ head = last;
+ } else {
+ current.addChild(last);
+ }
+ return head;
+ }
+
private void addValueR(DictNode node, byte[] value, int start) {
- assert value.length - start <= 255 : "value bytes overflow than 255";
// match the value part of current node
int i = 0, j = start;
int n = node.part.length, nn = value.length;
@@ -903,8 +1069,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
if (i < n) {
DictNode c1 = new DictNode(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children);
c1.id = node.id;
- DictNode c2 = new DictNode(BytesUtil.subarray(value, j, nn), true);
- c2.id = createNextId();
+ DictNode c2 = addNodeMaybeOverflow(value, j, nn);
node.reset(BytesUtil.subarray(node.part, 0, i), false);
if (comp < 0) {
node.addChild(c1);
@@ -940,18 +1105,17 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
addValueR(node.children.get(mid), value, j);
} else {
// otherwise, make the value a new child
- DictNode c = new DictNode(BytesUtil.subarray(value, j, nn), true);
- c.id = createNextId();
+ DictNode c = addNodeMaybeOverflow(value, j, nn);
node.addChild(comp <= 0 ? mid : mid + 1, c);
}
}
- public AppendTrieDictionary<T> build(int baseId) throws IOException {
- if (dict == null) {
- dict = new AppendTrieDictionary<T>();
- }
- dict.update(baseDir, baseId, maxId, maxValueLength, nValues, bytesConverter, (CachedTreeMap)mutableDictSliceMap);
- dict.flushIndex((CachedTreeMap) mutableDictSliceMap);
+ public synchronized AppendTrieDictionary<T> build(int baseId) throws IOException {
+ boolean keepAppend = releaseInstance(resourcePath);
+ CachedTreeMap dictSliceMap = (CachedTreeMap)mutableDictSliceMap;
+ dict.initParams(baseDir, baseId, maxId, maxValueLength, nValues, bytesConverter);
+ dict.flushIndex(dictSliceMap, keepAppend);
+ dict.initDictSliceMap(dictSliceMap);
return dict;
}
@@ -970,8 +1134,6 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
}
DictSlice slice = dictSliceMap.get(sliceKey);
int id = slice.getIdFromValueBytesImpl(value, offset, len, roundingFlag);
- if (id < 0)
- logger.error("Not a valid value: " + bytesConverter.convertFromBytes(value, offset, len));
return id;
}
@@ -1031,25 +1193,24 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
throw new UnsupportedOperationException("AppendTrieDictionary can't retrive value from id");
}
- public void flushIndex(CachedTreeMap dictSliceMap) throws IOException {
- Path filePath = new Path(dictSliceMap.getCurrentDir() + "/.index");
- Configuration conf = new Configuration();
- try (FSDataOutputStream indexOut = (FileSystem.get(filePath.toUri(), conf)).create(filePath, true, 8 * 1024 * 1024, (short) 5, 8 * 1024 * 1024 * 8)) {
+ public void flushIndex(CachedTreeMap dictSliceMap, boolean keepAppend) throws IOException {
+ try (FSDataOutputStream indexOut = dictSliceMap.openIndexOutput()) {
indexOut.writeInt(baseId);
indexOut.writeInt(maxId);
indexOut.writeInt(maxValueLength);
indexOut.writeInt(nValues);
indexOut.writeUTF(bytesConverter.getClass().getName());
dictSliceMap.write(indexOut);
+ dictSliceMap.commit(keepAppend);
}
- dictSliceMap.commit(false);
}
@Override
public AppendTrieDictionary copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
Configuration conf = new Configuration();
AppendTrieDictionary newDict = new AppendTrieDictionary();
- newDict.update(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues, bytesConverter, (CachedTreeMap)dictSliceMap);
+ newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues, bytesConverter);
+ newDict.initDictSliceMap((CachedTreeMap)dictSliceMap);
logger.info("Copy AppendDict from {} to {}", this.baseDir, newDict.baseDir);
Path srcPath = new Path(this.baseDir);
Path dstPath = new Path(newDict.baseDir);
@@ -1071,9 +1232,8 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
@Override
public void readFields(DataInput in) throws IOException {
String baseDir = in.readUTF();
- Path filePath = new Path(baseDir + "/.index");
Configuration conf = new Configuration();
- try (FSDataInputStream input = (FileSystem.get(filePath.toUri(), conf)).open(filePath, 8 * 1024 * 1024)) {
+ try (FSDataInputStream input = CachedTreeMap.openLatestIndexInput(conf, baseDir)) {
int baseId = input.readInt();
int maxId = input.readInt();
int maxValueLength = input.readInt();
@@ -1087,10 +1247,13 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
throw new IOException(e);
}
}
+ initParams(baseDir, baseId, maxId, maxValueLength, nValues, converter);
+
+ // Create instance for deserialize data, and update to map in dict
CachedTreeMap dictMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder()
- .baseDir(baseDir).persistent(true).immutable(true).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build();
+ .baseDir(baseDir).immutable(true).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build();
dictMap.readFields(input);
- update(baseDir, baseId, maxId, maxValueLength, nValues, converter, dictMap);
+ initDictSliceMap(dictMap);
}
}
@@ -1120,4 +1283,6 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
public boolean contains(Dictionary other) {
return false;
}
+
}
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionaryChecker.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionaryChecker.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionaryChecker.java
new file mode 100644
index 0000000..f231275
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionaryChecker.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.dict;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by sunyerui on 16/11/15.
+ */
+public class AppendTrieDictionaryChecker {
+
+ public boolean runChecker(String baseDir) throws IOException {
+ Configuration conf = new Configuration();
+ Path basePath = new Path(baseDir);
+ FileSystem fs = FileSystem.get(basePath.toUri(), conf);
+ List<Path> sliceList = new ArrayList<>();
+ List<Path> corruptedSliceList = new ArrayList<>();
+ listDictSlicePath(fs, fs.getFileStatus(basePath), sliceList);
+
+ for (Path path : sliceList) {
+ if (!doCheck(fs, path)) {
+ System.out.println("AppendDict Slice " + path + " corrupted");
+ corruptedSliceList.add(path);
+ } else {
+ System.out.println("AppendDict Slice " + path + " is right");
+ }
+ }
+
+ if (corruptedSliceList.isEmpty()) {
+ System.out.println("ALL AppendDict Slices is right");
+ return true;
+ } else {
+ System.out.println("Some AppendDict Slice(s) corrupted: ");
+ for (Path path : corruptedSliceList) {
+ System.out.println(path.toString());
+ }
+ return false;
+ }
+ }
+
+ public void listDictSlicePath(FileSystem fs, FileStatus path, List<Path> list) throws IOException {
+ if (path.isDirectory()) {
+ for (FileStatus status : fs.listStatus(path.getPath())) {
+ listDictSlicePath(fs, status, list);
+ }
+ } else {
+ if (path.getPath().getName().startsWith(CachedTreeMap.CACHED_PREFIX)) {
+ list.add(path.getPath());
+ }
+ }
+ }
+
+ public boolean doCheck(FileSystem fs, Path filePath) {
+ try (FSDataInputStream input = fs.open(filePath, CachedTreeMap.BUFFER_SIZE)) {
+ AppendTrieDictionary.DictSlice slice = new AppendTrieDictionary.DictSlice();
+ slice.readFields(input);
+ return slice.doCheck();
+ } catch (Exception e) {
+ return false;
+ } catch (Error e) {
+ return false;
+ }
+ }
+
+ public static void main(String[] args) throws IOException {
+ String path = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict/";
+ if (args.length > 0) {
+ path = args[0];
+ }
+ System.out.println("Recursive Check AppendTrieDictionary Slices in path " + path);
+ AppendTrieDictionaryChecker checker = new AppendTrieDictionaryChecker();
+ if (checker.runChecker(path)) {
+ System.exit(0);
+ } else {
+ System.exit(-1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
index 1ea3c1c..6acf764 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java
@@ -31,9 +31,11 @@ import java.util.concurrent.ExecutionException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.slf4j.Logger;
@@ -56,25 +58,29 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
private final Class<V> valueClazz;
transient volatile Collection<V> values;
private final LoadingCache<K, V> valueCache;
- private final TreeSet<String> fileList;
private final Configuration conf;
- private final String baseDir;
- private final String tmpDir;
+ private final Path baseDir;
+ private final Path versionDir;
+ private final Path workingDir;
private final FileSystem fs;
- private final boolean persistent;
private final boolean immutable;
- private long writeValueTime = 0;
- private long readValueTime = 0;
+ private final int maxVersions;
+ private final long versionTTL;
+ private boolean keepAppend;
- private static final int BUFFER_SIZE = 8 * 1024 * 1024;
+ public static final int BUFFER_SIZE = 8 * 1024 * 1024;
+
+ public static final String CACHED_PREFIX = "cached_";
+ public static final String VERSION_PREFIX = "version_";
public static class CachedTreeMapBuilder<K, V> {
private Class<K> keyClazz;
private Class<V> valueClazz;
private int maxCount = 8;
private String baseDir;
- private boolean persistent;
private boolean immutable;
+ private int maxVersions;
+ private long versionTTL;
public static CachedTreeMapBuilder newBuilder() {
return new CachedTreeMapBuilder();
@@ -103,13 +109,18 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
return this;
}
- public CachedTreeMapBuilder<K, V> persistent(boolean persistent) {
- this.persistent = persistent;
+ public CachedTreeMapBuilder<K, V> immutable(boolean immutable) {
+ this.immutable = immutable;
return this;
}
- public CachedTreeMapBuilder<K, V> immutable(boolean immutable) {
- this.immutable = immutable;
+ public CachedTreeMapBuilder<K, V> maxVersions(int maxVersions) {
+ this.maxVersions = maxVersions;
+ return this;
+ }
+
+ public CachedTreeMapBuilder<K, V> versionTTL(long versionTTL) {
+ this.versionTTL = versionTTL;
return this;
}
@@ -120,26 +131,38 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
if (keyClazz == null || valueClazz == null) {
throw new RuntimeException("CachedTreeMap need key and value clazz to serialize data");
}
- CachedTreeMap map = new CachedTreeMap(maxCount, keyClazz, valueClazz, baseDir, persistent, immutable);
+ CachedTreeMap map = new CachedTreeMap(maxCount, keyClazz, valueClazz, baseDir, immutable, maxVersions, versionTTL);
return map;
}
}
- private CachedTreeMap(int maxCount, Class<K> keyClazz, Class<V> valueClazz, String baseDir, boolean persistent, boolean immutable) throws IOException {
+ private CachedTreeMap(int maxCount, Class<K> keyClazz, Class<V> valueClazz, String basePath,
+ boolean immutable, int maxVersions, long versionTTL) throws IOException {
super();
this.keyClazz = keyClazz;
this.valueClazz = valueClazz;
- this.fileList = new TreeSet<>();
+ this.immutable = immutable;
+ this.keepAppend = true;
+ this.maxVersions = maxVersions;
+ this.versionTTL = versionTTL;
this.conf = new Configuration();
- if (baseDir.endsWith("/")) {
- this.baseDir = baseDir.substring(0, baseDir.length()-1);
- } else {
- this.baseDir = baseDir;
+ if (basePath.endsWith("/")) {
+ basePath = basePath.substring(0, basePath.length()-1);
+ }
+ this.baseDir = new Path(basePath);
+ this.fs = FileSystem.get(baseDir.toUri(), conf);
+ if (!fs.exists(baseDir)) {
+ fs.mkdirs(baseDir);
+ }
+ this.versionDir = getLatestVersion(conf, fs, baseDir);
+ this.workingDir = new Path(baseDir, "working");
+ if (!this.immutable) {
+ // For mutable map, copy all data into working dir and work on it, avoiding suddenly server crash made data corrupt
+ if (fs.exists(workingDir)) {
+ fs.delete(workingDir, true);
+ }
+ FileUtil.copy(fs, versionDir, fs, workingDir, false, true, conf);
}
- this.tmpDir = this.baseDir + ".tmp";
- this.fs = FileSystem.get(new Path(baseDir).toUri(), conf);
- this.persistent = persistent;
- this.immutable = immutable;
CacheBuilder builder = CacheBuilder.newBuilder().removalListener(new RemovalListener<K, V>() {
@Override
public void onRemoval(RemovalNotification<K, V> notification) {
@@ -152,24 +175,14 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
deleteValue(notification.getKey());
break;
default:
- throw new RuntimeException("unexpected evict reason " + notification.getCause());
}
}
});
- // For immutable values, load all values as much as possible, and evict by soft reference to free memory when gc
if (this.immutable) {
+ // For immutable values, load all values as much as possible, and evict by soft reference to free memory when gc
builder.softValues();
} else {
builder.maximumSize(maxCount);
- // For mutable map, copy all data into tmp and modify on tmp data, avoiding suddenly server crash made data corrupt
- if (fs.exists(new Path(tmpDir))) {
- fs.delete(new Path(tmpDir), true);
- }
- if (fs.exists(new Path(this.baseDir))) {
- FileUtil.copy(fs, new Path(this.baseDir), fs, new Path(tmpDir), false, true, conf);
- } else {
- fs.mkdirs(new Path(this.baseDir));
- }
}
this.valueCache = builder.build(new CacheLoader<K, V>() {
@Override
@@ -182,38 +195,108 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
}
private String generateFileName(K key) {
- String file = (immutable ? baseDir : tmpDir) + "/cached_" + key.toString();
+ String file = getCurrentDir() + "/" + CACHED_PREFIX + key.toString();
return file;
}
- public String getCurrentDir() {
- return immutable ? baseDir : tmpDir;
+ private String getCurrentDir() {
+ return immutable ? versionDir.toString() : workingDir.toString();
}
- public void commit(boolean stillMutable) throws IOException {
- assert !immutable : "Only support commit method with immutable false";
+ private static String[] listAllVersions(FileSystem fs, Path baseDir) throws IOException {
+ FileStatus[] fileStatus = fs.listStatus(baseDir, new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ if (path.getName().startsWith(VERSION_PREFIX)) {
+ return true;
+ }
+ return false;
+ }
+ });
+ TreeSet<String> versions = new TreeSet<>();
+ for (FileStatus status : fileStatus) {
+ versions.add(status.getPath().toString());
+ }
+ return versions.toArray(new String[versions.size()]);
+ }
- Path basePath = new Path(baseDir);
- Path backupPath = new Path(baseDir+".bak");
- Path tmpPath = new Path(tmpDir);
- try {
- fs.rename(basePath, backupPath);
- } catch (IOException e) {
- logger.info("CachedTreeMap commit backup basedir failed, " + e, e);
- throw e;
+ // only for test
+ public String getLatestVersion() throws IOException {
+ return getLatestVersion(conf, fs, baseDir).toUri().getPath();
+ }
+
+ private static Path getLatestVersion(Configuration conf, FileSystem fs, Path baseDir) throws IOException {
+ String[] versions = listAllVersions(fs, baseDir);
+ if (versions.length > 0) {
+ return new Path(versions[versions.length - 1]);
+ } else {
+ // Old format, directly use base dir, convert to new format
+ Path newVersionDir = new Path(baseDir, VERSION_PREFIX + System.currentTimeMillis());
+ Path tmpNewVersionDir = new Path(baseDir, "tmp_" + VERSION_PREFIX + System.currentTimeMillis());
+ Path indexFile = new Path(baseDir, ".index");
+ FileStatus[] cachedFiles;
+ try {
+ cachedFiles = fs.listStatus(baseDir, new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ if (path.getName().startsWith(CACHED_PREFIX)) {
+ return true;
+ }
+ return false;
+ }
+ });
+ fs.mkdirs(tmpNewVersionDir);
+ if (fs.exists(indexFile) && cachedFiles.length > 0) {
+ FileUtil.copy(fs, indexFile, fs, tmpNewVersionDir, false, true, conf);
+ for (FileStatus file : cachedFiles) {
+ FileUtil.copy(fs, file.getPath(), fs, tmpNewVersionDir, false, true, conf);
+ }
+ }
+ fs.rename(tmpNewVersionDir, newVersionDir);
+ if (fs.exists(indexFile) && cachedFiles.length > 0) {
+ fs.delete(indexFile, true);
+ for (FileStatus file : cachedFiles) {
+ fs.delete(file.getPath(), true);
+ }
+ }
+ } finally {
+ if (fs.exists(tmpNewVersionDir)) {
+ fs.delete(tmpNewVersionDir, true);
+ }
+ }
+ return newVersionDir;
}
+ }
- try {
- if (stillMutable) {
- FileUtil.copy(fs, tmpPath, fs, basePath, false, true, conf);
- } else {
- fs.rename(tmpPath, basePath);
+ public void commit(boolean keepAppend) throws IOException {
+ assert this.keepAppend & !immutable : "Only support commit method with immutable false and keepAppend true";
+
+ Path newVersionDir = new Path(baseDir, VERSION_PREFIX + System.currentTimeMillis());
+ if (keepAppend) {
+ // Copy to tmp dir, and rename to new version, make sure it's complete when be visible
+ Path tmpNewVersionDir = new Path(baseDir, "tmp_" + VERSION_PREFIX + System.currentTimeMillis());
+ try {
+ FileUtil.copy(fs, workingDir, fs, tmpNewVersionDir, false, true, conf);
+ fs.rename(tmpNewVersionDir, newVersionDir);
+ } finally {
+ if (fs.exists(tmpNewVersionDir)) {
+ fs.delete(tmpNewVersionDir, true);
+ }
+ }
+ } else {
+ fs.rename(workingDir, newVersionDir);
+ }
+ this.keepAppend = keepAppend;
+
+ // Check versions count, delete expired versions
+ String[] versions = listAllVersions(fs, baseDir);
+ long timestamp = System.currentTimeMillis();
+ for (int i = 0; i < versions.length - maxVersions; i++) {
+ String versionString = versions[i].substring(versions[i].lastIndexOf(VERSION_PREFIX) + VERSION_PREFIX.length());
+ long version = Long.parseLong(versionString);
+ if (version + versionTTL < timestamp) {
+ fs.delete(new Path(versions[i]), true);
}
- fs.delete(backupPath, true);
- } catch (IOException e) {
- fs.rename(backupPath, basePath);
- logger.info("CachedTreeMap commit move/copy tmpdir failed, " + e, e);
- throw e;
}
}
@@ -227,25 +310,17 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
if (immutable) {
return;
}
- long t0 = System.currentTimeMillis();
String fileName = generateFileName(key);
Path filePath = new Path(fileName);
try (FSDataOutputStream out = fs.create(filePath, true, BUFFER_SIZE, (short) 5, BUFFER_SIZE * 8)) {
value.write(out);
- if (!persistent) {
- fs.deleteOnExit(filePath);
- }
} catch (Exception e) {
logger.error(String.format("write value into %s exception: %s", fileName, e), e);
throw new RuntimeException(e.getCause());
- } finally {
- fileList.add(fileName);
- writeValueTime += System.currentTimeMillis() - t0;
}
}
private V readValue(K key) throws Exception {
- long t0 = System.currentTimeMillis();
String fileName = generateFileName(key);
Path filePath = new Path(fileName);
try (FSDataInputStream input = fs.open(filePath, BUFFER_SIZE)) {
@@ -255,13 +330,11 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
} catch (Exception e) {
logger.error(String.format("read value from %s exception: %s", fileName, e), e);
return null;
- } finally {
- readValueTime += System.currentTimeMillis() - t0;
}
}
private void deleteValue(K key) {
- if (persistent && immutable) {
+ if (immutable) {
return;
}
String fileName = generateFileName(key);
@@ -272,14 +345,12 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
}
} catch (Exception e) {
logger.error(String.format("delete value file %s exception: %s", fileName, e), e);
- } finally {
- fileList.remove(fileName);
}
}
@Override
public V put(K key, V value) {
- assert !immutable : "Only support put method with immutable false";
+ assert keepAppend & !immutable : "Only support put method with immutable false and keepAppend true";
super.put(key, null);
valueCache.put(key, value);
return null;
@@ -301,7 +372,7 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
@Override
public V remove(Object key) {
- assert !immutable : "Only support remove method with immutable false";
+ assert keepAppend & !immutable : "Only support remove method with immutable false keepAppend true";
super.remove(key);
valueCache.invalidate(key);
return null;
@@ -357,15 +428,32 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
@Override
public void remove() {
- assert !immutable : "Only support remove method with immutable false";
+ assert keepAppend & !immutable : "Only support remove method with immutable false and keepAppend true";
keyIterator.remove();
valueCache.invalidate(currentKey);
}
}
+ public FSDataOutputStream openIndexOutput() throws IOException {
+ assert keepAppend & !immutable : "Only support write method with immutable false and keepAppend true";
+ Path indexPath = new Path(getCurrentDir(), ".index");
+ return fs.create(indexPath, true, 8 * 1024 * 1024, (short) 5, 8 * 1024 * 1024 * 8);
+ }
+
+ public FSDataInputStream openIndexInput() throws IOException {
+ Path indexPath = new Path(getCurrentDir(), ".index");
+ return fs.open(indexPath, 8 * 1024 * 1024);
+ }
+
+ public static FSDataInputStream openLatestIndexInput(Configuration conf, String baseDir) throws IOException {
+ Path basePath = new Path(baseDir);
+ FileSystem fs = FileSystem.get(basePath.toUri(), conf);
+ Path indexPath = new Path(getLatestVersion(conf, fs, basePath), ".index");
+ return fs.open(indexPath, 8 * 1024 * 1024);
+ }
+
@Override
public void write(DataOutput out) throws IOException {
- assert persistent : "Only support serialize with persistent true";
out.writeInt(size());
for (K key : keySet()) {
key.write(out);
@@ -378,7 +466,6 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
@Override
public void readFields(DataInput in) throws IOException {
- assert persistent : "Only support deserialize with persistent true";
int size = in.readInt();
try {
for (int i = 0; i < size; i++) {
@@ -390,27 +477,4 @@ public class CachedTreeMap<K extends WritableComparable, V extends Writable> ext
throw new IOException(e);
}
}
-
- // clean up all tmp files
- @Override
- public void finalize() throws Throwable {
- if (persistent) {
- return;
- }
- try {
- this.clear();
- for (String file : fileList) {
- try {
- Path filePath = new Path(file);
- fs.delete(filePath, true);
- } catch (Throwable t) {
- //do nothing?
- }
- }
- } catch (Throwable t) {
- //do nothing
- } finally {
- super.finalize();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
index b2a3664..cda3c2b 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
@@ -19,14 +19,8 @@
package org.apache.kylin.dict;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.NavigableSet;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.MetadataManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* GlobalDictinary based on whole cube, to ensure one value has same dict id in different segments.
@@ -34,41 +28,15 @@ import org.slf4j.LoggerFactory;
* Created by sunyerui on 16/5/24.
*/
public class GlobalDictionaryBuilder implements IDictionaryBuilder {
- private static final Logger logger = LoggerFactory.getLogger(GlobalDictionaryBuilder.class);
-
AppendTrieDictionary.Builder<String> builder;
int baseId;
-
+
@Override
public void init(DictionaryInfo dictInfo, int baseId) throws IOException {
if (dictInfo == null) {
throw new IllegalArgumentException("GlobalDictinaryBuilder must used with an existing DictionaryInfo");
}
- String dictDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict" + dictInfo.getResourceDir() + "/";
-
- // Try to load the existing dict from cache, making sure there's only the same one object in memory
- NavigableSet<String> dicts = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getStore().listResources(dictInfo.getResourceDir());
- ArrayList<String> appendDicts = new ArrayList<>();
- if (dicts != null && !dicts.isEmpty()) {
- for (String dict : dicts) {
- DictionaryInfo info = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getStore().getResource(dict, DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
- if (info.getDictionaryClass().equals(AppendTrieDictionary.class.getName())) {
- appendDicts.add(dict);
- }
- }
- }
-
- if (appendDicts.isEmpty()) {
- logger.info("GlobalDict {} is empty, create new one", dictInfo.getResourceDir());
- this.builder = AppendTrieDictionary.Builder.create(dictDir);
- } else if (appendDicts.size() == 1) {
- logger.info("GlobalDict {} exist, append value", appendDicts.get(0));
- AppendTrieDictionary dict = (AppendTrieDictionary) DictionaryManager.getInstance(KylinConfig.getInstanceFromEnv()).getDictionary(appendDicts.get(0));
- this.builder = AppendTrieDictionary.Builder.create(dict);
- } else {
- throw new IllegalStateException(String.format("GlobalDict %s should have 0 or 1 append dict but %d", dictInfo.getResourceDir(), appendDicts.size()));
- }
-
+ this.builder = AppendTrieDictionary.Builder.getInstance(dictInfo.getResourceDir());
this.baseId = baseId;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index 5e1705a..a7e8152 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -20,13 +20,15 @@ package org.apache.kylin.dict;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
-import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -35,6 +37,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Random;
import java.util.TreeMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -51,6 +55,7 @@ import org.junit.Test;
public class AppendTrieDictionaryTest {
public static final String BASE_DIR = "/tmp/kylin_append_dict";
+ public static final String RESOURCE_DIR = "/dict/append_dict_test";
@BeforeClass
public static void setUp() {
@@ -64,22 +69,28 @@ public class AppendTrieDictionaryTest {
@AfterClass
public static void tearDown() {
- String workingDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
+ cleanup();
+ }
+
+// @After
+ public void afterTest() {
+ cleanup();
+ }
+
+ public static void cleanup() {
+ Configuration conf = new Configuration();
+ Path basePath = new Path(BASE_DIR);
try {
- FileSystem.get(new Path(workingDir).toUri(), new Configuration()).delete(new Path(workingDir), true);
- } catch (IOException e) {
- }
- File tmpLocalDir = new File(BASE_DIR);
- if (tmpLocalDir.exists()) {
- for (File f : tmpLocalDir.listFiles()) {
- f.delete();
- }
- tmpLocalDir.delete();
- }
+ FileSystem.get(basePath.toUri(), conf).delete(basePath, true);
+ } catch (IOException e) {}
}
public static final String[] words = new String[] { "paint", "par", "part", "parts", "partition", "partitions", "party", "partie", "parties", "patient", "taste", "tar", "trie", "try", "tries", "\u5b57\u5178", "\u5b57\u5178\u6811", "\u5b57\u6bcd", // non-ascii characters
"", // empty
+ "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii",
+ "paiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiipaiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii",
+ "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk",
"paint", "tar", "try", // some dup
};
@@ -131,8 +142,7 @@ public class AppendTrieDictionaryTest {
@Ignore("need huge key set")
@Test
public void testHugeKeySet() throws IOException {
- BytesConverter converter = new StringBytesConverter();
- AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.create(BASE_DIR);
+ AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.getInstance(RESOURCE_DIR);
AppendTrieDictionary<String> dict = null;
InputStream is = new FileInputStream("src/test/resources/dict/huge_key");
@@ -162,7 +172,7 @@ public class AppendTrieDictionaryTest {
}
BytesConverter converter = new StringBytesConverter();
- AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.create(BASE_DIR);
+ AppendTrieDictionary.Builder<String> b = AppendTrieDictionary.Builder.getInstance(RESOURCE_DIR);
AppendTrieDictionary<String> dict = null;
TreeMap<Integer, String> checkMap = new TreeMap<>();
int firstAppend = rnd.nextInt(strList.size() / 2);
@@ -179,12 +189,14 @@ public class AppendTrieDictionaryTest {
String str = strList.get(checkIndex);
byte[] bytes = converter.convertToBytes(str);
int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ assertNotEquals(String.format("Value %s not exist", str), -1, id);
assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
checkMap.put(id, str);
}
// reopen dict and append
- b = AppendTrieDictionary.Builder.create(dict);
+// b = AppendTrieDictionary.Builder.create(dict);
+ b = AppendTrieDictionary.Builder.getInstance(RESOURCE_DIR, dict);
for (; appendIndex < secondAppend; appendIndex++) {
b.addValue(strList.get(appendIndex));
}
@@ -197,6 +209,7 @@ public class AppendTrieDictionaryTest {
String str = strList.get(checkIndex);
byte[] bytes = converter.convertToBytes(str);
int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ assertNotEquals(String.format("Value %s not exist", str), -1, id);
if (checkIndex < firstAppend) {
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
} else {
@@ -207,7 +220,7 @@ public class AppendTrieDictionaryTest {
}
// reopen dict and append rest str
- b = AppendTrieDictionary.Builder.create(dict);
+ b = AppendTrieDictionary.Builder.getInstance(RESOURCE_DIR, dict);
for (; appendIndex < strList.size(); appendIndex++) {
b.addValue(strList.get(appendIndex));
}
@@ -220,6 +233,7 @@ public class AppendTrieDictionaryTest {
String str = strList.get(checkIndex);
byte[] bytes = converter.convertToBytes(str);
int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ assertNotEquals(String.format("Value %s not exist", str), -1, id);
if (checkIndex < secondAppend) {
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
} else {
@@ -240,6 +254,7 @@ public class AppendTrieDictionaryTest {
for (String str : strList) {
byte[] bytes = converter.convertToBytes(str);
int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ assertNotEquals(String.format("Value %s not exist", str), -1, id);
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
}
}
@@ -260,4 +275,103 @@ public class AppendTrieDictionaryTest {
throw new RuntimeException(e);
}
}
-}
\ No newline at end of file
+
+ @Test
+ public void testMaxInteger() throws IOException {
+ AppendTrieDictionary.Builder<String> builder = AppendTrieDictionary.Builder.getInstance(RESOURCE_DIR);
+ builder.setMaxId(Integer.MAX_VALUE - 2);
+ builder.addValue("a");
+ builder.addValue("ab");
+ builder.addValue("acd");
+ builder.addValue("ac");
+ AppendTrieDictionary dict = builder.build(0);
+ assertEquals(2147483646, dict.getIdFromValueImpl("a", 0));
+ assertEquals(2147483647, dict.getIdFromValueImpl("ab", 0));
+ assertEquals(-2147483647, dict.getIdFromValueImpl("ac", 0));
+ assertEquals(-2147483648, dict.getIdFromValueImpl("acd", 0));
+ }
+
+ @Ignore("Only occurred when value is very long (>8000 bytes)")
+ @Test
+ public void testSuperLongValue() throws IOException {
+ AppendTrieDictionary.Builder<String> builder = AppendTrieDictionary.Builder.getInstance(RESOURCE_DIR);
+ String value = "a";
+ for (int i = 0; i < 10000; i++) {
+ value += "a";
+ try {
+ builder.addValue(value);
+ } catch (StackOverflowError e) {
+ System.out.println("\nstack overflow " + i);
+ throw e;
+ }
+ }
+ AppendTrieDictionary dictionary = builder.build(0);
+ dictionary.getMaxId();
+ }
+
+ private static class SharedBuilderThread extends Thread {
+ CountDownLatch startLatch;
+ CountDownLatch finishLatch;
+ String resourcePath;
+ String prefix;
+ int count;
+
+ SharedBuilderThread(CountDownLatch startLatch, CountDownLatch finishLatch, String resourcePath, String prefix, int count) {
+ this.startLatch = startLatch;
+ this.finishLatch = finishLatch;
+ this.resourcePath = resourcePath;
+ this.prefix = prefix;
+ this.count = count;
+ }
+
+ @Override
+ public void run() {
+ try {
+ AppendTrieDictionary.Builder<String> builder = AppendTrieDictionary.Builder.getInstance(resourcePath);
+ startLatch.countDown();
+ for (int i = 0; i < count; i++) {
+ builder.addValue(prefix + i);
+ }
+ builder.build(0);
+ finishLatch.countDown();
+ } catch (IOException e) {}
+ }
+ }
+
+ @Test
+ public void testSharedBuilder() throws IOException, InterruptedException {
+ String resourcePath = "shared_builder";
+ final CountDownLatch startLatch = new CountDownLatch(3);
+ final CountDownLatch finishLatch = new CountDownLatch(3);
+
+ AppendTrieDictionary.Builder<String> builder = AppendTrieDictionary.Builder.getInstance(resourcePath);
+ Thread t1 = new SharedBuilderThread(startLatch, finishLatch, resourcePath, "t1_", 10000);
+ Thread t2 = new SharedBuilderThread(startLatch, finishLatch, resourcePath, "t2_", 10);
+ Thread t3 = new SharedBuilderThread(startLatch, finishLatch, resourcePath, "t3_", 100000);
+ t1.start();
+ t2.start();
+ t3.start();
+ startLatch.await();
+ AppendTrieDictionary dict = builder.build(0);
+ assertTrue("AppendDictBuilder Thread too slow", finishLatch.await(3000, TimeUnit.MILLISECONDS));
+ assertEquals(110010, dict.getMaxId());
+ try {
+ builder.addValue("fail");
+ fail("Builder should be closed");
+ } catch (Exception e) {}
+
+ builder = AppendTrieDictionary.Builder.getInstance(resourcePath, dict);
+ builder.addValue("success");
+ dict = builder.build(0);
+ for (int i = 0; i < 10000; i ++) {
+ assertNotEquals(-1, dict.getIdFromValue("t1_" + i));
+ }
+ for (int i = 0; i < 10; i ++) {
+ assertNotEquals(-1, dict.getIdFromValue("t2_" + i));
+ }
+ for (int i = 0; i < 100000; i ++) {
+ assertNotEquals(-1, dict.getIdFromValue("t3_" + i));
+ }
+ assertEquals(110011, dict.getIdFromValue("success"));
+ }
+}
[11/50] [abbrv] kylin git commit: filter null value in cubedesc and
model desc
Posted by li...@apache.org.
filter null value in cubedesc and model desc
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/875a5931
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/875a5931
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/875a5931
Branch: refs/heads/master-cdh5.7
Commit: 875a5931ad8cad79cdec1e80ab9388ded9446091
Parents: 3186d17
Author: luguosheng <55...@qq.com>
Authored: Wed Nov 30 10:51:51 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Nov 30 13:48:29 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 4 ++--
webapp/app/js/controllers/modelEdit.js | 13 ++++++++++---
webapp/app/js/utils/utils.js | 24 ++++++++++++++++++++++++
3 files changed, 36 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/875a5931/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index 888662d..0704a62 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -413,7 +413,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
if ($scope.isEdit) {
CubeService.update({}, {
- cubeDescData: $scope.state.cubeSchema,
+ cubeDescData: VdmUtil.filterNullValInObj($scope.state.cubeSchema),
cubeName: $routeParams.cubeName,
project: $scope.state.project
}, function (request) {
@@ -453,7 +453,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
});
} else {
CubeService.save({}, {
- cubeDescData: $scope.state.cubeSchema,
+ cubeDescData: VdmUtil.filterNullValInObj($scope.state.cubeSchema),
project: $scope.state.project
}, function (request) {
if (request.successful) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/875a5931/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index 0b3db66..2f1b35e 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -19,7 +19,7 @@
'use strict';
-KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $location, $templateCache, $interpolate, MessageService, TableService, CubeDescService, ModelService, loadingRequest, SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager) {
+KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $location, $templateCache, $interpolate, MessageService, TableService, CubeDescService, ModelService, loadingRequest, SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager,VdmUtil) {
//add or edit ?
var absUrl = $location.absUrl();
$scope.modelMode = absUrl.indexOf("/models/add")!=-1?'addNewModel':absUrl.indexOf("/models/edit")!=-1?'editExistModel':'default';
@@ -188,7 +188,11 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
loadingRequest.show();
if ($scope.isEdit) {
- ModelService.update({}, {modelDescData:$scope.state.modelSchema, modelName: $routeParams.modelName, project: $scope.state.project}, function (request) {
+ ModelService.update({}, {
+ modelDescData:VdmUtil.filterNullValInObj($scope.state.modelSchema),
+ modelName: $routeParams.modelName,
+ project: $scope.state.project
+ }, function (request) {
if (request.successful) {
$scope.state.modelSchema = request.modelSchema;
SweetAlert.swal('', 'Updated the model successfully.', 'success');
@@ -216,7 +220,10 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
loadingRequest.hide();
});
} else {
- ModelService.save({}, {modelDescData:$scope.state.modelSchema, project: $scope.state.project}, function (request) {
+ ModelService.save({}, {
+ modelDescData:VdmUtil.filterNullValInObj($scope.state.modelSchema),
+ project: $scope.state.project
+ }, function (request) {
if(request.successful) {
$scope.state.modelSchema = request.modelSchema;
http://git-wip-us.apache.org/repos/asf/kylin/blob/875a5931/webapp/app/js/utils/utils.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/utils/utils.js b/webapp/app/js/utils/utils.js
index 9da6196..0875e52 100644
--- a/webapp/app/js/utils/utils.js
+++ b/webapp/app/js/utils/utils.js
@@ -116,6 +116,30 @@ KylinApp.factory('VdmUtil', function ($modal, $timeout, $location, $anchorScroll
}
}
return newArr;
+ },
+ //\u8fc7\u6ee4\u5bf9\u8c61\u4e2d\u7684\u7a7a\u503c
+ filterNullValInObj:function(needFilterObj){
+ var newFilterObj,newObj;
+ if(typeof needFilterObj=='string'){
+ newObj=angular.fromJson(needFilterObj);
+ }else{
+ newObj=angular.extend({},needFilterObj);
+ }
+ function filterData(data){
+ var obj=data;
+ for(var i in obj){
+ if(obj[i]===null){
+ if(Object.prototype.toString.call(obj)=='[object Object]'){
+ delete obj[i];
+ }
+ }
+ else if(typeof obj[i]=== 'object'){
+ obj[i]=filterData(obj[i]);
+ }
+ }
+ return obj;
+ }
+ return angular.toJson(filterData(newObj),true);
}
}
[33/50] [abbrv] kylin git commit: minor,
add setter for BuiltInFunctionTupleFilter's reversed field
Posted by li...@apache.org.
minor, add setter for BuiltInFunctionTupleFilter's reversed field
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95561526
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95561526
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95561526
Branch: refs/heads/master-cdh5.7
Commit: 9556152622599a87610f16ae8f06185e24ec02d6
Parents: 00a4c50
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Dec 5 10:58:31 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Dec 5 10:58:31 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/95561526/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index b678394..90123c4 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -109,6 +109,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
return isReversed;
}
+ public void setReversed(boolean reversed) {
+ this.isReversed = reversed;
+ }
+
@Override
public void addChild(TupleFilter child) {
if (child instanceof ColumnTupleFilter || child instanceof BuiltInFunctionTupleFilter) {
[14/50] [abbrv] kylin git commit: KYLIN-2217 Code review,
refactor IDictionaryBuilder
Posted by li...@apache.org.
KYLIN-2217 Code review, refactor IDictionaryBuilder
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30cb1ac6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30cb1ac6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30cb1ac6
Branch: refs/heads/master-cdh5.7
Commit: 30cb1ac678b063ac164acd27591e8a1d0becafa0
Parents: 1af08e4
Author: Li Yang <li...@apache.org>
Authored: Wed Nov 30 15:30:15 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Nov 30 21:00:17 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 2 +-
.../apache/kylin/dict/DictionaryGenerator.java | 165 ++++++++++++-------
.../apache/kylin/dict/DictionaryManager.java | 16 +-
.../apache/kylin/dict/DictionaryProvider.java | 4 +-
.../dict/DictionaryReducerLocalGenerator.java | 156 ------------------
.../kylin/dict/GlobalDictionaryBuilder.java | 36 ++--
.../apache/kylin/dict/IDictionaryBuilder.java | 13 +-
.../dict/IDictionaryReducerLocalBuilder.java | 31 ----
.../kylin/dict/DictionaryProviderTest.java | 63 +++----
.../engine/mr/steps/CreateDictionaryJob.java | 42 ++---
.../mr/steps/FactDistinctColumnsReducer.java | 115 +++++--------
.../mr/steps/FactDistinctHiveColumnsMapper.java | 9 +-
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 6 +-
13 files changed, 231 insertions(+), 427 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index a4e1df0..163c6ca 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -60,7 +60,7 @@ public class DictionaryGeneratorCLI {
for (TblColRef col : cubeSeg.getCubeDesc().getAllColumnsNeedDictionaryBuilt()) {
logger.info("Building dictionary for " + col);
ReadableTable inpTable = decideInputTable(cubeSeg.getModel(), col, factTableValueProvider);
- if (config.isReducerLocalBuildDict() && dictProvider != null) {
+ if (dictProvider != null) {
Dictionary<String> dict = dictProvider.getDictionary(col);
if (dict != null) {
cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 810a392..cd13d59 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -19,12 +19,11 @@
package org.apache.kylin.dict;
import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.datatype.DataType;
import org.slf4j.Logger;
@@ -40,9 +39,7 @@ public class DictionaryGenerator {
private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);
- private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd", "yyyyMMdd" };
-
- public static Dictionary<String> buildDictionary(DataType dataType, IDictionaryValueEnumerator valueEnumerator) throws IOException {
+ public static IDictionaryBuilder newDictionaryBuilder(DataType dataType) {
Preconditions.checkNotNull(dataType, "dataType cannot be null");
// build dict, case by data type
@@ -57,16 +54,33 @@ public class DictionaryGenerator {
} else {
builder = new StringDictBuilder();
}
+ return builder;
+ }
- return buildDictionary(builder, null, valueEnumerator);
+ public static Dictionary<String> buildDictionary(DataType dataType, IDictionaryValueEnumerator valueEnumerator) throws IOException {
+ return buildDictionary(newDictionaryBuilder(dataType), null, valueEnumerator);
}
- public static Dictionary<String> buildDictionary(IDictionaryBuilder builder, DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator) throws IOException {
+ static Dictionary<String> buildDictionary(IDictionaryBuilder builder, DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator) throws IOException {
int baseId = 0; // always 0 for now
int nSamples = 5;
ArrayList<String> samples = new ArrayList<String>(nSamples);
- Dictionary<String> dict = builder.build(dictInfo, valueEnumerator, baseId, nSamples, samples);
+ // init the builder
+ builder.init(dictInfo, baseId);
+
+ // add values
+ while (valueEnumerator.moveNext()) {
+ String value = valueEnumerator.current();
+
+ boolean accept = builder.addValue(value);
+
+ if (accept && samples.size() < nSamples && samples.contains(value) == false)
+ samples.add(value);
+ }
+
+ // build
+ Dictionary<String> dict = builder.build();
// log a few samples
StringBuilder buf = new StringBuilder();
@@ -88,81 +102,114 @@ public class DictionaryGenerator {
}
private static class DateDictBuilder implements IDictionaryBuilder {
+ private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd", "yyyyMMdd" };
+
+ private int baseId;
+ private String datePattern;
+
@Override
- public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
- final int BAD_THRESHOLD = 0;
- String matchPattern = null;
- String value;
-
- for (String ptn : DATE_PATTERNS) {
- matchPattern = ptn; // be optimistic
- int badCount = 0;
- SimpleDateFormat sdf = new SimpleDateFormat(ptn);
- while (valueEnumerator.moveNext()) {
- value = valueEnumerator.current();
- if (value == null || value.length() == 0)
- continue;
+ public void init(DictionaryInfo info, int baseId) throws IOException {
+ this.baseId = baseId;
+ }
+ @Override
+ public boolean addValue(String value) {
+ if (StringUtils.isBlank(value)) // empty string is treated as null
+ return false;
+
+ // detect date pattern on the first value
+ if (datePattern == null) {
+ for (String p : DATE_PATTERNS) {
try {
- sdf.parse(value);
- if (returnSamples.size() < nSamples && returnSamples.contains(value) == false)
- returnSamples.add(value);
- } catch (ParseException e) {
- logger.info("Unrecognized date value: " + value);
- badCount++;
- if (badCount > BAD_THRESHOLD) {
- matchPattern = null;
- break;
- }
+ DateFormat.stringToDate(value, p);
+ datePattern = p;
+ break;
+ } catch (Exception e) {
+ // continue;
}
}
- if (matchPattern != null) {
- return new DateStrDictionary(matchPattern, baseId);
- }
+ if (datePattern == null)
+ throw new IllegalArgumentException("Unknown date pattern for input value: " + value);
}
+
+ // check the date format
+ DateFormat.stringToDate(value, datePattern);
+ return true;
+ }
+
+ @Override
+ public Dictionary<String> build() throws IOException {
+ if (datePattern == null)
+ datePattern = DATE_PATTERNS[0];
- throw new IllegalStateException("Unrecognized datetime value");
+ return new DateStrDictionary(datePattern, baseId);
}
}
private static class TimeDictBuilder implements IDictionaryBuilder {
+
@Override
- public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
+ public void init(DictionaryInfo info, int baseId) throws IOException {
+ }
+
+ @Override
+ public boolean addValue(String value) {
+ if (StringUtils.isBlank(value)) // empty string is treated as null
+ return false;
+
+ // check the time format
+ DateFormat.stringToMillis(value);
+ return true;
+ }
+
+ @Override
+ public Dictionary<String> build() throws IOException {
return new TimeStrDictionary(); // base ID is always 0
}
}
private static class StringDictBuilder implements IDictionaryBuilder {
+ TrieDictionaryForestBuilder builder;
+
@Override
- public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
- TrieDictionaryForestBuilder builder = new TrieDictionaryForestBuilder(new StringBytesConverter(), baseId);
- String value;
- while (valueEnumerator.moveNext()) {
- value = valueEnumerator.current();
- if (value == null)
- continue;
- builder.addValue(value);
- if (returnSamples.size() < nSamples && returnSamples.contains(value) == false)
- returnSamples.add(value);
- }
+ public void init(DictionaryInfo info, int baseId) throws IOException {
+ builder = new TrieDictionaryForestBuilder(new StringBytesConverter(), baseId);
+ }
+
+ @Override
+ public boolean addValue(String value) {
+ if (value == null)
+ return false;
+
+ builder.addValue(value);
+ return true;
+ }
+
+ @Override
+ public Dictionary<String> build() throws IOException {
return builder.build();
}
}
private static class NumberDictBuilder implements IDictionaryBuilder {
+ NumberDictionaryForestBuilder builder;
+
@Override
- public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
- NumberDictionaryForestBuilder builder = new NumberDictionaryForestBuilder(baseId);
- String value;
- while (valueEnumerator.moveNext()) {
- value = valueEnumerator.current();
- if (StringUtils.isBlank(value)) // empty string is null for numbers
- continue;
-
- builder.addValue(value);
- if (returnSamples.size() < nSamples && returnSamples.contains(value) == false)
- returnSamples.add(value);
- }
+ public void init(DictionaryInfo info, int baseId) throws IOException {
+ builder = new NumberDictionaryForestBuilder(baseId);
+ }
+
+ @Override
+ public boolean addValue(String value) {
+ if (StringUtils.isBlank(value)) // empty string is treated as null
+ return false;
+
+ builder.addValue(value);
+ return true;
+ }
+
+ @Override
+ public Dictionary<String> build() throws IOException {
return builder.build();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 0caef14..54bc1c4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -87,10 +87,6 @@ public class DictionaryManager {
private KylinConfig config;
private LoadingCache<String, DictionaryInfo> dictCache; // resource
-
- // path ==>
- // DictionaryInfo
-
private DictionaryManager(KylinConfig config) {
this.config = config;
this.dictCache = CacheBuilder.newBuilder().removalListener(new RemovalListener<String, DictionaryInfo>() {
@@ -276,12 +272,10 @@ public class DictionaryManager {
return buildDictionary(model, col, inpTable, null);
}
-
public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, ReadableTable inpTable, String builderClass) throws IOException {
if (inpTable.exists() == false)
return null;
-
logger.info("building dictionary for " + col);
DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable);
@@ -303,10 +297,12 @@ public class DictionaryManager {
IDictionaryValueEnumerator columnValueEnumerator = null;
try {
columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(), dictInfo.getSourceColumnIndex());
- if (builderClass == null)
+ if (builderClass == null) {
dictionary = DictionaryGenerator.buildDictionary(DataType.getType(dictInfo.getDataType()), columnValueEnumerator);
- else
- dictionary = DictionaryGenerator.buildDictionary((IDictionaryBuilder) ClassUtil.newInstance(builderClass), dictInfo, columnValueEnumerator);
+ } else {
+ IDictionaryBuilder builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
+ dictionary = DictionaryGenerator.buildDictionary(builder, dictInfo, columnValueEnumerator);
+ }
} catch (Exception ex) {
throw new RuntimeException("Failed to create dictionary on " + col, ex);
} finally {
@@ -365,7 +361,7 @@ public class DictionaryManager {
while (join != null) {
if (join.isInnerJoin() == false)
return false;
-
+
TableRef table = join.getFKSide();
join = model.getJoinByPKSide(table);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryProvider.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryProvider.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryProvider.java
index 6387535..8476f5c 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryProvider.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryProvider.java
@@ -17,6 +17,8 @@
*/
package org.apache.kylin.dict;
+import java.io.IOException;
+
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.model.TblColRef;
@@ -24,5 +26,5 @@ import org.apache.kylin.metadata.model.TblColRef;
* Created by xiefan on 16-11-23.
*/
public interface DictionaryProvider {
- public Dictionary<String> getDictionary(TblColRef col);
+ public Dictionary<String> getDictionary(TblColRef col) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryReducerLocalGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryReducerLocalGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryReducerLocalGenerator.java
deleted file mode 100644
index 35d379a..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryReducerLocalGenerator.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.dict;
-
-import com.google.common.base.Preconditions;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.datatype.DataType;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-
-/**
- * Created by xiefan on 16-11-16.
- *
- * TODO:sample,mergeDict
- */
-public class DictionaryReducerLocalGenerator {
-
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DictionaryReducerLocalGenerator.class);
-
- private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd", "yyyyMMdd" };
-
- public static IDictionaryReducerLocalBuilder getBuilder(DataType dataType) {
- Preconditions.checkNotNull(dataType, "dataType cannot be null");
-
- IDictionaryReducerLocalBuilder builder;
- if (dataType.isDateTimeFamily()) {
- if (dataType.isDate())
- builder = new DateDictBuilder();
- else
- builder = new TimeDictBuilder();
- } else if (dataType.isNumberFamily()) {
- builder = new NumberDictBuilder(0);
- } else {
- builder = new StringDictBuilder(0);
- }
- return builder;
- }
-
- private static class DateDictBuilder implements IDictionaryReducerLocalBuilder {
-
- private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd", "yyyyMMdd" };
-
- private String matchPattern = null;
-
- private boolean isRecognizeFormat = false;
-
- private SimpleDateFormat sdf;
-
- @Override
- public Dictionary<String> build(int baseId) throws Exception {
- if (isRecognizeFormat) {
- return new DateStrDictionary(matchPattern, baseId);
- } else {
- throw new IllegalStateException("Date format not match");
- }
- }
-
- @Override
- public void addValue(String value) throws Exception {
- if (matchPattern == null) { //init match pattern
- for (String ptn : DATE_PATTERNS) {
- matchPattern = ptn;
- SimpleDateFormat sdf = new SimpleDateFormat(ptn);
- try {
- sdf.parse(value);
- isRecognizeFormat = true;
- break;
- } catch (ParseException e) {
-
- }
- }
- sdf = new SimpleDateFormat(matchPattern);
- }
- if (!isRecognizeFormat) {
- throw new IllegalStateException("Date format not match");
- }
- try {
- sdf.parse(value);
- } catch (ParseException e) {
- isRecognizeFormat = false;
- logger.info("Unrecognized date value: " + value);
- }
- }
-
- }
-
- private static class TimeDictBuilder implements IDictionaryReducerLocalBuilder {
-
- @Override
- public Dictionary<String> build(int baseId) {
- return new TimeStrDictionary();
- }
-
- @Override
- public void addValue(String value) {
-
- }
-
- }
-
- private static class StringDictBuilder implements IDictionaryReducerLocalBuilder {
-
- private TrieDictionaryForestBuilder<String> builder;
-
- public StringDictBuilder(int baseId) {
- builder = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), 0);
- }
-
- @Override
- public Dictionary<String> build(int baseId) {
- return builder.build();
- }
-
- @Override
- public void addValue(String value) {
- builder.addValue(value);
- }
-
- }
-
- public static class NumberDictBuilder implements IDictionaryReducerLocalBuilder {
-
- private NumberDictionaryForestBuilder builder;
-
- public NumberDictBuilder(int baseId) {
- builder = new NumberDictionaryForestBuilder(baseId);
- }
-
- @Override
- public Dictionary<String> build(int baseId) {
- return builder.build();
- }
-
- @Override
- public void addValue(String value) {
- builder.addValue(value);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
index 7adc262..b2a3664 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
@@ -36,8 +36,11 @@ import org.slf4j.LoggerFactory;
public class GlobalDictionaryBuilder implements IDictionaryBuilder {
private static final Logger logger = LoggerFactory.getLogger(GlobalDictionaryBuilder.class);
+ AppendTrieDictionary.Builder<String> builder;
+ int baseId;
+
@Override
- public Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException {
+ public void init(DictionaryInfo dictInfo, int baseId) throws IOException {
if (dictInfo == null) {
throw new IllegalArgumentException("GlobalDictinaryBuilder must used with an existing DictionaryInfo");
}
@@ -55,28 +58,31 @@ public class GlobalDictionaryBuilder implements IDictionaryBuilder {
}
}
- AppendTrieDictionary.Builder<String> builder;
if (appendDicts.isEmpty()) {
logger.info("GlobalDict {} is empty, create new one", dictInfo.getResourceDir());
- builder = AppendTrieDictionary.Builder.create(dictDir);
+ this.builder = AppendTrieDictionary.Builder.create(dictDir);
} else if (appendDicts.size() == 1) {
logger.info("GlobalDict {} exist, append value", appendDicts.get(0));
AppendTrieDictionary dict = (AppendTrieDictionary) DictionaryManager.getInstance(KylinConfig.getInstanceFromEnv()).getDictionary(appendDicts.get(0));
- builder = AppendTrieDictionary.Builder.create(dict);
+ this.builder = AppendTrieDictionary.Builder.create(dict);
} else {
throw new IllegalStateException(String.format("GlobalDict %s should have 0 or 1 append dict but %d", dictInfo.getResourceDir(), appendDicts.size()));
}
-
- String value;
- while (valueEnumerator.moveNext()) {
- value = valueEnumerator.current();
- if (value == null) {
- continue;
- }
- builder.addValue(value);
- if (returnSamples.size() < nSamples && returnSamples.contains(value) == false)
- returnSamples.add(value);
- }
+
+ this.baseId = baseId;
+ }
+
+ @Override
+ public boolean addValue(String value) {
+ if (value == null)
+ return false;
+
+ builder.addValue(value);
+ return true;
+ }
+
+ @Override
+ public Dictionary<String> build() throws IOException {
return builder.build(baseId);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java
index 8f95a2a..0934a7d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryBuilder.java
@@ -19,11 +19,20 @@
package org.apache.kylin.dict;
import java.io.IOException;
-import java.util.ArrayList;
import org.apache.kylin.common.util.Dictionary;
+/**
+ * An once-only builder for dictionary.
+ */
public interface IDictionaryBuilder {
- Dictionary<String> build(DictionaryInfo dictInfo, IDictionaryValueEnumerator valueEnumerator, int baseId, int nSamples, ArrayList<String> returnSamples) throws IOException;
+ /** Sets the dictionary info for the dictionary being built. Mainly for GlobalDictionaryBuilder. */
+ void init(DictionaryInfo info, int baseId) throws IOException;
+
+ /** Add a new value into dictionary, returns it is accepted (not null) or not. */
+ boolean addValue(String value);
+
+ /** Build the dictionary */
+ Dictionary<String> build() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryReducerLocalBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryReducerLocalBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryReducerLocalBuilder.java
deleted file mode 100644
index 19b1d28..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryReducerLocalBuilder.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.dict;
-
-import org.apache.kylin.common.util.Dictionary;
-
-/**
- * Created by xiefan on 16-11-16.
- */
-public interface IDictionaryReducerLocalBuilder {
- Dictionary<String> build(int baseId) throws Exception;
-
- void addValue(String value) throws Exception;
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
index 0225737..a4aee76 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
@@ -1,26 +1,20 @@
package org.apache.kylin.dict;
-import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.Test;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
-import java.io.BufferedOutputStream;
-import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.lang.reflect.ParameterizedType;
import java.util.Arrays;
import java.util.Iterator;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.Test;
/**
* Created by xiefan on 16-11-23.
@@ -28,53 +22,48 @@ import static org.junit.Assert.fail;
public class DictionaryProviderTest {
@Test
- public void testReadWrite() throws Exception{
+ public void testReadWrite() throws Exception {
//string dict
- Dictionary<String> dict = getDict(DataType.getType("string"),
- Arrays.asList(new String[]{"a","b"}).iterator());
+ Dictionary<String> dict = getDict(DataType.getType("string"), Arrays.asList(new String[] { "a", "b" }).iterator());
readWriteTest(dict);
//number dict
- Dictionary<String> dict2 = getDict(DataType.getType("long"),
- Arrays.asList(new String[]{"1","2"}).iterator());
+ Dictionary<String> dict2 = getDict(DataType.getType("long"), Arrays.asList(new String[] { "1", "2" }).iterator());
readWriteTest(dict2);
//date dict
- Dictionary<String> dict3 = getDict(DataType.getType("datetime"),
- Arrays.asList(new String[]{"20161122","20161123"}).iterator());
+ Dictionary<String> dict3 = getDict(DataType.getType("datetime"), Arrays.asList(new String[] { "20161122", "20161123" }).iterator());
readWriteTest(dict3);
//date dict
- Dictionary<String> dict4 = getDict(DataType.getType("datetime"),
- Arrays.asList(new String[]{"2016-11-22","2016-11-23"}).iterator());
+ Dictionary<String> dict4 = getDict(DataType.getType("datetime"), Arrays.asList(new String[] { "2016-11-22", "2016-11-23" }).iterator());
readWriteTest(dict4);
//date dict
try {
- Dictionary<String> dict5 = getDict(DataType.getType("date"),
- Arrays.asList(new String[]{"2016-11-22", "20161122"}).iterator());
+ Dictionary<String> dict5 = getDict(DataType.getType("date"), Arrays.asList(new String[] { "2016-11-22", "20161122" }).iterator());
readWriteTest(dict5);
fail("Date format not correct.Should throw exception");
- }catch (IllegalStateException e){
+ } catch (IllegalArgumentException e) {
//correct
}
}
@Test
- public void testReadWriteTime(){
+ public void testReadWriteTime() {
System.out.println(Long.MAX_VALUE);
System.out.println(Long.MIN_VALUE);
}
-
- private Dictionary<String> getDict(DataType type, Iterator<String> values) throws Exception{
- IDictionaryReducerLocalBuilder builder = DictionaryReducerLocalGenerator.getBuilder(type);
- while(values.hasNext()){
+ private Dictionary<String> getDict(DataType type, Iterator<String> values) throws Exception {
+ IDictionaryBuilder builder = DictionaryGenerator.newDictionaryBuilder(type);
+ builder.init(null, 0);
+ while (values.hasNext()) {
builder.addValue(values.next());
}
- return builder.build(0);
+ return builder.build();
}
- private void readWriteTest(Dictionary<String> dict) throws Exception{
+ private void readWriteTest(Dictionary<String> dict) throws Exception {
final String path = "src/test/resources/dict/tmp_dict";
File f = new File(path);
f.deleteOnExit();
@@ -93,15 +82,9 @@ public class DictionaryProviderTest {
String dictClassName2 = in.readUTF();
dict2 = (Dictionary<String>) ClassUtil.newInstance(dictClassName2);
dict2.readFields(in);
- }catch(IOException e){
- e.printStackTrace();
- }finally {
- if(in != null){
- try {
- in.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
+ } finally {
+ if (in != null) {
+ in.close();
}
}
assertTrue(dict.equals(dict2));
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index 63005f9..4985503 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -18,8 +18,10 @@
package org.apache.kylin.engine.mr.steps;
+import java.io.IOException;
+
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -36,16 +38,8 @@ import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
-import java.io.IOException;
-
-/**
- * @author ysong1
- */
-
public class CreateDictionaryJob extends AbstractHadoopJob {
- private int returnCode = 0;
-
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
@@ -68,38 +62,28 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
}, new DictionaryProvider() {
@Override
- public Dictionary<String> getDictionary(TblColRef col) {
- if (!config.isReducerLocalBuildDict()) {
+ public Dictionary<String> getDictionary(TblColRef col) throws IOException {
+ Path colDir = new Path(factColumnsInputPath, col.getName());
+ Path dictFile = new Path(colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
+ FileSystem fs = HadoopUtil.getFileSystem(dictFile.toString());
+ if (fs.exists(dictFile) == false)
return null;
- }
+
FSDataInputStream is = null;
try {
- Path colDir = new Path(factColumnsInputPath, col.getName());
- Path outputFile = new Path(colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
- Configuration conf = HadoopUtil.getCurrentConfiguration();
- FileSystem fs = HadoopUtil.getFileSystem(outputFile.getName());
- is = fs.open(outputFile);
+ is = fs.open(dictFile);
String dictClassName = is.readUTF();
Dictionary<String> dict = (Dictionary<String>) ClassUtil.newInstance(dictClassName);
dict.readFields(is);
- logger.info("DictionaryProvider read dict form file : " + outputFile.getName());
+ logger.info("DictionaryProvider read dict from file: " + dictFile);
return dict;
- } catch (Exception e) {
- e.printStackTrace();
- return null;
} finally {
- if (is != null) {
- try {
- is.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
+ IOUtils.closeQuietly(is);
}
}
});
- return returnCode;
+ return 0;
}
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 5511626..8933ee2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -27,8 +27,6 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -43,14 +41,13 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.DictionaryReducerLocalGenerator;
-import org.apache.kylin.dict.IDictionaryReducerLocalBuilder;
+import org.apache.kylin.dict.DictionaryGenerator;
+import org.apache.kylin.dict.IDictionaryBuilder;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CubeStatsWriter;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
-import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,6 +59,8 @@ import com.google.common.collect.Maps;
*/
public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableKey, Text, NullWritable, Text> {
+ protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
+
private List<TblColRef> columnList;
private String statisticsOutput = null;
private List<Long> baseCuboidRowCountInMappers;
@@ -75,19 +74,16 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private boolean isStatistics = false;
private KylinConfig cubeConfig;
private int uhcReducerCount;
- private Map<Integer, Integer> ReducerIdToColumnIndex = new HashMap<>();
+ private Map<Integer, Integer> reducerIdToColumnIndex = new HashMap<>();
private int taskId;
- protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
-
//local build dict
private boolean isReducerLocalBuildDict;
- private IDictionaryReducerLocalBuilder builder;
- private FastDateFormat dateFormat;
+ private IDictionaryBuilder builder;
private long timeMaxValue = Long.MIN_VALUE;
private long timeMinValue = Long.MAX_VALUE;
- public static final String DICT_FILE_POSTFIX = ".RLD";
- public static final String PARTITION_COL_INFO_FILE_POSTFIX = ".PCI";
+ public static final String DICT_FILE_POSTFIX = ".rldict";
+ public static final String PARTITION_COL_INFO_FILE_POSTFIX = ".pci";
private boolean isPartitionCol = false;
@Override
@@ -121,43 +117,29 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
isPartitionCol = true;
col = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
colValues = Lists.newLinkedList();
- DataType partitionColType = col.getType();
- if (partitionColType.isDate()) {
- dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATE_PATTERN);
- } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) {
- dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
- } else if (partitionColType.isStringFamily()) {
- String partitionDateFormat = cubeDesc.getModel().getPartitionDesc().getPartitionDateFormat();
- if (StringUtils.isEmpty(partitionDateFormat)) {
- partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
- }
- dateFormat = DateFormat.getDateFormat(partitionDateFormat);
- } else {
- throw new IllegalStateException("Type " + partitionColType + " is not valid partition column type");
- }
} else {
- // col
+ // normal col
isStatistics = false;
- col = columnList.get(ReducerIdToColumnIndex.get(taskId));
+ col = columnList.get(reducerIdToColumnIndex.get(taskId));
colValues = Lists.newLinkedList();
+
+ // local build dict
+ isReducerLocalBuildDict = config.isReducerLocalBuildDict();
+ if (col != null && isReducerLocalBuildDict) {
+ builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
+ builder.init(null, 0);
+ }
}
-
- //local build dict
- isReducerLocalBuildDict = config.isReducerLocalBuildDict();
- if (col != null && isReducerLocalBuildDict) {
- builder = DictionaryReducerLocalGenerator.getBuilder(col.getType());
- }
-
}
private void initReducerIdToColumnIndex(KylinConfig config) throws IOException {
int[] uhcIndex = CubeManager.getInstance(config).getUHCIndex(cubeDesc);
int count = 0;
for (int i = 0; i < uhcIndex.length; i++) {
- ReducerIdToColumnIndex.put(count * (uhcReducerCount - 1) + i, i);
+ reducerIdToColumnIndex.put(count * (uhcReducerCount - 1) + i, i);
if (uhcIndex[i] == 1) {
for (int j = 1; j < uhcReducerCount; j++) {
- ReducerIdToColumnIndex.put(count * (uhcReducerCount - 1) + j + i, i);
+ reducerIdToColumnIndex.put(count * (uhcReducerCount - 1) + j + i, i);
}
count++;
}
@@ -167,7 +149,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
@Override
public void doReduce(SelfDefineSortableKey skey, Iterable<Text> values, Context context) throws IOException, InterruptedException {
Text key = skey.getText();
- if (isStatistics == true) {
+ if (isStatistics) {
// for hll
long cuboidId = Bytes.toLong(key.getBytes(), 1, Bytes.SIZEOF_LONG);
for (Text value : values) {
@@ -187,20 +169,17 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
cuboidHLLMap.put(cuboidId, hll);
}
}
+ } else if (isPartitionCol) {
+ // partition col
+ String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1);
+ long time = DateFormat.stringToMillis(value);
+ timeMinValue = Math.min(timeMinValue, time);
+ timeMaxValue = Math.max(timeMaxValue, time);
} else {
+ // normal col
if (isReducerLocalBuildDict) {
- String value = new String(key.getBytes(), 1, key.getLength() - 1);
- //partition col
- try {
- if (isPartitionCol) {
- long time = dateFormat.parse(value).getTime();
- timeMinValue = Math.min(timeMinValue, time);
- timeMaxValue = Math.max(timeMaxValue, time);
- }
- builder.addValue(value);
- } catch (Exception e) {
- e.printStackTrace();
- }
+ String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1);
+ builder.addValue(value);
} else {
colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1)));
if (colValues.size() == 1000000) { //spill every 1 million
@@ -210,7 +189,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
}
}
-
}
private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException {
@@ -286,25 +264,8 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
@Override
protected void doCleanup(Context context) throws IOException, InterruptedException {
- if (isStatistics == false) {
- if (isReducerLocalBuildDict) {
- try {
- if (isPartitionCol) {
- outputPartitionInfo(context);
- }
- Dictionary<String> dict = builder.build(0);
- outputDict(col, dict, context);
- } catch (Exception e) {
- e.printStackTrace();
- }
- } else {
- if (colValues.size() > 0) {
- outputDistinctValues(col, colValues, context);
- colValues.clear();
- }
- }
- } else {
- //output the hll info;
+ if (isStatistics) {
+ // output the hll info
long grandTotal = 0;
for (HyperLogLogPlusCounter hll : cuboidHLLMap.values()) {
grandTotal += hll.getCountEstimate();
@@ -316,6 +277,20 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
writeMapperAndCuboidStatistics(context); // for human check
CubeStatsWriter.writeCuboidStatistics(context.getConfiguration(), new Path(statisticsOutput), //
cuboidHLLMap, samplingPercentage, mapperNumber, mapperOverlapRatio);
+ } else if (isPartitionCol) {
+ // partition col
+ outputPartitionInfo(context);
+ } else {
+ // normal col
+ if (isReducerLocalBuildDict) {
+ Dictionary<String> dict = builder.build();
+ outputDict(col, dict, context);
+ } else {
+ if (colValues.size() > 0) {
+ outputDistinctValues(col, colValues, context);
+ colValues.clear();
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
index 762047b..a5c8fc0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
@@ -97,15 +97,8 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
// if partition col not on cube, no need
needFetchPartitionCol = false;
} else {
- for (int x : dictionaryColumnIndex) {
- if (x == partitionColumnIndex) {
- // if partition col already build dict, no need
- needFetchPartitionCol = false;
- break;
- }
- }
+ needFetchPartitionCol = true;
}
-
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30cb1ac6/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index 977196c..d3becfe 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -21,7 +21,6 @@ package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -84,8 +83,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
Path colDir = new Path(factColumnsInputPath, partitionCol.getName());
Path outputFile = new Path(colDir, partitionCol.getName() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
- Configuration conf = HadoopUtil.getCurrentConfiguration();
- FileSystem fs = HadoopUtil.getFileSystem(outputFile.getName());
+ FileSystem fs = HadoopUtil.getFileSystem(outputFile.toString());
FSDataInputStream is = null;
long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
try {
@@ -94,8 +92,6 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
long max = is.readLong();
minValue = Math.min(min, minValue);
maxValue = Math.max(max, maxValue);
- } catch (IOException e) {
- throw new IOException(e);
} finally {
IOUtils.closeQuietly(is);
}
[07/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample
data, and rename 'kylin_metadata' to 'kylin_default_instance'
Posted by li...@apache.org.
KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 'kylin_default_instance'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1a2a4a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1a2a4a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1a2a4a6
Branch: refs/heads/master-cdh5.7
Commit: a1a2a4a643d013471079ac0d353bbf179fd0d85c
Parents: c3d62e9
Author: Yang Li <li...@apache.org>
Authored: Sun Nov 27 18:26:33 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Tue Nov 29 07:25:25 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 2 +-
.../apache/kylin/common/KylinConfigBase.java | 11 +-
.../kylin/metadata/model/DataModelDesc.java | 3 +
.../kylin/metadata/model/MeasureDesc.java | 5 +
.../realization/IRealizationConstants.java | 2 +-
examples/sample_cube/create_sample_tables.sql | 21 +-
.../sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv | 10000 +++++++++
.../sample_cube/data/DEFAULT.KYLIN_SALES.csv | 20000 ++++++++---------
.../template/cube_desc/kylin_sales_cube.json | 157 +-
.../template/model_desc/kylin_sales_model.json | 50 +-
.../template/table/DEFAULT.KYLIN_ACCOUNT.json | 28 +
.../template/table/DEFAULT.KYLIN_SALES.json | 12 +-
.../test_case_data/sandbox/kylin.properties | 2 +-
.../apache/kylin/query/relnode/OLAPJoinRel.java | 25 +-
.../kylin/rest/security/AclHBaseStorage.java | 1 -
.../rest/security/RealAclHBaseStorage.java | 5 +-
.../apache/kylin/rest/service/QueryService.java | 6 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 2 +-
.../storage/hbase/util/StorageCleanupJob.java | 2 +-
.../apache/kylin/tool/StorageCleanupJob.java | 2 +-
20 files changed, 20246 insertions(+), 10090 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 1409394..131a725 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -18,7 +18,7 @@
### METADATA | ENV ###
# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
+kylin.metadata.url=kylin_default_instance@hbase
# Working folder in HDFS, make sure user has the right access to the hdfs directory
kylin.env.hdfs-working-dir=/kylin
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index aa5e6e1..7dcc771 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -188,13 +188,12 @@ abstract public class KylinConfigBase implements Serializable {
}
public String getMetadataUrlPrefix() {
- String hbaseMetadataUrl = getMetadataUrl();
- String defaultPrefix = "kylin_metadata";
+ String metadataUrl = getMetadataUrl();
+ String defaultPrefix = "kylin_default_instance";
- if (org.apache.commons.lang3.StringUtils.containsIgnoreCase(hbaseMetadataUrl, "@hbase")) {
- int cut = hbaseMetadataUrl.indexOf('@');
- String tmp = cut < 0 ? defaultPrefix : hbaseMetadataUrl.substring(0, cut);
- return tmp;
+ if (metadataUrl.endsWith("@hbase")) {
+ int cut = metadataUrl.lastIndexOf('@');
+ return metadataUrl.substring(0, cut);
} else {
return defaultPrefix;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index c2de5d6..ff92def 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -378,6 +378,9 @@ public class DataModelDesc extends RootPersistentEntity {
for (int i = 0; i < pks.length; i++) {
TblColRef col = dimTable.getColumn(pks[i]);
if (col == null) {
+ col = findColumn(pks[i]);
+ }
+ if (col == null || col.getTableRef().equals(dimTable) == false) {
throw new IllegalStateException("Can't find column " + pks[i] + " in table " + dimTable.getTableIdentity());
}
pkCols[i] = col;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index dcf6350..253b06b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.model;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -35,6 +36,8 @@ public class MeasureDesc {
@JsonProperty("function")
private FunctionDesc function;
@JsonProperty("dependent_measure_ref")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @Deprecated
private String dependentMeasureRef;
public String getName() {
@@ -53,10 +56,12 @@ public class MeasureDesc {
this.function = function;
}
+ @Deprecated
public String getDependentMeasureRef() {
return dependentMeasureRef;
}
+ @Deprecated
public void setDependentMeasureRef(String dependentMeasureRef) {
this.dependentMeasureRef = dependentMeasureRef;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index 2a3b01e..adc53a1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -28,7 +28,7 @@ public class IRealizationConstants {
/**
* For each cube htable, we leverage htable's metadata to keep track of
- * which kylin server(represented by its kylin_metadata prefix) owns this htable
+ * which kylin server(represented by its kylin_default_instance prefix) owns this htable
*/
public final static String HTableTag = "KYLIN_HOST";
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/create_sample_tables.sql
----------------------------------------------------------------------
diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql
index 3189b58..bdfc4d7 100644
--- a/examples/sample_cube/create_sample_tables.sql
+++ b/examples/sample_cube/create_sample_tables.sql
@@ -182,6 +182,19 @@ COUNTRY string
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
+DROP TABLE IF EXISTS DEFAULT.KYLIN_ACCOUNT;
+
+CREATE TABLE DEFAULT.KYLIN_ACCOUNT
+(
+ACCOUNT_ID bigint
+,ACCOUNT_BUYER_LEVEL int COMMENT 'Account Buyer Level'
+,ACCOUNT_SELLER_LEVEL int COMMENT 'Account Seller Level'
+,ACCOUNT_COUNTRY string COMMENT 'Account Country'
+,ACCOUNT_CONTACT string COMMENT 'Account Contact Info'
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+STORED AS TEXTFILE;
+
DROP TABLE IF EXISTS DEFAULT.KYLIN_SALES;
CREATE TABLE DEFAULT.KYLIN_SALES
@@ -195,16 +208,16 @@ TRANS_ID bigint
,PRICE decimal(19,4) COMMENT 'Order Price'
,ITEM_COUNT bigint COMMENT 'Number of Purchased Goods'
,SELLER_ID bigint COMMENT 'Seller ID'
-,BUYER_COUNTRY string COMMENT 'Buyer Country'
-,SELLER_COUNTRY string COMMENT 'Seller Country'
-,USER_ID string COMMENT 'System User ID'
-,REGION string COMMENT 'System User Region'
+,BUYER_ID bigint COMMENT 'Buyer ID'
+,OPS_USER_ID string COMMENT 'System User ID'
+,OPS_REGION string COMMENT 'System User Region'
)
COMMENT 'Sales order table, fact table'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
STORED AS TEXTFILE;
LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_SALES;
+LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_ACCOUNT;
LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_COUNTRY;
LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CAL_DT;
LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;
[34/50] [abbrv] kylin git commit: KYLIN-2236 let query cache honor
backdoortoggles in query request
Posted by li...@apache.org.
KYLIN-2236 let query cache honor backdoortoggles in query request
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8f3239bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8f3239bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8f3239bf
Branch: refs/heads/master-cdh5.7
Commit: 8f3239bf9ea4f1abd8d2c814967d425d4da68a45
Parents: 9556152
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Dec 5 13:43:03 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Dec 5 13:43:03 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 30 ++++++++++
.../apache/kylin/rest/request/SQLRequest.java | 63 +++++++-------------
.../apache/kylin/rest/service/QueryService.java | 1 +
3 files changed, 53 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8f3239bf/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index 4c847bf..2f8991b 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -261,6 +261,36 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
}
+
+ @Test
+ public void testResultSetWithMaxRows() throws Exception {
+ String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n" + " group by LSTG_FORMAT_NAME ";
+
+ Connection conn = getConnection();
+ Statement statement = conn.createStatement();
+ statement.setMaxRows(2);
+
+ statement.execute(sql);
+
+ ResultSet rs = statement.getResultSet();
+
+ int count = 0;
+ while (rs.next()) {
+ count++;
+ String lstg = rs.getString(1);
+ double gmv = rs.getDouble(2);
+ int trans_count = rs.getInt(3);
+
+ System.out.println("Get a line: LSTG_FORMAT_NAME=" + lstg + ", GMV=" + gmv + ", TRANS_CNT=" + trans_count);
+ }
+
+ Assert.assertTrue(count == 2);
+ statement.close();
+ rs.close();
+ conn.close();
+
+ }
+
private static class SystemPropertiesOverride {
HashMap<String, String> backup = new HashMap<String, String>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/8f3239bf/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index 96f5faa..bd8b7e2 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -83,50 +83,31 @@ public class SQLRequest implements Serializable {
this.acceptPartial = acceptPartial;
}
+
@Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (acceptPartial ? 1231 : 1237);
- result = prime * result + ((offset == null) ? 0 : offset.hashCode());
- result = prime * result + ((limit == null) ? 0 : limit.hashCode());
- result = prime * result + ((project == null) ? 0 : project.hashCode());
- result = prime * result + ((sql == null) ? 0 : sql.hashCode());
- return result;
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ SQLRequest that = (SQLRequest) o;
+
+ if (acceptPartial != that.acceptPartial) return false;
+ if (sql != null ? !sql.equals(that.sql) : that.sql != null) return false;
+ if (project != null ? !project.equals(that.project) : that.project != null) return false;
+ if (offset != null ? !offset.equals(that.offset) : that.offset != null) return false;
+ if (limit != null ? !limit.equals(that.limit) : that.limit != null) return false;
+ return backdoorToggles != null ? backdoorToggles.equals(that.backdoorToggles) : that.backdoorToggles == null;
+
}
@Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- SQLRequest other = (SQLRequest) obj;
- if (acceptPartial != other.acceptPartial)
- return false;
- if (offset == null) {
- if (other.offset != null)
- return false;
- } else if (!offset.equals(other.offset))
- return false;
- if (limit == null) {
- if (other.limit != null)
- return false;
- } else if (!limit.equals(other.limit))
- return false;
- if (project == null) {
- if (other.project != null)
- return false;
- } else if (!project.equals(other.project))
- return false;
- if (sql == null) {
- if (other.sql != null)
- return false;
- } else if (!sql.equals(other.sql))
- return false;
- return true;
+ public int hashCode() {
+ int result = sql != null ? sql.hashCode() : 0;
+ result = 31 * result + (project != null ? project.hashCode() : 0);
+ result = 31 * result + (offset != null ? offset.hashCode() : 0);
+ result = 31 * result + (limit != null ? limit.hashCode() : 0);
+ result = 31 * result + (acceptPartial ? 1 : 0);
+ result = 31 * result + (backdoorToggles != null ? backdoorToggles.hashCode() : 0);
+ return result;
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8f3239bf/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index e1787d7..8810c85 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -513,6 +513,7 @@ public class QueryService extends BasicService {
private void processStatementAttr(Statement s, SQLRequest sqlRequest) throws SQLException {
Integer statementMaxRows = BackdoorToggles.getStatementMaxRows();
if (statementMaxRows != null) {
+ logger.info("Setting current statement's max rows to {}", statementMaxRows);
s.setMaxRows(statementMaxRows);
}
}
[04/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample
data, and rename 'kylin_metadata' to 'kylin_default_instance'
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index eb17922..48eef46 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -6,27 +6,32 @@
"description" : null,
"null_string" : null,
"dimensions" : [ {
+ "name" : "TRANS_ID",
+ "table" : "DEFAULT.KYLIN_SALES",
+ "column" : "TRANS_ID",
+ "derived" : null
+ }, {
"name" : "CAL_DT",
"table" : "DEFAULT.KYLIN_CAL_DT",
"column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
+ "derived" : [ "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ]
}, {
"name" : "CATEGORY",
"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
"column" : "{FK}",
"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
}, {
- "name" : "CATEGORY_HIERARCHY",
+ "name" : "META_CATEG_NAME",
"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
"column" : "META_CATEG_NAME",
"derived" : null
}, {
- "name" : "CATEGORY_HIERARCHY",
+ "name" : "CATEG_LVL2_NAME",
"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
"column" : "CATEG_LVL2_NAME",
"derived" : null
}, {
- "name" : "CATEGORY_HIERARCHY",
+ "name" : "CATEG_LVL3_NAME",
"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
"column" : "CATEG_LVL3_NAME",
"derived" : null
@@ -36,14 +41,54 @@
"column" : "LSTG_FORMAT_NAME",
"derived" : null
}, {
- "name" : "USER_ID",
+ "name" : "SELLER_ID",
"table" : "DEFAULT.KYLIN_SALES",
- "column" : "USER_ID",
+ "column" : "SELLER_ID",
"derived" : null
}, {
- "name" : "REGION",
+ "name" : "BUYER_ID",
"table" : "DEFAULT.KYLIN_SALES",
- "column" : "REGION",
+ "column" : "BUYER_ID",
+ "derived" : null
+ }, {
+ "name" : "ACCOUNT_BUYER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL",
+ "derived" : null
+ }, {
+ "name" : "ACCOUNT_SELLER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL",
+ "derived" : null
+ }, {
+ "name" : "BUYER_COUNTRY",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY",
+ "derived" : null
+ }, {
+ "name" : "SELLER_COUNTRY",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY",
+ "derived" : null
+ }, {
+ "name" : "BUYER_COUNTRY_NAME",
+ "table" : "BUYER_COUNTRY",
+ "column" : "NAME",
+ "derived" : null
+ }, {
+ "name" : "SELLER_COUNTRY_NAME",
+ "table" : "SELLER_COUNTRY",
+ "column" : "NAME",
+ "derived" : null
+ }, {
+ "name" : "OPS_USER_ID",
+ "table" : "DEFAULT.KYLIN_SALES",
+ "column" : "OPS_USER_ID",
+ "derived" : null
+ }, {
+ "name" : "OPS_REGION",
+ "table" : "DEFAULT.KYLIN_SALES",
+ "column" : "OPS_REGION",
"derived" : null
} ],
"measures" : [ {
@@ -56,8 +101,7 @@
"next_parameter" : null
},
"returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
+ }
}, {
"name" : "GMV_MIN",
"function" : {
@@ -68,8 +112,7 @@
"next_parameter" : null
},
"returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
+ }
}, {
"name" : "GMV_MAX",
"function" : {
@@ -80,8 +123,7 @@
"next_parameter" : null
},
"returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
+ }
}, {
"name" : "TRANS_CNT",
"function" : {
@@ -92,8 +134,7 @@
"next_parameter" : null
},
"returntype" : "bigint"
- },
- "dependent_measure_ref" : null
+ }
}, {
"name" : "SELLER_CNT_HLL",
"function" : {
@@ -104,20 +145,7 @@
"next_parameter" : null
},
"returntype" : "hllc(10)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "SELLER_FORMAT_CNT",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME",
- "next_parameter" : null
- },
- "returntype" : "hllc(10)"
- },
- "dependent_measure_ref" : null
+ }
}, {
"name" : "TOP_SELLER",
"function" : {
@@ -132,13 +160,21 @@
}
},
"returntype" : "topn(100)"
- },
- "dependent_measure_ref" : null
+ }
} ],
"rowkey" : {
"rowkey_columns" : [ {
+ "column" : "BUYER_ID",
+ "encoding" : "integer:4"
+ }, {
+ "column" : "SELLER_ID",
+ "encoding" : "integer:4"
+ }, {
+ "column" : "TRANS_ID",
+ "encoding" : "integer:4"
+ }, {
"column" : "PART_DT",
- "encoding" : "dict"
+ "encoding" : "date"
}, {
"column" : "LEAF_CATEG_ID",
"encoding" : "dict"
@@ -152,17 +188,35 @@
"column" : "CATEG_LVL3_NAME",
"encoding" : "dict"
}, {
- "column" : "LSTG_FORMAT_NAME",
- "encoding" : "fixed_length:12"
+ "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
}, {
- "column" : "USER_ID",
+ "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
"encoding" : "dict"
}, {
- "column" : "REGION",
+ "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_COUNTRY.NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_COUNTRY.NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "LSTG_FORMAT_NAME",
"encoding" : "dict"
}, {
"column" : "LSTG_SITE_ID",
"encoding" : "dict"
+ }, {
+ "column" : "OPS_USER_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "OPS_REGION",
+ "encoding" : "dict"
} ]
},
"hbase_mapping" : {
@@ -170,21 +224,34 @@
"name" : "F1",
"columns" : [ {
"qualifier" : "M",
- "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TOP_SELLER" ]
+ "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
} ]
}, {
"name" : "F2",
"columns" : [ {
"qualifier" : "M",
- "measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT" ]
+ "measure_refs" : [ "SELLER_CNT_HLL", "TOP_SELLER" ]
} ]
} ]
},
"aggregation_groups" : [ {
- "includes" : [ "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "META_CATEG_NAME", "PART_DT" ],
+ "includes" : [ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
+ "mandatory_dims" : [ "PART_DT" ],
+ "joint_dims" : [ [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME" ], [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_COUNTRY.NAME" ],
+ [ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID" ], [ "OPS_USER_ID", "OPS_REGION" ] ]
+ }
+ }, {
+ "includes" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID",
+ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
"select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : [ ],
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID",
+ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
"joint_dims" : [ ]
}
} ],
@@ -194,6 +261,8 @@
"auto_merge_time_ranges" : null,
"retention_range" : 0,
"engine_type" : %default_engine_type%,
- "storage_type" : %default_storage_type%,
- "override_kylin_properties" : { }
+ "storage_type" : %default_engine_type%,
+ "override_kylin_properties" : {
+ "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/template/model_desc/kylin_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 3c4fa5a..2d3cfb6 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -15,16 +15,60 @@
"primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
"foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
}
+ }, {
+ "table" : "DEFAULT.KYLIN_ACCOUNT",
+ "alias" : "BUYER_ACCOUNT",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "ACCOUNT_ID" ],
+ "foreign_key" : [ "BUYER_ID" ]
+ }
+ }, {
+ "table" : "DEFAULT.KYLIN_ACCOUNT",
+ "alias" : "SELLER_ACCOUNT",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "ACCOUNT_ID" ],
+ "foreign_key" : [ "SELLER_ID" ]
+ }
+ }, {
+ "table" : "DEFAULT.KYLIN_COUNTRY",
+ "alias" : "BUYER_COUNTRY",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "COUNTRY" ],
+ "foreign_key" : [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY" ]
+ }
+ }, {
+ "table" : "DEFAULT.KYLIN_COUNTRY",
+ "alias" : "SELLER_COUNTRY",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "COUNTRY" ],
+ "foreign_key" : [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY" ]
+ }
}],
"dimensions" : [ {
+ "table" : "DEFAULT.KYLIN_SALES",
+ "columns" : [ "TRANS_ID", "SELLER_ID", "BUYER_ID", "PART_DT", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION" ]
+ }, {
"table" : "DEFAULT.KYLIN_CAL_DT",
"columns" : ["CAL_DT", "WEEK_BEG_DT"]
}, {
"table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
+ "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "SITE_ID" ]
}, {
- "table" : "DEFAULT.KYLIN_SALES",
- "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT", "USER_ID", "REGION" ]
+ "table" : "BUYER_ACCOUNT",
+ "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ]
+ }, {
+ "table" : "SELLER_ACCOUNT",
+ "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ]
+ }, {
+ "table" : "BUYER_COUNTRY",
+ "columns" : [ "COUNTRY", "NAME" ]
+ }, {
+ "table" : "SELLER_COUNTRY",
+ "columns" : [ "COUNTRY", "NAME" ]
} ],
"metrics": [
"PRICE",
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/template/table/DEFAULT.KYLIN_ACCOUNT.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_ACCOUNT.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_ACCOUNT.json
new file mode 100644
index 0000000..f1b7755
--- /dev/null
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_ACCOUNT.json
@@ -0,0 +1,28 @@
+{
+ "uuid" : "f386e39e-40d7-44c2-9eb3-41b365632231",
+
+ "name" : "KYLIN_ACCOUNT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "ACCOUNT_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "ACCOUNT_BUYER_LEVEL",
+ "datatype" : "int"
+ }, {
+ "id" : "3",
+ "name" : "ACCOUNT_SELLER_LEVEL",
+ "datatype" : "int"
+ }, {
+ "id" : "4",
+ "name" : "ACCOUNT_COUNTRY",
+ "datatype" : "string"
+ }, {
+ "id" : "5",
+ "name" : "ACCOUNT_CONTACT",
+ "datatype" : "string"
+ } ],
+ "database" : "DEFAULT",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
index fbeafb3..9286215 100644
--- a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
@@ -40,19 +40,15 @@
"datatype" : "bigint"
}, {
"id" : "10",
- "name" : "BUYER_COUNTRY",
- "datatype" : "string"
+ "name" : "BUYER_ID",
+ "datatype" : "bigint"
}, {
"id" : "11",
- "name" : "SELLER_COUNTRY",
+ "name" : "OPS_USER_ID",
"datatype" : "string"
}, {
"id" : "12",
- "name" : "USER_ID",
- "datatype" : "string"
- }, {
- "id" : "13",
- "name" : "REGION",
+ "name" : "OPS_REGION",
"datatype" : "string"
} ],
"database" : "DEFAULT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index dc7132a..20bc427 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -39,7 +39,7 @@ kylin.source.hive.client=cli
### STORAGE ###
# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase
+kylin.metadata.url=kylin_default_instance@hbase
# The storage for final cube file in hbase
kylin.storage.url=hbase
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
index 4c0a1dc..6869858 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java
@@ -70,6 +70,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
private OLAPContext context;
private ColumnRowType columnRowType;
+ private int columnRowTypeLeftRightCut;
private boolean isTopJoin;
private boolean hasSubQuery;
@@ -180,6 +181,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
ColumnRowType leftColumnRowType = olapLeft.getColumnRowType();
columns.addAll(leftColumnRowType.getAllColumns());
+ this.columnRowTypeLeftRightCut = columns.size();
+
OLAPRel olapRight = (OLAPRel) this.right;
ColumnRowType rightColumnRowType = olapRight.getColumnRowType();
columns.addAll(rightColumnRowType.getAllColumns());
@@ -198,21 +201,13 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
List<TblColRef> pkCols = new ArrayList<TblColRef>();
List<String> fks = new ArrayList<String>();
List<TblColRef> fkCols = new ArrayList<TblColRef>();
- String factTable = context.firstTableScan.getTableName();
for (Map.Entry<TblColRef, TblColRef> columnPair : joinColumns.entrySet()) {
TblColRef fromCol = columnPair.getKey();
TblColRef toCol = columnPair.getValue();
- if (factTable.equalsIgnoreCase(fromCol.getTable())) {
- fks.add(fromCol.getName());
- fkCols.add(fromCol);
- pks.add(toCol.getName());
- pkCols.add(toCol);
- } else {
- fks.add(toCol.getName());
- fkCols.add(toCol);
- pks.add(fromCol.getName());
- pkCols.add(fromCol);
- }
+ fks.add(fromCol.getName());
+ fkCols.add(fromCol);
+ pks.add(toCol.getName());
+ pkCols.add(toCol);
}
JoinDesc join = new JoinDesc();
@@ -237,7 +232,11 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel {
TblColRef col0 = columnRowType.getColumnByIndex(op0.getIndex());
RexInputRef op1 = (RexInputRef) operands.get(1);
TblColRef col1 = columnRowType.getColumnByIndex(op1.getIndex());
- joinColumns.put(col0, col1);
+ // map left => right
+ if (op0.getIndex() < columnRowTypeLeftRightCut)
+ joinColumns.put(col0, col1);
+ else
+ joinColumns.put(col1, col0);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index 38f299e..ea68855 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.hbase.client.HTableInterface;
/**
*/
public interface AclHBaseStorage {
- String DEFAULT_TABLE_PREFIX = "kylin_metadata";
String ACL_INFO_FAMILY = "i";
String ACL_ACES_FAMILY = "a";
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index ab18029..1d520c4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -38,10 +38,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
@Override
public String prepareHBaseTable(Class<?> clazz) throws IOException {
- String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ String metadataUrl = kylinConfig.getMetadataUrl();
int cut = metadataUrl.indexOf('@');
- String tableNameBase = cut < 0 ? DEFAULT_TABLE_PREFIX : metadataUrl.substring(0, cut);
hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+ String tableNameBase = kylinConfig.getMetadataUrlPrefix();
if (clazz == AclService.class) {
aclTableName = tableNameBase + ACL_TABLE_NAME;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 81af044..4389a5d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -104,7 +104,6 @@ public class QueryService extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(QueryService.class);
public static final String USER_QUERY_FAMILY = "q";
- private static final String DEFAULT_TABLE_PREFIX = "kylin_metadata";
private static final String USER_TABLE_NAME = "_user";
private static final String USER_QUERY_COLUMN = "c";
@@ -129,11 +128,12 @@ public class QueryService extends BasicService {
}
public QueryService() {
- String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl();
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ String metadataUrl = kylinConfig.getMetadataUrl();
// split TABLE@HBASE_URL
int cut = metadataUrl.indexOf('@');
- String tableNameBase = cut < 0 ? DEFAULT_TABLE_PREFIX : metadataUrl.substring(0, cut);
hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);
+ String tableNameBase = kylinConfig.getMetadataUrlPrefix();
userTableName = tableNameBase + USER_TABLE_NAME;
badQueryDetector.start();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 82cc93c..1d19983 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -58,7 +58,7 @@ public class HBaseResourceStore extends ResourceStore {
private static final Logger logger = LoggerFactory.getLogger(HBaseResourceStore.class);
- private static final String DEFAULT_TABLE_NAME = "kylin_metadata";
+ private static final String DEFAULT_TABLE_NAME = "kylin_default_instance";
private static final String FAMILY = "f";
private static final byte[] B_FAMILY = Bytes.toBytes(FAMILY);
private static final String COLUMN = "c";
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 90080b6..9fe5a23 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -185,7 +185,7 @@ public class StorageCleanupJob extends AbstractApplication {
// GlobFilter filter = new
// GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
// + "/kylin-.*");
- // TODO: when first use, /kylin/kylin_metadata does not exist.
+ // TODO: when first use, /kylin/kylin_default_instance does not exist.
FileStatus[] fStatus = fs.listStatus(new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()));
for (FileStatus status : fStatus) {
String path = status.getPath().getName();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index b4accac..05e0142 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -190,7 +190,7 @@ public class StorageCleanupJob extends AbstractApplication {
// GlobFilter filter = new
// GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
// + "/kylin-.*");
- // TODO: when first use, /kylin/kylin_metadata does not exist.
+ // TODO: when first use, /kylin/kylin_default_instance does not exist.
FileStatus[] fStatus = fs.listStatus(new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()));
for (FileStatus status : fStatus) {
String path = status.getPath().getName();
[32/50] [abbrv] kylin git commit: minor,
change isReverse visibility in BuiltInFunctionTupleFilter.java
Posted by li...@apache.org.
minor, change isReverse visibility in BuiltInFunctionTupleFilter.java
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/00a4c506
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/00a4c506
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/00a4c506
Branch: refs/heads/master-cdh5.7
Commit: 00a4c5064bfe894bb8c106a214277d4d1d0f3c70
Parents: a03eec8
Author: Hongbin Ma <ma...@apache.org>
Authored: Sun Dec 4 21:41:07 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Dec 4 21:41:07 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/00a4c506/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index f6e687b..b678394 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -47,7 +47,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
protected Method method;
protected List<Serializable> methodParams;
protected boolean isValidFunc = false;
- private boolean isReversed = false;
+ protected boolean isReversed = false;
public BuiltInFunctionTupleFilter(String name) {
this(name, null);
[17/50] [abbrv] kylin git commit: refine mapper and reducer log
Posted by li...@apache.org.
refine mapper and reducer log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/58224921
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/58224921
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/58224921
Branch: refs/heads/master-cdh5.7
Commit: 58224921d896e4479f5d034d43c044aacaf14200
Parents: 28ba1ea
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Dec 1 18:15:46 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Dec 1 18:15:46 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/mr/KylinMapper.java | 12 +++++++++---
.../java/org/apache/kylin/engine/mr/KylinReducer.java | 13 ++++++++++---
.../apache/kylin/engine/mr/steps/CuboidReducer.java | 11 ++++-------
.../kylin/engine/mr/steps/HiveToBaseCuboidMapper.java | 6 ------
.../kylin/engine/mr/steps/InMemCuboidMapper.java | 7 +------
.../kylin/engine/mr/steps/InMemCuboidReducer.java | 10 +++++-----
.../apache/kylin/engine/mr/steps/NDCuboidMapper.java | 10 ++++------
7 files changed, 33 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
index a01f7a2..2b564e9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinMapper.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.kylin.engine.mr.common.BatchConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,6 +31,8 @@ import org.slf4j.LoggerFactory;
public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
private static final Logger logger = LoggerFactory.getLogger(KylinMapper.class);
+ protected int mapCounter = 0;
+
protected void bindCurrentConfiguration(Configuration conf) {
logger.info("The conf for current mapper will be " + System.identityHashCode(conf));
HadoopUtil.setCurrentConfiguration(conf);
@@ -38,6 +41,9 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN,
@Override
final public void map(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
try {
+ if (mapCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
+ logger.info("Accepting Mapper Key with ordinal: " + mapCounter);
+ }
doMap(key, value, context);
} catch (IOException ex) { // KYLIN-2170
logger.error("", ex);
@@ -53,11 +59,11 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN,
throw ex;
}
}
-
+
protected void doMap(KEYIN key, VALUEIN value, Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
super.map(key, value, context);
}
-
+
@Override
final protected void cleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
try {
@@ -76,7 +82,7 @@ public class KylinMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Mapper<KEYIN,
throw ex;
}
}
-
+
protected void doCleanup(Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index 2b63ce0..cb2d7a7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.kylin.engine.mr.common.BatchConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,7 +30,9 @@ import org.slf4j.LoggerFactory;
*/
public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
private static final Logger logger = LoggerFactory.getLogger(KylinReducer.class);
-
+
+ protected int reduceCounter = 0;
+
protected void bindCurrentConfiguration(Configuration conf) {
HadoopUtil.setCurrentConfiguration(conf);
}
@@ -37,6 +40,10 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYI
@Override
final public void reduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
try {
+ if (reduceCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
+ logger.info("Accepting Mapper Key with ordinal: " + reduceCounter);
+ }
+
doReduce(key, values, context);
} catch (IOException ex) { // KYLIN-2170
logger.error("", ex);
@@ -52,11 +59,11 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYI
throw ex;
}
}
-
+
protected void doReduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
super.reduce(key, values, context);
}
-
+
@Override
final protected void cleanup(Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
index 9543f0a..b1d4aaa 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidReducer.java
@@ -50,11 +50,11 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text> {
private BufferedMeasureCodec codec;
private MeasureAggregators aggs;
- private int counter;
private int cuboidLevel;
private boolean[] needAggr;
private Object[] input;
private Object[] result;
+ private int vcounter;
private Text outputValue = new Text();
@@ -90,6 +90,9 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text> {
aggs.reset();
for (Text value : values) {
+ if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
+ logger.info("Handling value with ordinal: " + vcounter);
+ }
codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), input);
if (cuboidLevel > 0) {
aggs.aggregate(input, needAggr);
@@ -103,11 +106,5 @@ public class CuboidReducer extends KylinReducer<Text, Text, Text, Text> {
outputValue.set(valueBuf.array(), 0, valueBuf.position());
context.write(key, outputValue);
-
- counter++;
- if (counter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
- logger.info("Handled " + counter + " records!");
- }
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
index 9fa20ae..428f878 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/HiveToBaseCuboidMapper.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
import org.apache.kylin.engine.mr.MRUtil;
-import org.apache.kylin.engine.mr.common.BatchConstants;
/**
* @author George Song (ysong1)
@@ -39,11 +38,6 @@ public class HiveToBaseCuboidMapper<KEYIN> extends BaseCuboidMapperBase<KEYIN, O
@Override
public void doMap(KEYIN key, Object value, Context context) throws IOException, InterruptedException {
- counter++;
- if (counter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
- logger.info("Handled " + counter + " records!");
- }
-
String[] row = flatTableInputFormat.parseMapperInput(value);
try {
outputKV(row, context);
http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index 15bfd2e..116d5e0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -64,7 +64,6 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArr
private CubeSegment cubeSegment;
private IMRTableInputFormat flatTableInputFormat;
- private int counter;
private BlockingQueue<List<String>> queue = new ArrayBlockingQueue<List<String>>(64);
private Future<?> future;
@@ -120,10 +119,6 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArr
while (!future.isDone()) {
if (queue.offer(rowAsList, 1, TimeUnit.SECONDS)) {
- counter++;
- if (counter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
- logger.info("Handled " + counter + " records!");
- }
break;
}
}
@@ -131,7 +126,7 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArr
@Override
protected void doCleanup(Context context) throws IOException, InterruptedException {
- logger.info("Totally handled " + counter + " records!");
+ logger.info("Totally handled " + mapCounter + " records!");
while (!future.isDone()) {
if (queue.offer(Collections.<String> emptyList(), 1, TimeUnit.SECONDS)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
index d0a7062..04c9e90 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidReducer.java
@@ -46,10 +46,11 @@ public class InMemCuboidReducer extends KylinReducer<ByteArrayWritable, ByteArra
private BufferedMeasureCodec codec;
private MeasureAggregators aggs;
- private int counter;
private Object[] input;
private Object[] result;
+ private int vcounter;
+
private Text outputKey;
private Text outputValue;
@@ -78,6 +79,9 @@ public class InMemCuboidReducer extends KylinReducer<ByteArrayWritable, ByteArra
aggs.reset();
for (ByteArrayWritable value : values) {
+ if (vcounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
+ logger.info("Handling value with ordinal: " + vcounter);
+ }
codec.decode(value.asBuffer(), input);
aggs.aggregate(input);
}
@@ -92,10 +96,6 @@ public class InMemCuboidReducer extends KylinReducer<ByteArrayWritable, ByteArra
context.write(outputKey, outputValue);
- counter++;
- if (counter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
- logger.info("Handled " + counter + " records!");
- }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/58224921/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
index 8107e52..01cdd4a 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/NDCuboidMapper.java
@@ -129,18 +129,16 @@ public class NDCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
// if still empty or null
if (myChildren == null || myChildren.size() == 0) {
context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Skipped records").increment(1L);
- skipCounter++;
- if (skipCounter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
- logger.info("Skipped " + skipCounter + " records!");
+ if (skipCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
+ logger.info("Skipping record with ordinal: " + skipCounter);
}
return;
}
context.getCounter(BatchConstants.MAPREDUCE_COUNTER_GROUP_NAME, "Processed records").increment(1L);
- handleCounter++;
- if (handleCounter % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
- logger.info("Handled " + handleCounter + " records!");
+ if (handleCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
+ logger.info("Handling record with ordinal: " + handleCounter);
}
for (Long child : myChildren) {
[41/50] [abbrv] kylin git commit: minor,
generate item_count value for sample data
Posted by li...@apache.org.
minor, generate item_count value for sample data
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d2aaf270
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d2aaf270
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d2aaf270
Branch: refs/heads/master-cdh5.7
Commit: d2aaf2705a9035be9dbef8205fc9a1358b4316c2
Parents: 9968c22
Author: Yiming Liu <li...@gmail.com>
Authored: Fri Dec 2 16:39:00 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Dec 6 13:50:28 2016 +0800
----------------------------------------------------------------------
.../sample_cube/data/DEFAULT.KYLIN_SALES.csv | 20000 ++++++++---------
1 file changed, 10000 insertions(+), 10000 deletions(-)
----------------------------------------------------------------------
[13/50] [abbrv] kylin git commit: KYLIN-2217 Reducers build
dictionaries locally
Posted by li...@apache.org.
KYLIN-2217 Reducers build dictionaries locally
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1af08e4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1af08e4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1af08e4b
Branch: refs/heads/master-cdh5.7
Commit: 1af08e4b8875d33bfc5dd124fed72d6042456c32
Parents: b1b90ad
Author: xiefan46 <95...@qq.com>
Authored: Wed Nov 23 09:48:55 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Nov 30 15:31:58 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 9 +-
.../java/org/apache/kylin/cube/CubeManager.java | 3 +-
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 25 ++-
.../apache/kylin/dict/DictionaryManager.java | 17 +-
.../apache/kylin/dict/DictionaryProvider.java | 28 ++++
.../dict/DictionaryReducerLocalGenerator.java | 156 +++++++++++++++++++
.../dict/IDictionaryReducerLocalBuilder.java | 31 ++++
.../kylin/dict/DictionaryProviderTest.java | 109 +++++++++++++
.../storage/translate/ColumnValueRange.java | 2 +-
.../engine/mr/steps/CreateDictionaryJob.java | 44 +++++-
.../mr/steps/FactDistinctColumnsReducer.java | 123 +++++++++++++--
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 54 +++----
.../storage/hbase/cube/v1/CubeStorageQuery.java | 6 +-
13 files changed, 547 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7dcc771..766c04d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -719,7 +719,14 @@ abstract public class KylinConfigBase implements Serializable {
//UHC: ultra high cardinality columns, contain the ShardByColumns and the GlobalDictionaryColumns
public int getUHCReducerCount() {
- return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "3"));
+ return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
+ }
+
+ public boolean isReducerLocalBuildDict() {
+ if (getUHCReducerCount() != 1) {
+ return false;
+ }
+ return Boolean.parseBoolean(getOptional("kylin.engine.mr.reducer-local-build-dict", "true"));
}
public String getYarnStatusCheckUrl() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index b4422d2..119a21a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -213,6 +213,7 @@ public class CubeManager implements IRealizationProvider {
return result;
}
+
public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, ReadableTable inpTable) throws IOException {
CubeDesc cubeDesc = cubeSeg.getCubeDesc();
if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
@@ -221,6 +222,7 @@ public class CubeManager implements IRealizationProvider {
String builderClass = cubeDesc.getDictionaryBuilderClass(col);
DictionaryInfo dictInfo = getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, builderClass);
+
saveDictionaryInfo(cubeSeg, col, dictInfo);
return dictInfo;
}
@@ -266,7 +268,6 @@ public class CubeManager implements IRealizationProvider {
} catch (IOException e) {
throw new IllegalStateException("Failed to get dictionary for cube segment" + cubeSeg + ", col" + col, e);
}
-
return (Dictionary<String>) info.getDictionaryObject();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
index a6aeb96..a4e1df0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
@@ -22,11 +22,13 @@ import java.io.IOException;
import java.util.Set;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.DictionaryManager;
+import org.apache.kylin.dict.DictionaryProvider;
import org.apache.kylin.dict.DistinctColumnValuesProvider;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -44,21 +46,30 @@ public class DictionaryGeneratorCLI {
private static final Logger logger = LoggerFactory.getLogger(DictionaryGeneratorCLI.class);
- public static void processSegment(KylinConfig config, String cubeName, String segmentID, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+ public static void processSegment(KylinConfig config, String cubeName, String segmentID, DistinctColumnValuesProvider factTableValueProvider, DictionaryProvider dictProvider) throws IOException {
CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
CubeSegment segment = cube.getSegmentById(segmentID);
- processSegment(config, segment, factTableValueProvider);
+ processSegment(config, segment, factTableValueProvider, dictProvider);
}
- private static void processSegment(KylinConfig config, CubeSegment cubeSeg, DistinctColumnValuesProvider factTableValueProvider) throws IOException {
+ private static void processSegment(KylinConfig config, CubeSegment cubeSeg, DistinctColumnValuesProvider factTableValueProvider, DictionaryProvider dictProvider) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(config);
// dictionary
for (TblColRef col : cubeSeg.getCubeDesc().getAllColumnsNeedDictionaryBuilt()) {
logger.info("Building dictionary for " + col);
ReadableTable inpTable = decideInputTable(cubeSeg.getModel(), col, factTableValueProvider);
- cubeMgr.buildDictionary(cubeSeg, col, inpTable);
+ if (config.isReducerLocalBuildDict() && dictProvider != null) {
+ Dictionary<String> dict = dictProvider.getDictionary(col);
+ if (dict != null) {
+ cubeMgr.saveDictionary(cubeSeg, col, inpTable, dict);
+ } else {
+ cubeMgr.buildDictionary(cubeSeg, col, inpTable);
+ }
+ } else {
+ cubeMgr.buildDictionary(cubeSeg, col, inpTable);
+ }
}
// snapshot
@@ -68,19 +79,19 @@ public class DictionaryGeneratorCLI {
if (cubeSeg.getModel().isLookupTable(table))
toSnapshot.add(table.getTableIdentity());
}
-
+
for (String tableIdentity : toSnapshot) {
logger.info("Building snapshot of " + tableIdentity);
cubeMgr.buildSnapshotTable(cubeSeg, tableIdentity);
}
}
-
+
private static ReadableTable decideInputTable(DataModelDesc model, TblColRef col, DistinctColumnValuesProvider factTableValueProvider) {
KylinConfig config = model.getConfig();
DictionaryManager dictMgr = DictionaryManager.getInstance(config);
TblColRef srcCol = dictMgr.decideSourceData(model, col);
String srcTable = srcCol.getTable();
-
+
ReadableTable inpTable;
if (model.isFactTable(srcTable)) {
inpTable = factTableValueProvider.getDistinctValuesFor(srcCol);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 6178234..0caef14 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -87,6 +87,7 @@ public class DictionaryManager {
private KylinConfig config;
private LoadingCache<String, DictionaryInfo> dictCache; // resource
+
// path ==>
// DictionaryInfo
@@ -275,10 +276,12 @@ public class DictionaryManager {
return buildDictionary(model, col, inpTable, null);
}
+
public DictionaryInfo buildDictionary(DataModelDesc model, TblColRef col, ReadableTable inpTable, String builderClass) throws IOException {
if (inpTable.exists() == false)
return null;
+
logger.info("building dictionary for " + col);
DictionaryInfo dictInfo = createDictionaryInfo(model, col, inpTable);
@@ -291,6 +294,12 @@ public class DictionaryManager {
logger.info("Building dictionary object " + JsonUtil.writeValueAsString(dictInfo));
Dictionary<String> dictionary;
+ dictionary = buildDictFromReadableTable(inpTable, dictInfo, builderClass, col);
+ return trySaveNewDict(dictionary, dictInfo);
+ }
+
+ private Dictionary<String> buildDictFromReadableTable(ReadableTable inpTable, DictionaryInfo dictInfo, String builderClass, TblColRef col) throws IOException {
+ Dictionary<String> dictionary;
IDictionaryValueEnumerator columnValueEnumerator = null;
try {
columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(), dictInfo.getSourceColumnIndex());
@@ -304,7 +313,7 @@ public class DictionaryManager {
if (columnValueEnumerator != null)
columnValueEnumerator.close();
}
- return trySaveNewDict(dictionary, dictInfo);
+ return dictionary;
}
public DictionaryInfo saveDictionary(DataModelDesc model, TblColRef col, ReadableTable inpTable, Dictionary<String> dictionary) throws IOException {
@@ -336,19 +345,19 @@ public class DictionaryManager {
// FK on fact table and join type is inner, use PK from lookup instead
if (model.isFactTable(col.getTable()) == false)
return col;
-
+
// find a lookup table that the col joins as FK
for (TableRef lookup : model.getLookupTables()) {
JoinDesc lookupJoin = model.getJoinByPKSide(lookup);
int find = ArrayUtils.indexOf(lookupJoin.getForeignKeyColumns(), col);
if (find < 0)
continue;
-
+
// make sure the joins are all inner up to the root
if (isAllInnerJoinsToRoot(model, lookupJoin))
return lookupJoin.getPrimaryKeyColumns()[find];
}
-
+
return col;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryProvider.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryProvider.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryProvider.java
new file mode 100644
index 0000000..6387535
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryProvider.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ * Created by xiefan on 16-11-23.
+ */
+public interface DictionaryProvider {
+ public Dictionary<String> getDictionary(TblColRef col);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryReducerLocalGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryReducerLocalGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryReducerLocalGenerator.java
new file mode 100644
index 0000000..35d379a
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryReducerLocalGenerator.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.dict;
+
+import com.google.common.base.Preconditions;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.datatype.DataType;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+
+/**
+ * Created by xiefan on 16-11-16.
+ *
+ * TODO:sample,mergeDict
+ */
+public class DictionaryReducerLocalGenerator {
+
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DictionaryReducerLocalGenerator.class);
+
+ private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd", "yyyyMMdd" };
+
+ public static IDictionaryReducerLocalBuilder getBuilder(DataType dataType) {
+ Preconditions.checkNotNull(dataType, "dataType cannot be null");
+
+ IDictionaryReducerLocalBuilder builder;
+ if (dataType.isDateTimeFamily()) {
+ if (dataType.isDate())
+ builder = new DateDictBuilder();
+ else
+ builder = new TimeDictBuilder();
+ } else if (dataType.isNumberFamily()) {
+ builder = new NumberDictBuilder(0);
+ } else {
+ builder = new StringDictBuilder(0);
+ }
+ return builder;
+ }
+
+ private static class DateDictBuilder implements IDictionaryReducerLocalBuilder {
+
+ private static final String[] DATE_PATTERNS = new String[] { "yyyy-MM-dd", "yyyyMMdd" };
+
+ private String matchPattern = null;
+
+ private boolean isRecognizeFormat = false;
+
+ private SimpleDateFormat sdf;
+
+ @Override
+ public Dictionary<String> build(int baseId) throws Exception {
+ if (isRecognizeFormat) {
+ return new DateStrDictionary(matchPattern, baseId);
+ } else {
+ throw new IllegalStateException("Date format not match");
+ }
+ }
+
+ @Override
+ public void addValue(String value) throws Exception {
+ if (matchPattern == null) { //init match pattern
+ for (String ptn : DATE_PATTERNS) {
+ matchPattern = ptn;
+ SimpleDateFormat sdf = new SimpleDateFormat(ptn);
+ try {
+ sdf.parse(value);
+ isRecognizeFormat = true;
+ break;
+ } catch (ParseException e) {
+
+ }
+ }
+ sdf = new SimpleDateFormat(matchPattern);
+ }
+ if (!isRecognizeFormat) {
+ throw new IllegalStateException("Date format not match");
+ }
+ try {
+ sdf.parse(value);
+ } catch (ParseException e) {
+ isRecognizeFormat = false;
+ logger.info("Unrecognized date value: " + value);
+ }
+ }
+
+ }
+
+ private static class TimeDictBuilder implements IDictionaryReducerLocalBuilder {
+
+ @Override
+ public Dictionary<String> build(int baseId) {
+ return new TimeStrDictionary();
+ }
+
+ @Override
+ public void addValue(String value) {
+
+ }
+
+ }
+
+ private static class StringDictBuilder implements IDictionaryReducerLocalBuilder {
+
+ private TrieDictionaryForestBuilder<String> builder;
+
+ public StringDictBuilder(int baseId) {
+ builder = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), 0);
+ }
+
+ @Override
+ public Dictionary<String> build(int baseId) {
+ return builder.build();
+ }
+
+ @Override
+ public void addValue(String value) {
+ builder.addValue(value);
+ }
+
+ }
+
+ public static class NumberDictBuilder implements IDictionaryReducerLocalBuilder {
+
+ private NumberDictionaryForestBuilder builder;
+
+ public NumberDictBuilder(int baseId) {
+ builder = new NumberDictionaryForestBuilder(baseId);
+ }
+
+ @Override
+ public Dictionary<String> build(int baseId) {
+ return builder.build();
+ }
+
+ @Override
+ public void addValue(String value) {
+ builder.addValue(value);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryReducerLocalBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryReducerLocalBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryReducerLocalBuilder.java
new file mode 100644
index 0000000..19b1d28
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/IDictionaryReducerLocalBuilder.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.Dictionary;
+
+/**
+ * Created by xiefan on 16-11-16.
+ */
+public interface IDictionaryReducerLocalBuilder {
+ Dictionary<String> build(int baseId) throws Exception;
+
+ void addValue(String value) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
new file mode 100644
index 0000000..0225737
--- /dev/null
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryProviderTest.java
@@ -0,0 +1,109 @@
+package org.apache.kylin.dict;
+
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.Test;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.lang.reflect.ParameterizedType;
+import java.util.Arrays;
+import java.util.Iterator;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Created by xiefan on 16-11-23.
+ */
+public class DictionaryProviderTest {
+
+ @Test
+ public void testReadWrite() throws Exception{
+ //string dict
+ Dictionary<String> dict = getDict(DataType.getType("string"),
+ Arrays.asList(new String[]{"a","b"}).iterator());
+ readWriteTest(dict);
+ //number dict
+ Dictionary<String> dict2 = getDict(DataType.getType("long"),
+ Arrays.asList(new String[]{"1","2"}).iterator());
+ readWriteTest(dict2);
+
+ //date dict
+ Dictionary<String> dict3 = getDict(DataType.getType("datetime"),
+ Arrays.asList(new String[]{"20161122","20161123"}).iterator());
+ readWriteTest(dict3);
+
+ //date dict
+ Dictionary<String> dict4 = getDict(DataType.getType("datetime"),
+ Arrays.asList(new String[]{"2016-11-22","2016-11-23"}).iterator());
+ readWriteTest(dict4);
+
+ //date dict
+ try {
+ Dictionary<String> dict5 = getDict(DataType.getType("date"),
+ Arrays.asList(new String[]{"2016-11-22", "20161122"}).iterator());
+ readWriteTest(dict5);
+ fail("Date format not correct.Should throw exception");
+ }catch (IllegalStateException e){
+ //correct
+ }
+ }
+
+ @Test
+ public void testReadWriteTime(){
+ System.out.println(Long.MAX_VALUE);
+ System.out.println(Long.MIN_VALUE);
+ }
+
+
+ private Dictionary<String> getDict(DataType type, Iterator<String> values) throws Exception{
+ IDictionaryReducerLocalBuilder builder = DictionaryReducerLocalGenerator.getBuilder(type);
+ while(values.hasNext()){
+ builder.addValue(values.next());
+ }
+ return builder.build(0);
+ }
+
+ private void readWriteTest(Dictionary<String> dict) throws Exception{
+ final String path = "src/test/resources/dict/tmp_dict";
+ File f = new File(path);
+ f.deleteOnExit();
+ f.createNewFile();
+ String dictClassName = dict.getClass().getName();
+ DataOutputStream out = new DataOutputStream(new FileOutputStream(f));
+ out.writeUTF(dictClassName);
+ dict.write(out);
+ out.close();
+ //read dict
+ DataInputStream in = null;
+ Dictionary<String> dict2 = null;
+ try {
+ File f2 = new File(path);
+ in = new DataInputStream(new FileInputStream(f2));
+ String dictClassName2 = in.readUTF();
+ dict2 = (Dictionary<String>) ClassUtil.newInstance(dictClassName2);
+ dict2.readFields(in);
+ }catch(IOException e){
+ e.printStackTrace();
+ }finally {
+ if(in != null){
+ try {
+ in.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ assertTrue(dict.equals(dict2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java
index 0dc1afa..56b1106 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/ColumnValueRange.java
@@ -168,7 +168,7 @@ public class ColumnValueRange {
// remove invalid EQ/IN values and round start/end according to dictionary
public void preEvaluateWithDict(Dictionary<String> dict) {
- if (dict == null)
+ if (dict == null || dict.getSize() == 0)
return;
if (equalValues != null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index 5d7cb21..63005f9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -19,15 +19,25 @@
package org.apache.kylin.engine.mr.steps;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.cli.DictionaryGeneratorCLI;
+import org.apache.kylin.dict.DictionaryProvider;
import org.apache.kylin.dict.DistinctColumnValuesProvider;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.SortedColumnDFSFile;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
+import java.io.IOException;
+
/**
* @author ysong1
*/
@@ -48,13 +58,45 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
final String segmentID = getOptionValue(OPTION_SEGMENT_ID);
final String factColumnsInputPath = getOptionValue(OPTION_INPUT_PATH);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
+ final KylinConfig config = KylinConfig.getInstanceFromEnv();
DictionaryGeneratorCLI.processSegment(config, cubeName, segmentID, new DistinctColumnValuesProvider() {
@Override
public ReadableTable getDistinctValuesFor(TblColRef col) {
return new SortedColumnDFSFile(factColumnsInputPath + "/" + col.getName(), col.getType());
}
+ }, new DictionaryProvider() {
+
+ @Override
+ public Dictionary<String> getDictionary(TblColRef col) {
+ if (!config.isReducerLocalBuildDict()) {
+ return null;
+ }
+ FSDataInputStream is = null;
+ try {
+ Path colDir = new Path(factColumnsInputPath, col.getName());
+ Path outputFile = new Path(colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
+ Configuration conf = HadoopUtil.getCurrentConfiguration();
+ FileSystem fs = HadoopUtil.getFileSystem(outputFile.getName());
+ is = fs.open(outputFile);
+ String dictClassName = is.readUTF();
+ Dictionary<String> dict = (Dictionary<String>) ClassUtil.newInstance(dictClassName);
+ dict.readFields(is);
+ logger.info("DictionaryProvider read dict form file : " + outputFile.getName());
+ return dict;
+ } catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ } finally {
+ if (is != null) {
+ try {
+ is.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
});
return returnCode;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 6e24d61..5511626 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -36,14 +38,19 @@ import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.dict.DictionaryReducerLocalGenerator;
+import org.apache.kylin.dict.IDictionaryReducerLocalBuilder;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CubeStatsWriter;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,10 +80,19 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
+ //local build dict
+ private boolean isReducerLocalBuildDict;
+ private IDictionaryReducerLocalBuilder builder;
+ private FastDateFormat dateFormat;
+ private long timeMaxValue = Long.MIN_VALUE;
+ private long timeMinValue = Long.MAX_VALUE;
+ public static final String DICT_FILE_POSTFIX = ".RLD";
+ public static final String PARTITION_COL_INFO_FILE_POSTFIX = ".PCI";
+ private boolean isPartitionCol = false;
+
@Override
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
-
Configuration conf = context.getConfiguration();
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
@@ -102,14 +118,36 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
} else if (collectStatistics && (taskId == numberOfTasks - 2)) {
// partition col
isStatistics = false;
+ isPartitionCol = true;
col = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
colValues = Lists.newLinkedList();
+ DataType partitionColType = col.getType();
+ if (partitionColType.isDate()) {
+ dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATE_PATTERN);
+ } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) {
+ dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ } else if (partitionColType.isStringFamily()) {
+ String partitionDateFormat = cubeDesc.getModel().getPartitionDesc().getPartitionDateFormat();
+ if (StringUtils.isEmpty(partitionDateFormat)) {
+ partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
+ }
+ dateFormat = DateFormat.getDateFormat(partitionDateFormat);
+ } else {
+ throw new IllegalStateException("Type " + partitionColType + " is not valid partition column type");
+ }
} else {
// col
isStatistics = false;
col = columnList.get(ReducerIdToColumnIndex.get(taskId));
colValues = Lists.newLinkedList();
}
+
+ //local build dict
+ isReducerLocalBuildDict = config.isReducerLocalBuildDict();
+ if (col != null && isReducerLocalBuildDict) {
+ builder = DictionaryReducerLocalGenerator.getBuilder(col.getType());
+ }
+
}
private void initReducerIdToColumnIndex(KylinConfig config) throws IOException {
@@ -150,11 +188,26 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
}
} else {
- colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1)));
- if (colValues.size() == 1000000) { //spill every 1 million
- logger.info("spill values to disk...");
- outputDistinctValues(col, colValues, context);
- colValues.clear();
+ if (isReducerLocalBuildDict) {
+ String value = new String(key.getBytes(), 1, key.getLength() - 1);
+ //partition col
+ try {
+ if (isPartitionCol) {
+ long time = dateFormat.parse(value).getTime();
+ timeMinValue = Math.min(timeMinValue, time);
+ timeMaxValue = Math.max(timeMaxValue, time);
+ }
+ builder.addValue(value);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ } else {
+ colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1)));
+ if (colValues.size() == 1000000) { //spill every 1 million
+ logger.info("spill values to disk...");
+ outputDistinctValues(col, colValues, context);
+ colValues.clear();
+ }
}
}
@@ -191,12 +244,64 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
}
+ private void outputDict(TblColRef col, Dictionary<String> dict, Context context) throws IOException {
+ final String fileName = col.getName() + DICT_FILE_POSTFIX;
+ FSDataOutputStream out = getOutputStream(context, fileName);
+ try {
+ String dictClassName = dict.getClass().getName();
+ out.writeUTF(dictClassName);
+ dict.write(out);
+ logger.info("reducer id is:+" + taskId + " colName:" + col.getName() + " writing dict at file : " + fileName + " dict class:" + dictClassName);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+ }
+
+ private void outputPartitionInfo(Context context) throws IOException {
+ final String fileName = col.getName() + PARTITION_COL_INFO_FILE_POSTFIX;
+ FSDataOutputStream out = getOutputStream(context, fileName);
+ try {
+ out.writeLong(timeMinValue);
+ out.writeLong(timeMaxValue);
+ logger.info("write partition info for col : " + col.getName() + " minValue:" + timeMinValue + " maxValue:" + timeMaxValue);
+ } finally {
+ IOUtils.closeQuietly(out);
+ }
+ }
+
+ private FSDataOutputStream getOutputStream(Context context, String outputFileName) throws IOException {
+ final Configuration conf = context.getConfiguration();
+ final FileSystem fs = FileSystem.get(conf);
+ final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
+ final Path colDir = new Path(outputPath, col.getName());
+ final Path outputFile = new Path(colDir, outputFileName);
+ FSDataOutputStream out = null;
+ if (!fs.exists(colDir)) {
+ fs.mkdirs(colDir);
+ }
+ fs.deleteOnExit(outputFile);
+ out = fs.create(outputFile);
+ return out;
+ }
+
@Override
protected void doCleanup(Context context) throws IOException, InterruptedException {
if (isStatistics == false) {
- if (colValues.size() > 0) {
- outputDistinctValues(col, colValues, context);
- colValues.clear();
+ if (isReducerLocalBuildDict) {
+ try {
+ if (isPartitionCol) {
+ outputPartitionInfo(context);
+ }
+ Dictionary<String> dict = builder.build(0);
+ outputDict(col, dict, context);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ } else {
+ if (colValues.size() > 0) {
+ outputDistinctValues(col, colValues, context);
+ colValues.clear();
+ }
}
} else {
//output the hll info;
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index eb06f07..977196c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -19,25 +19,23 @@
package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
-import java.text.ParseException;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.time.FastDateFormat;
-import org.apache.kylin.common.util.DateFormat;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.SortedColumnDFSFile;
+import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.ReadableTable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -83,39 +81,25 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
private void updateTimeRange(CubeSegment segment) throws IOException {
final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
- final DataType partitionColType = partitionCol.getType();
- final FastDateFormat dateFormat;
- if (partitionColType.isDate()) {
- dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATE_PATTERN);
- } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) {
- dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
- } else if (partitionColType.isStringFamily()) {
- String partitionDateFormat = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateFormat();
- if (StringUtils.isEmpty(partitionDateFormat)) {
- partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
- }
- dateFormat = DateFormat.getDateFormat(partitionDateFormat);
- } else {
- throw new IllegalStateException("Type " + partitionColType + " is not valid partition column type");
- }
-
- final String factDistinctPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
- //final ReadableTable readableTable = new DFSFileTable(factDistinctPath + "/" + partitionCol.getName(), -1);
- final ReadableTable readableTable = new SortedColumnDFSFile(factDistinctPath + "/" + partitionCol.getName(), partitionCol.getType());
- final ReadableTable.TableReader tableReader = readableTable.getReader();
+ final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
+ Path colDir = new Path(factColumnsInputPath, partitionCol.getName());
+ Path outputFile = new Path(colDir, partitionCol.getName() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
+ Configuration conf = HadoopUtil.getCurrentConfiguration();
+ FileSystem fs = HadoopUtil.getFileSystem(outputFile.getName());
+ FSDataInputStream is = null;
long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
try {
- while (tableReader.next()) {
- long time = dateFormat.parse(tableReader.getRow()[0]).getTime();
- minValue = Math.min(minValue, time);
- maxValue = Math.max(maxValue, time);
- }
- } catch (ParseException e) {
+ is = fs.open(outputFile);
+ long min = is.readLong();
+ long max = is.readLong();
+ minValue = Math.min(min, minValue);
+ maxValue = Math.max(max, maxValue);
+ } catch (IOException e) {
throw new IOException(e);
} finally {
- IOUtils.closeQuietly(tableReader);
+ IOUtils.closeQuietly(is);
}
-
+ logger.info("updateTimeRange step. minValue:" + minValue + " maxValue:" + maxValue);
segment.setDateRangeStart(minValue);
segment.setDateRangeEnd(maxValue);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1af08e4b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 9af0faf..02aa64a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -437,7 +437,11 @@ public class CubeStorageQuery implements IStorageQuery {
// build row key range for each cube segment
StringBuilder sb = new StringBuilder("hbasekeyrange trace: ");
for (CubeSegment cubeSeg : segs) {
-
+ CubeDesc cubeDesc = cubeSeg.getCubeDesc();
+ if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
+ logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
+ continue;
+ }
// consider derived (lookup snapshot), filter on dimension may
// differ per segment
List<Collection<ColumnValueRange>> orAndDimRanges = translateToOrAndDimRanges(flatFilter, cubeSeg);
[48/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
+ hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -448,11 +450,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -472,7 +474,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -503,8 +504,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -538,13 +539,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -561,7 +561,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 8bd4abf..20d0f7d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -61,7 +61,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -130,9 +130,8 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
-
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -189,10 +188,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 8f69c18..8f7430e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -190,7 +192,7 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeObserverClass, hdfsCoprocessorJar, 1002, null);
}
- public static boolean resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -205,7 +207,7 @@ public class DeployCoprocessorCLI {
logger.info("reset coprocessor on " + tableName);
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
while (desc.hasCoprocessor(CubeObserverClass)) {
desc.removeCoprocessor(CubeObserverClass);
@@ -231,16 +233,15 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
return true;
}
-
- private static List<String> resetCoprocessorOnHTables(final HBaseAdmin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static List<String> resetCoprocessorOnHTables(final Admin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processedTables = Collections.synchronizedList(new ArrayList<String>());
ExecutorService coprocessorPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -261,12 +262,12 @@ public class DeployCoprocessorCLI {
private static class ResetCoprocessorWorker implements Runnable {
private final CountDownLatch countDownLatch;
- private final HBaseAdmin hbaseAdmin;
+ private final Admin hbaseAdmin;
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List<String> processedTables;
- public ResetCoprocessorWorker(CountDownLatch countDownLatch, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
+ public ResetCoprocessorWorker(CountDownLatch countDownLatch, Admin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
this.countDownLatch = countDownLatch;
this.hbaseAdmin = hbaseAdmin;
this.hdfsCoprocessorJar = hdfsCoprocessorJar;
@@ -387,7 +388,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index 61c73d5..1cdb2f8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -25,10 +25,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -235,9 +236,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -257,7 +258,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 86ba22f..dd5f8fa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ Connection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void prepareData(Connection conn) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 6749d6c..940d64a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,9 +24,11 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -55,8 +57,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -71,12 +73,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 937b65f..1daca0a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -31,12 +32,15 @@ import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -58,30 +62,31 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ Table table = null;
+ Admin admin = null;
try {
+ table = hbaseConnection.getTable(TableName.valueOf(tableName));
+ admin = hbaseConnection.getAdmin();
+
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+ logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
+ List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
+ for (HRegionLocation hRegionLocation : regionLocationList) {
+ tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -105,7 +110,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- IOUtils.closeQuietly(hBaseAdmin);
+ IOUtils.closeQuietly(admin);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index 266f7e7..a2f60d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,9 +23,10 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -42,8 +43,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index e26c8e8..da13fa4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -58,11 +58,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- HConnection conn = getConnection();
- HBaseAdmin hadmin = new HBaseAdmin(conn);
+ Connection conn = getConnection();
+ Admin hadmin = conn.getAdmin();
try {
- boolean tableExist = hadmin.tableExists(tableName);
+ boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -119,8 +119,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
byte[] key = new byte[8 + 4];//time + id
@@ -135,7 +135,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -170,8 +170,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -206,7 +206,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(HTableInterface table) throws IOException {
+ private static long getFirstKeyTime(Table table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -224,8 +224,8 @@ public class HbaseStreamingInput {
}
- private static HConnection getConnection() throws IOException {
- return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
+ private static Connection getConnection() throws IOException {
+ return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ca1a060..ea05ab2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -50,8 +51,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index 8ff5b0f..df4e912 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,10 +30,14 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,9 +56,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
-
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -73,12 +77,13 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ TableName tableName = TableName.valueOf(htableName);
+ if (hbaseAdmin.tableExists(tableName)) {
+ if (hbaseAdmin.isTableEnabled(tableName)) {
+ hbaseAdmin.disableTable(tableName);
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(tableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index e219c5a..8a93160 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -22,12 +22,13 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -58,12 +59,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- HConnection conn = null;
- HTableInterface table = null;
+ Connection conn = null;
+ Table table = null;
ResultScanner scanner = null;
try {
- conn = HConnectionManager.createConnection(hconf);
- table = conn.getTable(hbaseTable);
+ conn = ConnectionFactory.createConnection(hconf);
+ table = conn.getTable(TableName.valueOf(hbaseTable));
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index 01edb1f..db516bb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,11 +22,12 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -70,8 +71,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
+ Connection conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 9fe5a23..74a4718 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -56,6 +58,7 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,7 +79,8 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -152,22 +156,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- HBaseAdmin hbaseAdmin;
+ Admin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index e36f662..42a54c8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,16 +24,18 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,14 +51,15 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ hbaseAdmin = conn.getAdmin();
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -166,9 +169,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index c25b690..4695353 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.gridtable.StorageSideBehavior;
import org.apache.kylin.metadata.datatype.LongMutable;
@@ -229,15 +230,8 @@ public class AggregateRegionObserverTest {
return nextRaw(results);
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.InternalScanner#next(java.util
- * .List, int)
- */
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -306,6 +300,11 @@ public class AggregateRegionObserverTest {
return 0;
}
+ @Override
+ public int getBatch() {
+ return 0;
+ }
+
/*
* (non-Javadoc)
*
@@ -322,16 +321,9 @@ public class AggregateRegionObserverTest {
return i < input.size();
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#nextRaw(java.util
- * .List, int)
- */
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return nextRaw(result);
+ public boolean nextRaw(List<Cell> list, ScannerContext scannerContext) throws IOException {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 1d85922..04e2e8b 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -136,7 +137,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
Put p = new Put(rk);
p.setDurability(Durability.SKIP_WAL);
- p.add(cf.getBytes(), cq, Bytes.toBytes(c));
+ p.addColumn(cf.getBytes(), cq, Bytes.toBytes(c));
ht.put(p);
}
}
@@ -224,7 +225,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
scan.addFamily(cf.getBytes());
scan.setFilter(filter);
List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
- HRegion first = regions.get(0);
+ Region first = regions.get(0);
first.getScanner(scan);
RegionScanner scanner = first.getScanner(scan);
List<Cell> results = new ArrayList<Cell>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index 40306c9..fa551f8 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -230,6 +230,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -446,11 +447,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -470,7 +471,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -536,13 +536,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -559,7 +558,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/469d9cc5/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index 19e5db0..f52fc3e 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,10 +25,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -231,9 +232,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -253,7 +254,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
[05/50] [abbrv] kylin git commit: KYLIN-1875 A good snowflake sample
data, and rename 'kylin_metadata' to 'kylin_default_instance'
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
----------------------------------------------------------------------
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index d8158ec..7349b37 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,10000 +1,10000 @@
-0,2012-12-14,Others,88750,0,11,36.2828,0,10000349,US,JP,ANALYST,Beijing
-1,2012-08-28,Others,175750,0,13,23.8563,0,10000927,JP,JP,ANALYST,Beijing
-2,2012-02-16,ABIN,148324,15,13,88.3418,0,10000005,DE,FR,ADMIN,Shanghai
-3,2013-10-19,FP-non GTC,37831,0,13,47.3015,0,10000209,UK,UK,ANALYST,Beijing
-4,2012-10-22,Others,140746,100,11,83.454,0,10000154,JP,CN,ADMIN,Shanghai
-5,2013-01-25,FP-GTC,16509,0,-99,22.9896,0,10000372,DE,UK,ADMIN,Shanghai
-6,2013-04-04,Others,963,0,13,88.5907,0,10000648,US,JP,MODELER,Hongkong
-7,2012-04-11,Others,15687,0,15,88.194,0,10000866,US,UK,ADMIN,Shanghai
-8,2013-01-19,ABIN,60606,3,13,77.9727,0,10000936,FR,CN,MODELER,Hongkong
-9,2012-04-30,FP-non GTC,106246,0,14,52.295,0,10000047,CN,FR,MODELER,Hongkong
-10,2013-02-03,Auction,45333,0,16,56.3584,0,10000470,UK,JP,MODELER,Hongkong
-11,2012-09-02,FP-GTC,25147,0,-99,15.3553,0,10000203,US,UK,ANALYST,Beijing
-12,2013-07-30,Others,24760,0,16,25.077,0,10000382,CN,JP,ADMIN,Shanghai
-13,2012-04-17,Auction,31519,0,16,72.7384,0,10000069,CN,JP,ADMIN,Shanghai
-14,2013-04-30,FP-GTC,51582,0,15,75.82,0,10000085,CN,DE,ANALYST,Beijing
-15,2013-03-23,FP-GTC,16509,0,15,8.6653,0,10000559,US,FR,MODELER,Hongkong
-16,2013-04-01,ABIN,175750,3,16,64.2802,0,10000507,JP,CN,MODELER,Hongkong
-17,2013-07-12,ABIN,50677,0,-99,24.5987,0,10000938,UK,FR,ADMIN,Shanghai
-18,2012-04-23,FP-GTC,45333,0,5,72.6553,0,10000863,CN,CN,MODELER,Hongkong
-19,2012-12-19,ABIN,3838,0,12,87.4142,0,10000612,DE,JP,ADMIN,Shanghai
-20,2012-08-15,FP-GTC,62179,0,16,67.4238,0,10000839,JP,US,MODELER,Hongkong
-21,2013-08-22,Auction,1357,0,5,18.36,0,10000303,CN,JP,ANALYST,Beijing
-22,2013-08-23,ABIN,31519,0,5,67.6344,0,10000877,FR,JP,ANALYST,Beijing
-23,2013-09-12,FP-GTC,223,0,14,71.7898,0,10000848,US,CN,MODELER,Hongkong
-24,2012-03-14,FP-GTC,132939,0,15,26.1496,0,10000815,JP,DE,MODELER,Hongkong
-25,2013-07-13,Others,67703,3,16,83.9661,0,10000858,CN,CN,MODELER,Hongkong
-26,2013-12-15,FP-non GTC,100847,0,5,34.3451,0,10000387,CN,CN,ADMIN,Shanghai
-27,2012-08-30,Auction,62179,0,11,21.0496,0,10000393,JP,FR,ANALYST,Beijing
-28,2013-01-10,FP-GTC,48027,0,13,23.4142,0,10000757,FR,FR,MODELER,Hongkong
-29,2013-01-25,ABIN,145970,0,5,26.7842,0,10000402,DE,CN,ANALYST,Beijing
-30,2012-07-05,Auction,48027,0,-99,46.732,0,10000972,DE,UK,MODELER,Hongkong
-31,2012-11-07,FP-GTC,20213,0,-99,19.727,0,10000873,UK,CN,ANALYST,Beijing
-32,2013-06-20,ABIN,100847,0,5,76.2783,0,10000409,CN,DE,ADMIN,Shanghai
-33,2012-10-08,ABIN,26262,0,5,48.7447,0,10000029,CN,FR,ADMIN,Shanghai
-34,2013-04-30,ABIN,87118,0,15,63.2187,0,10000529,FR,DE,ANALYST,Beijing
-35,2013-04-24,FP-non GTC,20213,0,16,39.5057,0,10000329,JP,FR,ANALYST,Beijing
-36,2012-05-13,Others,164261,0,13,84.1246,0,10000688,CN,GB,ADMIN,Shanghai
-37,2013-08-02,ABIN,31519,0,5,37.1504,0,10000905,FR,DE,MODELER,Hongkong
-38,2012-04-03,Auction,164261,0,11,92.0974,0,10000341,CN,UK,ADMIN,Shanghai
-39,2012-09-02,FP-GTC,31519,0,14,87.7829,0,10000460,DE,UK,ANALYST,Beijing
-40,2013-10-05,ABIN,13987,0,16,74.0719,0,10000381,CN,UK,MODELER,Hongkong
-41,2012-01-13,FP-GTC,48027,0,15,71.2637,0,10000915,UK,CN,ADMIN,Shanghai
-42,2013-03-01,Others,13836,0,14,16.7288,0,10000953,CN,UK,ADMIN,Shanghai
-43,2012-09-12,FP-non GTC,16509,0,5,12.2933,0,10000277,FR,CN,MODELER,Hongkong
-44,2012-02-07,Auction,45333,0,5,64.977,0,10000040,FR,JP,MODELER,Hongkong
-45,2013-12-14,FP-non GTC,158798,0,16,72.4413,0,10000500,DE,CN,MODELER,Hongkong
-46,2013-10-13,Auction,31519,0,5,79.3053,0,10000816,CN,JP,MODELER,Hongkong
-47,2012-06-11,ABIN,4943,0,5,11.6942,0,10000696,US,UK,ANALYST,Beijing
-48,2012-10-18,ABIN,80053,0,-99,54.0933,0,10000604,FR,JP,MODELER,Hongkong
-49,2012-11-03,Auction,20886,0,5,9.8258,0,10000764,US,CN,MODELER,Hongkong
-50,2012-01-15,Auction,44079,0,14,13.0371,0,10000343,CN,US,MODELER,Hongkong
-51,2012-07-03,FP-GTC,159184,0,-99,92.5314,0,10000724,CN,GB,ADMIN,Shanghai
-52,2012-02-16,FP-GTC,314,0,13,49.1825,0,10000334,JP,CN,ADMIN,Shanghai
-53,2012-06-17,FP-non GTC,31387,3,12,0.6677,0,10000603,JP,JP,ANALYST,Beijing
-54,2012-07-15,FP-non GTC,32876,0,-99,50.9634,0,10000248,CN,UK,ANALYST,Beijing
-55,2012-07-24,FP-GTC,150265,15,16,57.5645,0,10000653,CN,CN,ADMIN,Shanghai
-56,2012-10-06,Auction,145970,0,5,44.3091,0,10000082,JP,DE,ADMIN,Shanghai
-57,2013-06-14,ABIN,80053,0,5,20.2388,0,10000632,JP,UK,MODELER,Hongkong
-58,2013-10-10,FP-non GTC,15868,0,11,28.629,0,10000523,US,FR,MODELER,Hongkong
-59,2012-09-30,ABIN,31673,0,12,75.8658,0,10000719,CN,FR,ADMIN,Shanghai
-60,2012-04-02,FP-GTC,63861,0,5,25.8748,0,10000311,JP,FR,ANALYST,Beijing
-61,2012-03-16,Others,314,0,-99,9.2733,0,10000409,US,FR,MODELER,Hongkong
-62,2012-09-30,FP-non GTC,60340,0,14,74.661,0,10000790,FR,DE,MODELER,Hongkong
-63,2012-03-20,Others,87118,0,5,79.3781,0,10000384,CN,GB,MODELER,Hongkong
-64,2012-09-12,FP-GTC,1120,3,5,61.4408,0,10000645,US,JP,ANALYST,Beijing
-65,2013-12-10,FP-non GTC,20886,0,11,14.814,0,10000943,DE,CN,ANALYST,Beijing
-66,2012-12-05,FP-GTC,106246,0,12,24.938,0,10000858,FR,JP,ANALYST,Beijing
-67,2012-08-21,Auction,43479,0,12,22.2008,0,10000310,DE,DE,MODELER,Hongkong
-68,2012-11-30,FP-GTC,175750,0,15,35.1161,0,10000440,CN,DE,ADMIN,Shanghai
-69,2013-10-10,Others,95672,0,11,82.3718,0,10000128,JP,CN,ADMIN,Shanghai
-70,2012-10-17,Auction,80053,0,15,9.8166,0,10000105,FR,DE,ADMIN,Shanghai
-71,2012-03-16,FP-non GTC,57784,0,5,66.2369,0,10000439,UK,FR,MODELER,Hongkong
-72,2012-11-14,ABIN,65,0,11,17.5513,0,10000855,CN,CN,MODELER,Hongkong
-73,2013-06-24,FP-GTC,16509,0,15,53.9195,0,10000130,UK,CN,MODELER,Hongkong
-74,2012-08-13,FP-non GTC,65,0,5,69.8447,0,10000548,CN,UK,MODELER,Hongkong
-75,2012-01-03,Others,38238,0,11,90.3722,0,10000739,UK,US,ADMIN,Shanghai
-76,2013-11-24,FP-non GTC,175750,3,13,47.3441,0,10000291,CN,DE,ANALYST,Beijing
-77,2013-11-08,Auction,106246,0,5,38.6729,0,10000310,CN,DE,ADMIN,Shanghai
-78,2013-04-07,Others,73506,0,-99,69.8999,0,10000235,US,DE,ADMIN,Shanghai
-79,2012-10-15,FP-GTC,314,0,13,44.1598,0,10000265,FR,US,ADMIN,Shanghai
-80,2012-01-20,FP-GTC,80287,0,-99,85.7263,0,10000704,FR,US,ANALYST,Beijing
-81,2012-08-17,ABIN,32876,0,16,21.2934,0,10000163,UK,CN,ANALYST,Beijing
-82,2013-07-09,Auction,37831,0,15,38.2213,0,10000557,JP,UK,ADMIN,Shanghai
-83,2013-09-21,Others,45333,0,13,25.5384,0,10000184,JP,UK,MODELER,Hongkong
-84,2013-10-28,FP-non GTC,11554,0,16,89.9654,0,10000940,CN,UK,MODELER,Hongkong
-85,2013-04-05,Auction,11554,0,14,41.8676,0,10000384,CN,DE,ANALYST,Beijing
-86,2013-03-25,FP-GTC,43398,0,14,75.011,0,10000697,JP,UK,ANALYST,Beijing
-87,2012-05-03,ABIN,33038,15,12,64.6735,0,10000147,US,CN,ADMIN,Shanghai
-88,2012-10-13,ABIN,95672,3,14,3.2128,0,10000547,JP,CN,MODELER,Hongkong
-89,2012-09-11,FP-GTC,37831,0,16,60.3228,0,10000899,DE,UK,MODELER,Hongkong
-90,2012-08-31,Auction,314,0,5,13.248,0,10000350,CN,US,ANALYST,Beijing
-91,2012-12-25,Others,165888,0,16,60.3492,0,10000367,CN,CN,ANALYST,Beijing
-92,2012-12-26,FP-non GTC,50677,0,5,58.6078,0,10000523,JP,CN,ADMIN,Shanghai
-93,2012-05-24,FP-GTC,145970,0,5,33.1342,0,10000926,JP,US,MODELER,Hongkong
-94,2012-06-27,Auction,174106,3,13,55.3039,0,10000430,DE,JP,ANALYST,Beijing
-95,2012-01-02,ABIN,87118,0,14,48.2433,0,10000731,US,CN,ADMIN,Shanghai
-96,2013-12-14,FP-GTC,50508,0,5,51.5822,0,10000977,DE,US,ANALYST,Beijing
-97,2012-12-30,Auction,63889,0,12,3.8393,0,10000957,UK,DE,MODELER,Hongkong
-98,2013-10-26,FP-non GTC,24541,0,5,23.6812,0,10000350,DE,CN,ANALYST,Beijing
-99,2013-12-29,ABIN,38238,0,15,99.4389,0,10000605,US,UK,ANALYST,Beijing
-100,2012-05-26,FP-GTC,100847,0,15,11.0672,0,10000291,UK,DE,ADMIN,Shanghai
-101,2012-12-18,FP-non GTC,164261,0,11,54.9352,0,10000509,CN,US,ANALYST,Beijing
-102,2013-03-12,ABIN,95672,0,15,85.7152,0,10000742,FR,CN,ADMIN,Shanghai
-103,2013-02-22,ABIN,43398,0,11,65.197,0,10000294,CN,UK,ADMIN,Shanghai
-104,2012-02-20,Others,175750,0,14,63.2187,0,10000364,DE,UK,MODELER,Hongkong
-105,2013-08-06,Others,32876,0,13,50.8111,0,10000542,US,UK,MODELER,Hongkong
-106,2013-06-27,Others,13987,0,13,2.3165,0,10000979,CN,JP,ADMIN,Shanghai
-107,2012-07-17,Others,13987,0,16,25.2136,0,10000744,JP,FR,ADMIN,Shanghai
-108,2013-10-12,FP-non GTC,16509,0,5,15.3986,0,10000652,US,JP,MODELER,Hongkong
-109,2012-12-07,Others,158798,0,5,1.6111,0,10000149,FR,JP,ANALYST,Beijing
-110,2013-02-09,Auction,32876,0,5,37.3237,0,10000467,DE,US,ADMIN,Shanghai
-111,2012-10-26,Others,44079,0,12,18.187,0,10000420,UK,CN,ADMIN,Shanghai
-112,2012-04-10,ABIN,132939,0,13,88.595,0,10000103,JP,CN,MODELER,Hongkong
-113,2013-04-08,FP-non GTC,159184,0,5,92.4421,0,10000933,UK,CN,ADMIN,Shanghai
-114,2012-05-30,Others,80287,0,14,68.1663,0,10000195,FR,DE,ADMIN,Shanghai
-115,2012-02-23,FP-GTC,88750,0,11,19.4075,0,10000038,JP,JP,ADMIN,Shanghai
-116,2013-10-25,ABIN,11554,0,11,27.2512,0,10000228,CN,FR,ANALYST,Beijing
-117,2012-12-02,FP-GTC,41940,0,14,49.5504,0,10000927,DE,FR,ANALYST,Beijing
-118,2013-09-16,FP-GTC,314,0,-99,3.4785,0,10000304,US,JP,MODELER,Hongkong
-119,2013-11-01,Others,31519,3,15,46.1783,0,10000771,CN,US,MODELER,Hongkong
-120,2012-10-21,FP-non GTC,31519,0,15,1.7696,0,10000507,CN,FR,MODELER,Hongkong
-121,2013-08-25,Auction,95173,0,13,89.2083,0,10000868,US,FR,MODELER,Hongkong
-122,2013-11-09,FP-non GTC,150047,3,12,58.2634,0,10000497,FR,CN,ANALYST,Beijing
-123,2012-02-04,FP-GTC,175750,0,11,41.385,0,10000947,CN,FR,ANALYST,Beijing
-124,2012-10-17,ABIN,26249,0,14,10.5789,0,10000965,UK,FR,ADMIN,Shanghai
-125,2012-03-16,FP-non GTC,963,0,5,17.8949,0,10000908,UK,CN,ANALYST,Beijing
-126,2012-08-06,Others,94847,0,13,96.0332,0,10000595,UK,JP,ANALYST,Beijing
-127,2013-12-23,FP-non GTC,24760,0,16,90.7096,0,10000700,US,FR,ADMIN,Shanghai
-128,2013-03-29,FP-GTC,9426,3,13,50.9023,0,10000090,FR,JP,ADMIN,Shanghai
-129,2013-03-25,Others,20886,0,-99,35.0414,0,10000171,CN,GB,ANALYST,Beijing
-130,2013-12-10,Auction,95672,0,15,29.8872,0,10000216,DE,JP,MODELER,Hongkong
-131,2012-03-14,Others,12688,0,14,10.4067,0,10000626,DE,UK,MODELER,Hongkong
-132,2013-11-29,FP-non GTC,31519,0,13,89.2334,0,10000199,CN,FR,ANALYST,Beijing
-133,2013-04-29,ABIN,31519,0,-99,99.246,0,10000565,CN,FR,ADMIN,Shanghai
-134,2012-02-01,FP-GTC,161567,15,11,56.3989,0,10000651,JP,US,ANALYST,Beijing
-135,2012-03-17,FP-GTC,4943,0,16,0.4422,0,10000829,US,FR,MODELER,Hongkong
-136,2012-11-06,FP-GTC,156614,0,12,2.3454,0,10000370,DE,UK,ANALYST,Beijing
-137,2012-12-03,Auction,155226,0,13,51.1354,0,10000867,FR,CN,MODELER,Hongkong
-138,2012-09-09,FP-GTC,99985,0,12,9.1236,0,10000281,UK,JP,ANALYST,Beijing
-139,2012-06-16,ABIN,99985,0,16,16.407,0,10000143,CN,CN,MODELER,Hongkong
-140,2012-05-06,FP-GTC,99985,0,14,6.4304,0,10000514,JP,US,MODELER,Hongkong
-141,2013-10-20,FP-GTC,36250,0,13,55.6895,0,10000390,JP,UK,ADMIN,Shanghai
-142,2012-10-11,ABIN,106246,0,15,30.6373,0,10000980,CN,CN,ADMIN,Shanghai
-143,2013-08-08,Others,103178,15,12,41.4904,0,10000751,CN,UK,ADMIN,Shanghai
-144,2012-09-05,Auction,314,0,16,3.2615,0,10000280,CN,DE,MODELER,Hongkong
-145,2012-08-07,ABIN,48904,0,13,57.7688,0,10000245,US,DE,ANALYST,Beijing
-146,2013-07-05,ABIN,314,0,14,28.9444,0,10000888,US,CN,MODELER,Hongkong
-147,2012-09-10,FP-non GTC,95672,0,-99,54.0914,0,10000601,JP,FR,ANALYST,Beijing
-148,2012-09-18,Auction,25147,0,12,74.4422,0,10000679,JP,CN,ANALYST,Beijing
-149,2012-08-24,FP-non GTC,175750,3,16,45.8282,0,10000397,US,UK,ANALYST,Beijing
-150,2013-05-29,Others,26249,0,14,12.7681,0,10000967,UK,JP,ANALYST,Beijing
-151,2012-11-17,FP-GTC,94847,0,14,74.4181,0,10000175,JP,CN,ANALYST,Beijing
-152,2012-09-22,ABIN,175750,0,15,85.6879,0,10000453,DE,CN,ADMIN,Shanghai
-153,2013-09-17,ABIN,533,0,11,85.5838,0,10000642,FR,JP,ADMIN,Shanghai
-154,2013-03-28,Others,60340,0,12,72.6495,0,10000397,CN,DE,ADMIN,Shanghai
-155,2013-08-09,Others,216,0,15,88.6918,0,10000867,DE,CN,ADMIN,Shanghai
-156,2013-12-19,Others,43398,0,-99,88.0444,0,10000482,CN,JP,MODELER,Hongkong
-157,2013-12-02,FP-non GTC,15115,0,-99,29.6027,0,10000538,CN,JP,MODELER,Hongkong
-158,2012-11-23,Others,46575,0,5,49.3124,0,10000678,FR,CN,ANALYST,Beijing
-159,2013-06-25,FP-non GTC,16145,0,15,7.4413,0,10000897,DE,CN,MODELER,Hongkong
-160,2012-01-22,FP-GTC,164,0,14,71.8071,0,10000510,CN,JP,ANALYST,Beijing
-161,2013-04-24,FP-GTC,1504,0,12,95.5706,0,10000554,CN,UK,MODELER,Hongkong
-162,2012-03-05,FP-non GTC,12688,0,5,47.7646,0,10000175,CN,UK,ADMIN,Shanghai
-163,2013-02-15,FP-non GTC,16145,0,13,40.9511,0,10000647,CN,UK,MODELER,Hongkong
-164,2013-01-18,Others,36250,0,13,77.3572,0,10000687,JP,JP,ADMIN,Shanghai
-165,2012-06-04,Others,15115,0,15,25.34,0,10000493,UK,US,MODELER,Hongkong
-166,2013-10-13,FP-GTC,165888,0,-99,58.1861,0,10000753,US,DE,ANALYST,Beijing
-167,2012-05-21,Auction,145970,0,5,37.2504,0,10000292,CN,US,ADMIN,Shanghai
-168,2012-04-26,Auction,11848,0,13,29.2954,0,10000804,FR,US,MODELER,Hongkong
-169,2013-09-26,FP-GTC,61323,0,13,84.7095,0,10000761,US,UK,ANALYST,Beijing
-170,2013-03-06,Auction,100847,0,-99,81.798,0,10000593,JP,US,ANALYST,Beijing
-171,2012-08-08,ABIN,51582,0,15,15.979,0,10000138,CN,DE,ADMIN,Shanghai
-172,2012-01-09,Others,1504,0,14,34.0508,0,10000954,CN,DE,ANALYST,Beijing
-173,2013-10-20,ABIN,20886,0,12,36.9353,0,10000408,FR,JP,ADMIN,Shanghai
-174,2013-06-25,Auction,53064,0,11,99.7714,0,10000409,CN,DE,MODELER,Hongkong
-175,2013-05-03,FP-GTC,63861,0,5,39.2974,0,10000668,JP,CN,MODELER,Hongkong
-176,2013-01-26,FP-GTC,100847,0,13,59.5009,0,10000536,CN,UK,ADMIN,Shanghai
-177,2012-09-19,FP-GTC,164261,0,13,15.7583,0,10000231,CN,US,ANALYST,Beijing
-178,2013-11-16,FP-GTC,164261,0,16,83.6234,0,10000927,FR,DE,ADMIN,Shanghai
-179,2013-05-07,FP-non GTC,20485,0,5,52.91,0,10000491,FR,DE,MODELER,Hongkong
-180,2013-08-15,Others,32996,15,12,54.7122,0,10000411,CN,JP,MODELER,Hongkong
-181,2013-10-31,FP-GTC,63861,0,14,0.3599,0,10000903,CN,JP,MODELER,Hongkong
-182,2012-06-13,FP-non GTC,159184,0,11,49.955,0,10000849,US,US,ANALYST,Beijing
-183,2012-12-25,Auction,20213,0,-99,11.4752,0,10000583,FR,US,ANALYST,Beijing
-184,2013-12-15,Auction,759,0,13,33.6991,0,10000390,FR,US,ADMIN,Shanghai
-185,2012-04-30,Others,87118,0,16,51.463,0,10000572,CN,DE,MODELER,Hongkong
-186,2012-07-14,Others,10866,0,11,36.9614,0,10000467,DE,DE,ANALYST,Beijing
-187,2013-06-25,FP-non GTC,1357,0,12,17.5618,0,10000426,JP,CN,ANALYST,Beijing
-188,2013-02-06,Auction,13836,0,12,74.3179,0,10000346,CN,DE,ANALYST,Beijing
-189,2012-06-03,ABIN,60340,0,12,82.3722,0,10000531,DE,CN,ANALYST,Beijing
-190,2013-07-23,FP-GTC,139973,0,-99,75.2548,0,10000385,UK,FR,ADMIN,Shanghai
-191,2013-07-06,Others,11554,0,14,40.1129,0,10000724,DE,DE,ADMIN,Shanghai
-192,2013-08-10,FP-non GTC,31519,0,-99,5.2432,0,10000356,CN,US,ADMIN,Shanghai
-193,2013-07-31,Auction,31673,0,14,51.1089,0,10000598,CN,UK,ADMIN,Shanghai
-194,2013-12-05,Auction,314,0,13,19.9031,0,10000068,CN,US,ADMIN,Shanghai
-195,2012-12-30,ABIN,50508,0,5,73.8659,0,10000482,CN,US,MODELER,Hongkong
-196,2012-09-22,Auction,165888,0,12,81.9826,0,10000832,CN,CN,MODELER,Hongkong
-197,2012-07-16,Auction,223,0,5,76.0284,0,10000354,DE,CN,MODELER,Hongkong
-198,2013-01-26,Others,20485,0,12,71.6623,0,10000552,UK,CN,ANALYST,Beijing
-199,2012-11-07,FP-non GTC,57013,0,16,73.2931,0,10000801,CN,UK,ANALYST,Beijing
-200,2012-01-26,FP-non GTC,175750,0,13,18.897,0,10000649,CN,UK,ADMIN,Shanghai
-201,2012-06-09,FP-non GTC,60340,0,12,11.1614,0,10000830,DE,UK,ADMIN,Shanghai
-202,2012-06-20,Auction,57990,3,5,69.1183,0,10000557,US,US,MODELER,Hongkong
-203,2013-01-01,FP-non GTC,45333,0,13,94.0586,0,10000253,JP,JP,ANALYST,Beijing
-204,2013-07-26,FP-non GTC,139973,0,11,39.7239,0,10000441,DE,JP,ADMIN,Shanghai
-205,2012-09-15,FP-GTC,24541,0,15,0.3239,0,10000379,UK,JP,ANALYST,Beijing
-206,2013-09-22,Auction,31519,3,12,80.0315,0,10000436,US,FR,ANALYST,Beijing
-207,2012-10-27,FP-GTC,223,0,15,39.379,0,10000420,CN,DE,ANALYST,Beijing
-208,2012-11-20,FP-non GTC,533,0,5,49.4951,0,10000390,UK,CN,ADMIN,Shanghai
-209,2013-12-07,FP-GTC,1349,0,15,93.3008,0,10000398,US,US,MODELER,Hongkong
-210,2012-05-30,FP-non GTC,46575,0,12,66.7652,0,10000915,JP,FR,ADMIN,Shanghai
-211,2012-09-07,FP-non GTC,95672,0,14,29.597,0,10000231,CN,FR,MODELER,Hongkong
-212,2013-04-26,Auction,31673,0,-99,15.8347,0,10000260,DE,JP,ANALYST,Beijing
-213,2012-09-11,FP-non GTC,50508,0,14,80.5444,0,10000263,JP,DE,ADMIN,Shanghai
-214,2013-09-03,FP-non GTC,95173,0,16,33.1487,0,10000493,DE,US,ADMIN,Shanghai
-215,2012-10-03,FP-non GTC,44079,0,15,99.7408,0,10000828,FR,US,ADMIN,Shanghai
-216,2012-12-26,ABIN,16145,0,14,40.7826,0,10000062,CN,CN,ANALYST,Beijing
-217,2012-07-17,Auction,26249,0,15,28.6663,0,10000534,US,DE,ADMIN,Shanghai
-218,2013-06-30,ABIN,63861,0,14,75.6832,0,10000944,DE,JP,ADMIN,Shanghai
-219,2012-01-09,ABIN,156614,0,11,98.4996,0,10000353,DE,JP,ADMIN,Shanghai
-220,2013-09-08,FP-GTC,156614,0,13,12.7473,0,10000063,JP,UK,ADMIN,Shanghai
-221,2013-09-23,Auction,41940,0,5,37.5254,0,10000383,CN,CN,ADMIN,Shanghai
-222,2012-11-11,Auction,145970,0,5,77.1361,0,10000420,CN,DE,MODELER,Hongkong
-223,2013-08-26,Others,11848,0,16,28.2799,0,10000323,DE,UK,MODELER,Hongkong
-224,2012-09-15,Others,11554,0,13,63.7232,0,10000729,FR,UK,ADMIN,Shanghai
-225,2012-01-09,FP-non GTC,25147,0,11,7.3009,0,10000745,FR,US,ADMIN,Shanghai
-226,2012-08-16,Others,87118,0,15,47.8997,0,10000240,CN,UK,MODELER,Hongkong
-227,2013-12-26,FP-non GTC,16145,0,12,51.0442,0,10000050,US,US,MODELER,Hongkong
-228,2012-03-24,ABIN,48027,0,5,26.2083,0,10000600,DE,CN,ANALYST,Beijing
-229,2012-08-22,ABIN,216,0,5,79.9268,0,10000706,UK,DE,ANALYST,Beijing
-230,2013-11-03,FP-GTC,24541,0,12,83.0235,0,10000293,UK,CN,ADMIN,Shanghai
-231,2013-05-18,FP-non GTC,88750,0,-99,84.8856,0,10000147,FR,CN,ADMIN,Shanghai
-232,2013-01-15,ABIN,95672,0,15,86.0386,0,10000590,DE,UK,MODELER,Hongkong
-233,2012-12-03,ABIN,50508,0,11,67.9981,0,10000582,JP,CN,MODELER,Hongkong
-234,2012-06-19,FP-GTC,148324,15,15,53.4811,0,10000793,JP,JP,ANALYST,Beijing
-235,2013-10-06,Auction,6762,0,14,73.5338,0,10000544,CN,GB,MODELER,Hongkong
-236,2013-12-18,ABIN,63861,0,5,50.2892,0,10000769,FR,JP,ADMIN,Shanghai
-237,2013-12-04,Auction,25147,0,5,48.3303,0,10000590,FR,JP,MODELER,Hongkong
-238,2012-04-25,FP-non GTC,62179,0,14,72.3156,0,10000102,DE,JP,ADMIN,Shanghai
-239,2013-11-20,Auction,24760,0,14,91.4883,0,10000370,CN,JP,ADMIN,Shanghai
-240,2013-02-08,Others,20485,0,11,29.1083,0,10000261,DE,UK,MODELER,Hongkong
-241,2013-04-18,Others,2023,0,5,48.4321,0,10000355,UK,CN,ANALYST,Beijing
-242,2012-11-21,Others,32876,0,12,16.7365,0,10000352,FR,JP,ANALYST,Beijing
-243,2012-03-07,FP-GTC,61323,0,-99,32.5836,0,10000494,US,DE,ADMIN,Shanghai
-244,2012-01-20,FP-non GTC,175750,0,16,52.8153,0,10000025,UK,DE,ADMIN,Shanghai
-245,2013-10-06,FP-GTC,26249,0,-99,1.9539,0,10000660,JP,US,MODELER,Hongkong
-246,2012-01-10,Auction,57990,0,14,57.58,0,10000830,FR,JP,ANALYST,Beijing
-247,2013-09-13,Others,46575,0,15,60.3315,0,10000528,JP,FR,MODELER,Hongkong
-248,2012-11-19,FP-non GTC,44079,0,11,73.0457,0,10000482,US,UK,ANALYST,Beijing
-249,2012-09-23,Auction,95672,0,13,84.6345,0,10000586,US,JP,MODELER,Hongkong
-250,2013-08-31,Others,216,0,12,34.7429,0,10000676,FR,FR,ANALYST,Beijing
-251,2013-04-11,FP-GTC,963,0,-99,77.7226,0,10000367,FR,FR,ADMIN,Shanghai
-252,2013-03-28,ABIN,15687,0,-99,8.2384,0,10000542,UK,JP,ANALYST,Beijing
-253,2012-05-24,ABIN,152801,0,15,39.0204,0,10000277,CN,CN,ANALYST,Beijing
-254,2013-03-05,ABIN,67698,0,16,40.6091,0,10000886,CN,DE,ADMIN,Shanghai
-255,2013-08-06,Auction,13987,0,13,36.5088,0,10000628,JP,DE,MODELER,Hongkong
-256,2013-02-04,ABIN,20213,0,11,73.5301,0,10000134,FR,CN,MODELER,Hongkong
-257,2012-05-19,FP-non GTC,38238,0,12,59.3237,0,10000500,CN,GB,ADMIN,Shanghai
-258,2013-08-19,ABIN,57990,0,12,10.1688,0,10000715,CN,CN,ANALYST,Beijing
-259,2012-03-28,FP-non GTC,46575,0,16,77.7396,0,10000756,US,FR,ADMIN,Shanghai
-260,2012-02-19,Auction,43479,0,-99,45.245,0,10000049,DE,CN,ANALYST,Beijing
-261,2012-04-02,FP-GTC,41940,0,15,58.8415,0,10000195,US,CN,ADMIN,Shanghai
-262,2013-08-24,Others,16145,0,11,5.4971,0,10000119,DE,UK,MODELER,Hongkong
-263,2013-01-10,Auction,106340,15,12,24.1336,0,10000603,FR,CN,MODELER,Hongkong
-264,2012-09-23,Auction,152801,0,5,70.4024,0,10000384,JP,CN,ANALYST,Beijing
-265,2013-07-29,FP-GTC,73506,0,16,14.5932,0,10000580,UK,UK,ADMIN,Shanghai
-266,2012-02-16,FP-non GTC,67698,0,5,4.4981,0,10000524,FR,DE,MODELER,Hongkong
-267,2013-01-06,Others,65,0,13,64.8343,0,10000731,US,FR,MODELER,Hongkong
-268,2013-03-02,Auction,43398,0,16,22.6621,0,10000734,US,UK,ADMIN,Shanghai
-269,2013-10-25,Auction,57990,0,13,64.3384,0,10000961,JP,US,ADMIN,Shanghai
-270,2012-07-07,FP-GTC,156356,0,16,47.3744,0,10000795,FR,UK,ADMIN,Shanghai
-271,2013-12-19,FP-GTC,67698,0,16,96.9535,0,10000207,UK,JP,MODELER,Hongkong
-272,2013-11-04,Auction,63861,0,5,34.8173,0,10000964,JP,CN,MODELER,Hongkong
-273,2013-01-18,FP-non GTC,174053,3,5,94.6036,0,10000393,JP,CN,ANALYST,Beijing
-274,2013-01-30,Auction,43479,0,16,28.9982,0,10000854,CN,UK,ANALYST,Beijing
-275,2012-05-16,ABIN,24541,0,15,91.6347,0,10000536,CN,JP,MODELER,Hongkong
-276,2013-06-03,FP-non GTC,159184,0,15,27.9722,0,10000644,DE,CN,ADMIN,Shanghai
-277,2013-08-24,FP-GTC,2635,0,14,95.7894,0,10000503,US,UK,ADMIN,Shanghai
-278,2012-07-02,FP-non GTC,16509,0,5,76.9646,0,10000320,FR,CN,ADMIN,Shanghai
-279,2013-09-09,ABIN,80287,0,16,68.962,0,10000934,UK,FR,MODELER,Hongkong
-280,2012-04-15,FP-GTC,156356,0,12,95.364,0,10000269,UK,DE,ANALYST,Beijing
-281,2013-07-10,FP-GTC,1349,0,13,82.7969,0,10000033,US,US,MODELER,Hongkong
-282,2012-12-02,FP-GTC,963,0,13,31.4196,0,10000717,FR,JP,ADMIN,Shanghai
-283,2013-02-17,Auction,26249,0,14,9.5202,0,10000543,US,UK,ANALYST,Beijing
-284,2013-02-17,Others,87118,0,16,77.3899,0,10000647,CN,CN,ADMIN,Shanghai
-285,2012-12-12,Others,6762,0,15,76.3422,0,10000076,FR,UK,ADMIN,Shanghai
-286,2013-07-08,Others,80053,0,5,95.2412,0,10000723,US,CN,MODELER,Hongkong
-287,2012-08-19,ABIN,32876,0,12,40.9587,0,10000352,FR,CN,ANALYST,Beijing
-288,2013-07-24,Auction,314,0,11,78.8147,0,10000996,UK,UK,ANALYST,Beijing
-289,2013-10-10,Others,57990,0,12,89.8124,0,10000549,UK,US,ANALYST,Beijing
-290,2013-11-13,Others,139973,0,16,85.5451,0,10000089,CN,CN,ADMIN,Shanghai
-291,2013-03-29,FP-non GTC,25147,0,16,16.229,0,10000017,JP,DE,MODELER,Hongkong
-292,2012-06-04,FP-non GTC,44079,0,-99,0.1588,0,10000758,JP,CN,MODELER,Hongkong
-293,2012-07-19,FP-non GTC,57784,0,12,84.9441,0,10000553,CN,DE,ANALYST,Beijing
-294,2012-01-08,FP-GTC,155226,0,15,31.9705,0,10000778,US,FR,ADMIN,Shanghai
-295,2012-09-13,FP-non GTC,963,0,16,39.6392,0,10000799,DE,FR,ADMIN,Shanghai
-296,2012-12-24,Others,175750,0,15,77.3617,0,10000164,CN,DE,MODELER,Hongkong
-297,2013-08-31,Auction,57990,0,12,76.7885,0,10000580,UK,CN,ANALYST,Beijing
-298,2013-08-06,FP-non GTC,156356,0,13,86.0218,0,10000445,JP,CN,ANALYST,Beijing
-299,2013-05-29,FP-non GTC,11848,0,11,23.0269,0,10000372,CN,GB,MODELER,Hongkong
-300,2013-05-19,Others,53064,0,13,66.399,0,10000390,FR,US,MODELER,Hongkong
-301,2012-11-01,FP-non GTC,51582,0,14,57.6157,0,10000061,US,JP,ADMIN,Shanghai
-302,2013-01-01,Auction,1357,0,-99,18.2117,0,10000860,UK,JP,ANALYST,Beijing
-303,2013-05-17,Others,121153,0,14,79.4534,0,10000155,CN,FR,MODELER,Hongkong
-304,2013-07-04,FP-GTC,80053,0,5,43.3271,0,10000456,FR,UK,ANALYST,Beijing
-305,2013-12-17,Others,165888,0,-99,9.8402,0,10000357,CN,JP,MODELER,Hongkong
-306,2013-11-28,FP-GTC,26262,0,-99,1.4013,0,10000731,UK,CN,ADMIN,Shanghai
-307,2013-02-08,FP-GTC,65,0,11,61.9604,0,10000176,JP,CN,ADMIN,Shanghai
-308,2012-05-17,ABIN,9426,3,12,68.017,0,10000895,DE,UK,ANALYST,Beijing
-309,2012-08-29,Auction,159184,0,13,43.7406,0,10000723,JP,CN,MODELER,Hongkong
-310,2013-10-06,FP-GTC,61323,0,14,85.9325,0,10000692,JP,FR,ANALYST,Beijing
-311,2013-08-08,FP-GTC,20886,0,14,88.0137,0,10000063,FR,UK,ADMIN,Shanghai
-312,2013-04-06,FP-non GTC,156356,0,16,67.1201,0,10000441,DE,JP,ADMIN,Shanghai
-313,2012-03-09,Auction,1504,0,13,45.8878,0,10000862,DE,CN,ANALYST,Beijing
-314,2013-12-13,FP-GTC,139973,3,16,58.7879,0,10000947,FR,US,ADMIN,Shanghai
-315,2013-02-03,ABIN,95173,0,-99,33.2714,0,10000634,US,US,ANALYST,Beijing
-316,2013-02-19,Others,41940,0,5,32.2179,0,10000257,JP,DE,ANALYST,Beijing
-317,2012-05-09,ABIN,65,0,11,96.0579,0,10000675,CN,DE,ANALYST,Beijing
-318,2012-10-30,Auction,80135,0,11,55.5335,0,10000288,FR,DE,ANALYST,Beijing
-319,2013-07-14,ABIN,11554,0,16,9.7012,0,10000210,CN,CN,MODELER,Hongkong
-320,2012-07-29,Others,11554,0,5,91.5938,0,10000736,UK,UK,ANALYST,Beijing
-321,2012-06-03,ABIN,80287,0,13,99.7431,0,10000403,US,US,ADMIN,Shanghai
-322,2012-10-10,Auction,36250,0,13,14.7056,0,10000561,UK,US,ANALYST,Beijing
-323,2013-06-20,ABIN,20485,0,5,23.912,0,10000798,DE,FR,ANALYST,Beijing
-324,2013-12-26,Others,11554,0,15,27.2028,0,10000154,DE,JP,MODELER,Hongkong
-325,2013-05-07,Auction,11848,0,-99,73.7354,0,10000448,CN,FR,MODELER,Hongkong
-326,2013-12-14,ABIN,61323,0,16,44.6086,0,10000555,CN,UK,ADMIN,Shanghai
-327,2012-08-29,FP-GTC,139973,0,14,34.7404,0,10000686,CN,JP,ANALYST,Beijing
-328,2013-12-31,FP-GTC,100847,0,15,70.6027,0,10000534,CN,FR,MODELER,Hongkong
-329,2013-12-13,FP-non GTC,16145,3,12,76.5802,0,10000864,US,FR,ADMIN,Shanghai
-330,2012-01-24,FP-non GTC,57990,0,-99,54.7904,0,10000325,US,CN,ADMIN,Shanghai
-331,2013-07-26,Others,20485,0,11,9.313,0,10000991,US,JP,MODELER,Hongkong
-332,2013-09-30,ABIN,31519,0,15,95.7797,0,10000701,JP,JP,ADMIN,Shanghai
-333,2012-03-30,FP-non GTC,88750,0,16,34.6777,0,10000494,US,CN,ANALYST,Beijing
-334,2012-05-11,Others,20485,0,16,2.2782,0,10000436,CN,CN,MODELER,Hongkong
-335,2012-11-21,FP-non GTC,156356,0,13,3.4276,0,10000574,CN,UK,MODELER,Hongkong
-336,2012-11-19,FP-non GTC,36250,0,16,42.1665,0,10000885,FR,DE,ADMIN,Shanghai
-337,2013-01-07,Others,20485,0,13,29.1424,0,10000931,FR,CN,ANALYST,Beijing
-338,2013-01-16,ABIN,88750,0,13,87.3858,0,10000276,US,FR,ADMIN,Shanghai
-339,2012-07-20,ABIN,25147,0,-99,41.9809,0,10000928,CN,DE,ADMIN,Shanghai
-340,2013-06-27,FP-non GTC,75665,0,11,65.3889,0,10000587,DE,FR,ADMIN,Shanghai
-341,2013-12-04,FP-non GTC,45333,0,15,41.7577,0,10000979,JP,CN,ADMIN,Shanghai
-342,2012-05-01,Auction,4943,0,11,76.3978,0,10000385,FR,CN,ANALYST,Beijing
-343,2013-05-03,Others,66767,15,-99,69.062,0,10000186,JP,DE,ANALYST,Beijing
-344,2013-02-14,FP-non GTC,95672,0,14,15.6152,0,10000356,UK,UK,ANALYST,Beijing
-345,2012-01-01,Others,1357,0,13,16.7271,0,10000263,CN,JP,ANALYST,Beijing
-346,2012-01-05,FP-non GTC,67703,3,5,32.2005,0,10000141,FR,US,ADMIN,Shanghai
-347,2013-12-15,Others,2023,0,-99,55.3349,0,10000920,UK,CN,ADMIN,Shanghai
-348,2012-10-02,FP-GTC,314,0,14,95.9217,0,10000647,DE,CN,ANALYST,Beijing
-349,2012-10-27,Others,64076,0,5,3.4266,0,10000486,JP,DE,ADMIN,Shanghai
-350,2013-10-17,Others,41940,0,15,3.0363,0,10000299,UK,CN,ADMIN,Shanghai
-351,2013-02-09,Others,13836,0,15,37.1746,0,10000799,CN,FR,MODELER,Hongkong
-352,2013-04-03,FP-GTC,31519,0,15,83.3031,0,10000656,CN,FR,ADMIN,Shanghai
-353,2013-08-30,Others,63861,0,13,37.862,0,10000958,CN,DE,ADMIN,Shanghai
-354,2012-01-01,Others,1349,0,13,0.7227,0,10000774,UK,UK,ADMIN,Shanghai
-355,2013-03-24,FP-non GTC,87118,0,13,32.5615,0,10000661,FR,DE,ADMIN,Shanghai
-356,2012-12-03,FP-non GTC,216,0,5,40.1447,0,10000923,US,JP,ANALYST,Beijing
-357,2012-07-12,FP-GTC,94847,0,5,74.4586,0,10000161,DE,CN,MODELER,Hongkong
-358,2013-03-18,Others,1161,3,-99,70.1521,0,10000123,CN,DE,ANALYST,Beijing
-359,2013-11-17,Auction,165888,0,5,16.1344,0,10000029,UK,UK,ANALYST,Beijing
-360,2013-11-25,FP-non GTC,44079,0,5,18.0996,0,10000461,CN,FR,MODELER,Hongkong
-361,2012-12-06,FP-GTC,159184,0,15,63.6868,0,10000399,US,CN,ANALYST,Beijing
-362,2013-09-27,FP-non GTC,161567,15,15,35.8966,0,10000283,JP,CN,MODELER,Hongkong
-363,2013-11-02,ABIN,20865,0,5,70.2863,0,10000724,CN,FR,ADMIN,Shanghai
-364,2013-04-29,Auction,314,0,13,53.6862,0,10000676,JP,CN,MODELER,Hongkong
-365,2013-01-21,Auction,20886,0,-99,32.5548,0,10000052,US,CN,MODELER,Hongkong
-366,2012-04-18,FP-GTC,67698,0,14,90.8206,0,10000221,CN,FR,ANALYST,Beijing
-367,2013-10-14,Others,46575,0,13,92.8392,0,10000187,FR,CN,ANALYST,Beijing
-368,2013-07-06,FP-GTC,26249,0,-99,54.6729,0,10000651,UK,JP,MODELER,Hongkong
-369,2012-09-15,Others,16145,0,15,70.0461,0,10000279,FR,CN,ADMIN,Shanghai
-370,2013-11-02,Others,38238,0,11,22.9595,0,10000451,US,UK,MODELER,Hongkong
-371,2012-06-03,ABIN,57013,0,11,28.9005,0,10000028,CN,CN,ADMIN,Shanghai
-372,2013-10-08,FP-GTC,20886,0,14,24.6958,0,10000676,CN,DE,ANALYST,Beijing
-373,2013-04-11,Auction,100847,0,14,42.1878,0,10000645,FR,UK,MODELER,Hongkong
-374,2013-11-07,Auction,45333,0,13,58.1089,0,10000182,US,CN,ANALYST,Beijing
-375,2012-10-18,FP-non GTC,57990,0,15,37.9049,0,10000696,UK,FR,ADMIN,Shanghai
-376,2012-11-28,FP-non GTC,63889,0,5,59.1268,0,10000116,FR,CN,ADMIN,Shanghai
-377,2012-03-02,Auction,45333,0,13,37.4799,0,10000298,CN,JP,ANALYST,Beijing
-378,2013-07-27,FP-non GTC,75665,0,12,19.423,0,10000333,UK,FR,ADMIN,Shanghai
-379,2012-12-01,Auction,44079,0,11,95.4596,0,10000163,FR,CN,ANALYST,Beijing
-380,2012-04-11,FP-non GTC,161567,15,13,47.4582,0,10000082,FR,CN,ADMIN,Shanghai
-381,2013-06-23,ABIN,175750,3,13,23.9511,0,10000560,CN,US,ANALYST,Beijing
-382,2012-02-17,Others,46575,0,15,25.7613,0,10000308,DE,FR,ADMIN,Shanghai
-383,2013-05-19,ABIN,223,0,15,64.7127,0,10000850,CN,CN,ANALYST,Beijing
-384,2012-10-29,FP-non GTC,159184,0,5,15.4188,0,10000995,US,DE,ADMIN,Shanghai
-385,2012-07-17,Auction,4943,0,11,32.1734,0,10000820,US,JP,ANALYST,Beijing
-386,2013-01-03,ABIN,57784,0,5,47.7542,0,10000563,FR,DE,MODELER,Hongkong
-387,2013-07-17,FP-GTC,150047,3,16,35.0905,0,10000545,DE,CN,ADMIN,Shanghai
-388,2013-07-20,Others,87118,0,14,24.6574,0,10000985,JP,JP,MODELER,Hongkong
-389,2013-12-14,Others,11848,0,14,80.9504,0,10000318,US,UK,MODELER,Hongkong
-390,2013-01-16,Auction,106246,0,12,94.2201,0,10000113,JP,FR,ADMIN,Shanghai
-391,2012-11-04,Others,63861,0,12,71.8354,0,10000985,DE,CN,MODELER,Hongkong
-392,2012-08-25,FP-non GTC,100847,0,5,68.7418,0,10000581,DE,CN,ADMIN,Shanghai
-393,2013-07-22,Auction,156356,0,15,63.6314,0,10000140,CN,FR,ANALYST,Beijing
-394,2013-11-04,Auction,67698,0,16,15.6061,0,10000128,CN,UK,MODELER,Hongkong
-395,2013-03-01,FP-GTC,20485,0,-99,49.2253,0,10000722,CN,US,MODELER,Hongkong
-396,2013-12-03,FP-GTC,46575,0,-99,71.5261,0,10000796,CN,DE,ADMIN,Shanghai
-397,2013-03-25,FP-non GTC,60340,0,11,23.0796,0,10000605,JP,JP,ADMIN,Shanghai
-398,2012-11-16,FP-non GTC,99985,0,11,91.6223,0,10000619,UK,DE,ANALYST,Beijing
-399,2012-05-24,Auction,16509,0,16,97.7044,0,10000002,US,CN,ADMIN,Shanghai
-400,2012-04-04,ABIN,139973,3,13,16.7393,0,10000923,UK,CN,MODELER,Hongkong
-401,2012-07-20,FP-GTC,174106,3,13,8.8874,0,10000134,CN,DE,MODELER,Hongkong
-402,2013-09-12,Auction,33038,15,13,46.7475,0,10000154,US,FR,MODELER,Hongkong
-403,2012-06-12,ABIN,57990,0,11,33.1479,0,10000036,CN,DE,ADMIN,Shanghai
-404,2013-08-31,ABIN,46575,0,12,41.0635,0,10000310,DE,UK,ADMIN,Shanghai
-405,2012-12-18,Auction,1349,0,5,14.6328,0,10000296,FR,JP,MODELER,Hongkong
-406,2012-02-18,ABIN,16509,0,-99,76.3163,0,10000486,DE,CN,MODELER,Hongkong
-407,2013-07-02,Auction,3838,0,11,95.7402,0,10000963,JP,FR,ANALYST,Beijing
-408,2012-06-28,Auction,145970,0,16,41.1425,0,10000646,JP,JP,ADMIN,Shanghai
-409,2013-11-03,FP-GTC,12688,0,5,73.9106,0,10000357,JP,CN,MODELER,Hongkong
-410,2012-03-30,FP-non GTC,16509,0,5,39.5703,0,10000495,CN,DE,MODELER,Hongkong
-411,2012-08-18,FP-non GTC,75665,0,11,61.4928,0,10000522,JP,JP,MODELER,Hongkong
-412,2013-07-29,Others,156356,0,16,98.2405,0,10000417,JP,FR,ANALYST,Beijing
-413,2013-07-19,Others,32876,0,12,68.5148,0,10000306,CN,FR,ANALYST,Beijing
-414,2012-02-13,ABIN,175750,0,16,64.5444,0,10000274,UK,CN,ANALYST,Beijing
-415,2013-02-04,FP-non GTC,13836,0,-99,65.7663,0,10000511,DE,CN,ADMIN,Shanghai
-416,2013-07-24,FP-GTC,165888,0,16,91.7808,0,10000330,UK,JP,ADMIN,Shanghai
-417,2012-04-25,Others,41940,0,-99,6.5053,0,10000357,US,CN,MODELER,Hongkong
-418,2013-11-28,ABIN,106340,15,12,71.4264,0,10000155,FR,DE,ANALYST,Beijing
-419,2012-09-16,FP-non GTC,170083,3,-99,80.2442,0,10000690,US,JP,ANALYST,Beijing
-420,2012-05-10,FP-GTC,16145,0,14,11.0775,0,10000372,CN,US,ANALYST,Beijing
-421,2013-10-28,FP-non GTC,15687,0,16,65.9436,0,10000032,US,CN,ANALYST,Beijing
-422,2013-05-17,FP-non GTC,94847,0,13,37.352,0,10000044,FR,JP,ADMIN,Shanghai
-423,2013-05-31,FP-GTC,223,0,5,93.3181,0,10000365,CN,US,MODELER,Hongkong
-424,2012-01-07,FP-non GTC,73506,0,13,13.1225,0,10000218,JP,CN,ANALYST,Beijing
-425,2013-07-01,FP-GTC,64076,0,15,78.1094,0,10000081,FR,DE,ADMIN,Shanghai
-426,2013-07-26,Others,67698,0,11,14.9245,0,10000775,CN,FR,ANALYST,Beijing
-427,2013-03-02,Auction,75665,0,5,84.2445,0,10000162,UK,US,ANALYST,Beijing
-428,2012-01-13,FP-GTC,1120,3,16,43.8855,0,10000157,JP,CN,MODELER,Hongkong
-429,2012-06-02,ABIN,53064,0,12,17.36,0,10000091,CN,JP,ADMIN,Shanghai
-430,2012-02-23,ABIN,31387,3,14,43.1757,0,10000071,US,US,ADMIN,Shanghai
-431,2013-10-22,FP-non GTC,158798,0,15,10.9759,0,10000341,CN,UK,MODELER,Hongkong
-432,2013-05-23,FP-GTC,11848,0,-99,7.523,0,10000277,DE,DE,MODELER,Hongkong
-433,2013-07-25,FP-GTC,43479,0,14,13.262,0,10000500,CN,UK,MODELER,Hongkong
-434,2012-03-03,FP-non GTC,11848,0,14,95.0683,0,10000164,CN,JP,ADMIN,Shanghai
-435,2012-01-09,ABIN,11554,0,15,6.8777,0,10000785,JP,FR,ANALYST,Beijing
-436,2013-05-08,ABIN,80287,0,14,95.532,0,10000595,CN,JP,ANALYST,Beijing
-437,2013-07-26,FP-GTC,121153,0,12,28.7213,0,10000829,JP,CN,MODELER,Hongkong
-438,2012-11-27,FP-GTC,31673,0,14,7.8033,0,10000498,FR,JP,ANALYST,Beijing
-439,2013-06-04,ABIN,43398,0,15,76.0187,0,10000061,CN,FR,ANALYST,Beijing
-440,2012-10-05,ABIN,15687,0,13,87.1235,0,10000649,US,CN,MODELER,Hongkong
-441,2013-06-29,Auction,61323,0,14,78.4,0,10000765,JP,DE,ANALYST,Beijing
-442,2012-07-25,ABIN,216,0,12,43.1556,0,10000597,FR,UK,ANALYST,Beijing
-443,2013-01-03,FP-non GTC,121153,0,12,70.1009,0,10000201,CN,FR,MODELER,Hongkong
-444,2013-10-22,FP-non GTC,170302,15,12,92.5959,0,10000151,UK,DE,MODELER,Hongkong
-445,2012-02-03,FP-non GTC,155226,0,12,68.6486,0,10000870,DE,CN,MODELER,Hongkong
-446,2013-03-30,FP-GTC,139973,0,11,93.3324,0,10000585,JP,US,ANALYST,Beijing
-447,2012-11-21,Others,20865,0,11,68.0525,0,10000115,DE,US,ADMIN,Shanghai
-448,2012-11-09,ABIN,11848,0,13,66.3745,0,10000652,UK,US,ADMIN,Shanghai
-449,2012-10-22,Auction,2023,0,13,52.4918,0,10000836,CN,FR,ANALYST,Beijing
-450,2013-01-07,Auction,164261,0,-99,56.468,0,10000560,FR,FR,ANALYST,Beijing
-451,2012-01-05,Others,80053,0,-99,34.1305,0,10000905,CN,DE,ADMIN,Shanghai
-452,2012-08-27,FP-GTC,64076,0,14,3.0691,0,10000389,CN,DE,ADMIN,Shanghai
-453,2013-06-23,Others,139973,0,16,89.7673,0,10000636,US,JP,ANALYST,Beijing
-454,2013-09-13,Auction,67698,0,15,62.9192,0,10000596,CN,US,MODELER,Hongkong
-455,2012-08-18,FP-GTC,43398,0,12,45.3384,0,10000382,US,FR,ADMIN,Shanghai
-456,2012-10-27,FP-non GTC,1120,3,5,84.6466,0,10000487,FR,CN,ANALYST,Beijing
-457,2012-05-24,Auction,24541,0,14,56.7041,0,10000366,US,UK,ADMIN,Shanghai
-458,2013-02-12,Auction,64076,0,13,1.136,0,10000089,CN,GB,ADMIN,Shanghai
-459,2012-02-19,FP-non GTC,43398,0,14,7.2416,0,10000129,UK,CN,ANALYST,Beijing
-460,2012-07-30,ABIN,48027,0,11,41.6704,0,10000761,UK,UK,MODELER,Hongkong
-461,2013-10-04,FP-non GTC,37831,0,16,31.9567,0,10000610,UK,CN,MODELER,Hongkong
-462,2013-11-18,Auction,223,0,16,21.8155,0,10000079,UK,UK,ANALYST,Beijing
-463,2012-06-08,Others,80135,0,16,90.0166,0,10000478,FR,UK,MODELER,Hongkong
-464,2012-10-25,Others,159184,0,16,92.0436,0,10000166,FR,JP,ANALYST,Beijing
-465,2013-10-22,ABIN,13987,0,15,62.5546,0,10000592,CN,DE,ADMIN,Shanghai
-466,2013-11-01,ABIN,103324,15,16,14.9475,0,10000924,CN,US,ADMIN,Shanghai
-467,2013-03-21,FP-GTC,1120,3,-99,29.5252,0,10000758,CN,CN,ADMIN,Shanghai
-468,2012-03-13,Others,759,0,16,62.4158,0,10000810,CN,DE,MODELER,Hongkong
-469,2012-05-14,Auction,95672,0,16,92.0781,0,10000609,JP,JP,ANALYST,Beijing
-470,2013-09-28,FP-GTC,64076,0,12,37.3599,0,10000487,US,JP,ADMIN,Shanghai
-471,2013-10-09,ABIN,60606,3,13,48.9467,0,10000007,CN,US,ANALYST,Beijing
-472,2013-04-21,ABIN,26249,0,14,67.152,0,10000257,JP,UK,MODELER,Hongkong
-473,2013-05-08,FP-GTC,20886,0,13,19.5685,0,10000122,CN,CN,ANALYST,Beijing
-474,2013-12-29,ABIN,1357,0,12,89.9069,0,10000900,FR,CN,MODELER,Hongkong
-475,2013-02-14,FP-GTC,63889,0,16,74.5657,0,10000676,CN,GB,ANALYST,Beijing
-476,2012-02-25,Auction,88750,0,-99,92.3939,0,10000769,FR,CN,MODELER,Hongkong
-477,2012-06-22,Others,13836,0,14,77.1277,0,10000795,DE,JP,MODELER,Hongkong
-478,2012-01-30,FP-GTC,43479,0,15,48.2346,0,10000013,US,UK,ANALYST,Beijing
-479,2013-08-17,ABIN,57990,3,-99,87.0638,0,10000277,FR,DE,ADMIN,Shanghai
-480,2013-06-24,FP-GTC,1357,0,15,75.0985,0,10000056,CN,CN,ANALYST,Beijing
-481,2013-04-08,ABIN,57013,0,11,21.4485,0,10000195,JP,JP,ANALYST,Beijing
-482,2012-04-07,FP-non GTC,24541,0,-99,40.6197,0,10000313,CN,GB,ANALYST,Beijing
-483,2012-07-26,Others,963,0,16,99.8137,0,10000257,DE,US,ANALYST,Beijing
-484,2012-07-18,Others,15115,0,-99,9.9153,0,10000937,CN,JP,MODELER,Hongkong
-485,2012-01-04,Auction,63889,0,13,84.5004,0,10000467,UK,FR,MODELER,Hongkong
-486,2013-05-25,Auction,1120,3,14,40.3618,0,10000076,JP,CN,MODELER,Hongkong
-487,2013-03-30,FP-GTC,38238,0,12,94.987,0,10000997,UK,FR,ANALYST,Beijing
-488,2012-05-12,Auction,32876,0,-99,38.58,0,10000943,DE,CN,ANALYST,Beijing
-489,2012-11-21,FP-GTC,100847,0,14,7.7495,0,10000588,US,FR,MODELER,Hongkong
-490,2013-10-02,Auction,15868,0,5,92.2309,0,10000520,FR,UK,ADMIN,Shanghai
-491,2013-11-05,ABIN,31673,0,-99,75.9838,0,10000344,CN,US,ANALYST,Beijing
-492,2013-03-23,FP-non GTC,48904,0,-99,15.426,0,10000090,DE,FR,MODELER,Hongkong
-493,2012-12-27,FP-non GTC,43479,0,5,35.7689,0,10000066,US,CN,ANALYST,Beijing
-494,2013-01-11,FP-non GTC,223,0,12,5.9423,0,10000174,UK,DE,ANALYST,Beijing
-495,2012-05-25,Auction,50508,0,16,81.5853,0,10000337,US,CN,MODELER,Hongkong
-496,2012-09-08,Others,6762,0,15,11.8473,0,10000718,CN,GB,ANALYST,Beijing
-497,2013-07-08,Auction,139973,0,5,11.7025,0,10000537,FR,FR,ANALYST,Beijing
-498,2013-04-23,ABIN,32876,0,12,40.4116,0,10000798,CN,FR,ADMIN,Shanghai
-499,2013-03-22,Auction,2023,0,15,89.8093,0,10000662,UK,US,ANALYST,Beijing
-500,2012-11-01,Others,75708,3,11,95.9527,0,10000776,FR,JP,MODELER,Hongkong
-501,2012-01-29,ABIN,156356,0,16,64.6181,0,10000910,FR,JP,ADMIN,Shanghai
-502,2012-01-20,Others,2635,0,11,81.6196,0,10000893,US,CN,ANALYST,Beijing
-503,2012-05-27,FP-non GTC,94847,0,15,87.1713,0,10000121,JP,UK,ANALYST,Beijing
-504,2013-12-17,Others,156614,0,14,87.7791,0,10000531,US,FR,MODELER,Hongkong
-505,2013-03-05,ABIN,80135,0,16,64.8966,0,10000452,US,FR,MODELER,Hongkong
-506,2013-01-26,ABIN,156356,0,16,59.0108,0,10000671,CN,JP,ADMIN,Shanghai
-507,2012-01-08,FP-non GTC,57990,0,15,56.347,0,10000028,DE,JP,ADMIN,Shanghai
-508,2013-08-04,ABIN,13836,0,11,98.5679,0,10000481,JP,US,ADMIN,Shanghai
-509,2013-04-04,Others,63861,0,13,14.3911,0,10000944,CN,JP,ANALYST,Beijing
-510,2013-02-15,FP-GTC,2023,0,5,5.5878,0,10000816,CN,GB,MODELER,Hongkong
-511,2013-05-05,FP-non GTC,175750,0,-99,80.9471,0,10000727,FR,JP,MODELER,Hongkong
-512,2013-06-24,FP-GTC,38238,0,12,96.7825,0,10000202,CN,US,ANALYST,Beijing
-513,2013-11-04,ABIN,45333,0,14,57.954,0,10000767,DE,FR,ANALYST,Beijing
-514,2012-02-23,FP-non GTC,57990,0,5,16.479,0,10000146,FR,UK,ADMIN,Shanghai
-515,2013-01-13,ABIN,158798,0,11,45.5371,0,10000133,JP,US,ANALYST,Beijing
-516,2013-11-16,ABIN,73506,0,16,35.7207,0,10000304,CN,FR,ANALYST,Beijing
-517,2012-11-15,Auction,64076,0,15,66.0059,0,10000375,CN,DE,ANALYST,Beijing
-518,2012-01-03,ABIN,156614,0,15,84.8759,0,10000095,JP,UK,ADMIN,Shanghai
-519,2012-09-22,Auction,57990,0,14,42.7313,0,10000953,CN,CN,MODELER,Hongkong
-520,2013-11-03,FP-non GTC,67698,0,11,8.9322,0,10000913,JP,DE,MODELER,Hongkong
-521,2012-08-31,ABIN,1504,0,5,34.3706,0,10000748,FR,UK,ANALYST,Beijing
-522,2013-12-13,FP-GTC,15868,0,-99,18.3204,0,10000495,DE,FR,MODELER,Hongkong
-523,2012-07-20,ABIN,533,0,11,24.9451,0,10000237,CN,CN,MODELER,Hongkong
-524,2012-07-14,ABIN,41940,0,11,42.9796,0,10000328,UK,DE,ANALYST,Beijing
-525,2013-11-06,Others,32876,0,15,4.2693,0,10000383,US,FR,ADMIN,Shanghai
-526,2012-07-02,Auction,41940,0,12,19.2987,0,10000196,JP,CN,ADMIN,Shanghai
-527,2012-06-21,FP-GTC,20865,0,-99,36.5029,0,10000098,JP,FR,ANALYST,Beijing
-528,2012-09-19,ABIN,159184,0,15,65.9768,0,10000517,JP,DE,ANALYST,Beijing
-529,2012-02-11,Others,16145,0,14,41.4762,0,10000733,FR,CN,ANALYST,Beijing
-530,2012-07-19,Auction,63861,0,11,30.7294,0,10000351,UK,JP,ANALYST,Beijing
-531,2013-05-24,ABIN,1349,0,12,61.0346,0,10000078,JP,JP,MODELER,Hongkong
-532,2013-06-26,Auction,139973,3,11,49.3849,0,10000537,UK,DE,ADMIN,Shanghai
-533,2013-11-30,Others,1357,0,16,62.0388,0,10000532,CN,US,MODELER,Hongkong
-534,2012-11-01,Others,121153,0,5,64.8814,0,10000820,FR,US,ANALYST,Beijing
-535,2013-10-20,Auction,20485,0,5,88.9093,0,10000467,DE,US,MODELER,Hongkong
-536,2013-04-15,FP-GTC,43479,0,13,72.5969,0,10000352,UK,UK,MODELER,Hongkong
-537,2012-04-02,FP-GTC,33977,15,16,42.1845,0,10000614,UK,FR,ANALYST,Beijing
-538,2012-03-20,ABIN,2023,0,16,95.0254,0,10000014,US,CN,ADMIN,Shanghai
-539,2013-02-21,FP-non GTC,57990,0,5,25.0404,0,10000386,US,JP,ANALYST,Beijing
-540,2013-06-02,FP-non GTC,6762,0,12,60.3684,0,10000783,US,US,ADMIN,Shanghai
-541,2012-01-07,FP-GTC,759,0,11,80.172,0,10000952,JP,CN,MODELER,Hongkong
-542,2012-12-01,FP-non GTC,533,0,12,47.1653,0,10000550,DE,CN,ADMIN,Shanghai
-543,2012-11-03,ABIN,99985,0,16,40.828,0,10000629,US,UK,MODELER,Hongkong
-544,2013-10-15,FP-GTC,64076,0,15,55.9541,0,10000531,UK,CN,ANALYST,Beijing
-545,2013-12-30,Auction,80135,0,13,13.2503,0,10000425,CN,US,ADMIN,Shanghai
-546,2012-06-28,FP-GTC,15868,0,12,76.836,0,10000557,JP,CN,MODELER,Hongkong
-547,2013-04-07,FP-GTC,10866,0,16,37.754,0,10000810,CN,GB,ADMIN,Shanghai
-548,2013-08-03,FP-GTC,139255,15,13,70.8704,0,10000514,CN,GB,MODELER,Hongkong
-549,2012-07-23,ABIN,121153,0,5,65.6362,0,10000166,FR,DE,MODELER,Hongkong
-550,2013-05-04,FP-GTC,31519,0,5,38.9751,0,10000932,CN,FR,ANALYST,Beijing
-551,2013-01-27,ABIN,175750,3,-99,81.7736,0,10000529,UK,FR,ADMIN,Shanghai
-552,2013-03-30,ABIN,11554,0,15,37.8703,0,10000509,DE,CN,MODELER,Hongkong
-553,2012-06-03,FP-GTC,20485,0,-99,62.4729,0,10000404,FR,DE,MODELER,Hongkong
-554,2012-05-09,FP-non GTC,63861,0,11,35.1334,0,10000175,CN,JP,ADMIN,Shanghai
-555,2013-12-25,FP-non GTC,6762,0,12,95.0363,0,10000101,DE,US,ANALYST,Beijing
-556,2012-11-18,FP-GTC,175750,0,5,95.8938,0,10000537,UK,UK,ADMIN,Shanghai
-557,2013-01-07,FP-non GTC,57784,0,14,0.2995,0,10000847,JP,CN,ADMIN,Shanghai
-558,2012-03-11,Auction,63861,0,12,58.1028,0,10000912,US,UK,ANALYST,Beijing
-559,2012-03-23,Others,80287,0,14,24.79,0,10000967,CN,US,ADMIN,Shanghai
-560,2012-03-19,ABIN,99985,0,16,5.9564,0,10000684,UK,FR,ADMIN,Shanghai
-561,2013-11-24,Auction,20886,0,15,28.1384,0,10000239,FR,CN,ADMIN,Shanghai
-562,2012-04-04,FP-GTC,100847,0,11,91.4446,0,10000743,CN,CN,ADMIN,Shanghai
-563,2013-12-15,ABIN,44079,0,16,39.686,0,10000476,DE,FR,MODELER,Hongkong
-564,2013-10-06,FP-GTC,132939,0,12,20.8855,0,10000854,JP,DE,ADMIN,Shanghai
-565,2013-12-07,Others,25147,0,14,16.4357,0,10000198,UK,CN,ADMIN,Shanghai
-566,2012-02-04,Others,158798,0,13,44.5451,0,10000543,FR,CN,MODELER,Hongkong
-567,2013-11-07,FP-GTC,10866,0,16,8.5249,0,10000105,DE,UK,ADMIN,Shanghai
-568,2012-08-04,FP-GTC,2635,0,12,11.7572,0,10000382,UK,JP,ADMIN,Shanghai
-569,2012-08-14,Auction,152801,0,15,46.6818,0,10000205,CN,US,ADMIN,Shanghai
-570,2013-02-25,Auction,50508,0,15,37.8443,0,10000644,DE,FR,MODELER,Hongkong
-571,2013-10-09,ABIN,223,0,13,58.8049,0,10000768,US,JP,ANALYST,Beijing
-572,2012-01-10,Auction,20485,0,11,92.2997,0,10000975,DE,US,ADMIN,Shanghai
-573,2013-05-03,FP-non GTC,37831,0,12,15.9399,0,10000280,CN,CN,MODELER,Hongkong
-574,2013-11-12,ABIN,73506,0,12,13.9272,0,10000332,UK,FR,ANALYST,Beijing
-575,2013-08-16,Others,164,0,12,39.2371,0,10000887,DE,FR,MODELER,Hongkong
-576,2013-10-27,FP-non GTC,15868,0,16,51.4769,0,10000303,UK,US,MODELER,Hongkong
-577,2013-04-27,FP-non GTC,80287,0,15,90.8971,0,10000207,DE,CN,MODELER,Hongkong
-578,2013-11-06,FP-non GTC,11554,0,12,26.0123,0,10000855,FR,CN,ADMIN,Shanghai
-579,2013-11-25,Others,6762,0,14,96.0198,0,10000418,JP,CN,ANALYST,Beijing
-580,2012-08-09,ABIN,80135,0,15,89.0544,0,10000861,JP,DE,MODELER,Hongkong
-581,2013-09-28,ABIN,16509,0,12,90.2997,0,10000520,US,US,MODELER,Hongkong
-582,2012-03-19,FP-GTC,43479,0,-99,93.8319,0,10000045,US,US,MODELER,Hongkong
-583,2012-05-25,FP-GTC,139973,0,16,8.9816,0,10000684,JP,US,ADMIN,Shanghai
-584,2013-04-04,Auction,25147,0,11,43.4557,0,10000801,UK,CN,MODELER,Hongkong
-585,2013-07-07,Auction,67698,0,14,59.3634,0,10000305,UK,DE,ADMIN,Shanghai
-586,2013-10-27,ABIN,63861,0,14,48.1526,0,10000963,UK,CN,MODELER,Hongkong
-587,2012-05-31,Auction,130,0,5,1.056,0,10000357,CN,CN,MODELER,Hongkong
-588,2012-08-21,Auction,13836,0,16,14.4247,0,10000278,JP,US,ADMIN,Shanghai
-589,2013-07-24,Auction,15568,15,14,14.5748,0,10000590,CN,CN,ADMIN,Shanghai
-590,2013-04-05,ABIN,12688,0,12,2.0636,0,10000951,CN,CN,ADMIN,Shanghai
-591,2013-02-27,FP-non GTC,73506,0,14,2.7139,0,10000134,JP,JP,ADMIN,Shanghai
-592,2013-06-16,FP-non GTC,164,0,5,59.4198,0,10000745,DE,US,ANALYST,Beijing
-593,2012-04-01,Others,20213,0,11,77.849,0,10000929,FR,UK,ANALYST,Beijing
-594,2013-12-02,FP-non GTC,155226,0,16,71.11,0,10000433,CN,CN,ADMIN,Shanghai
-595,2013-06-07,Auction,158798,0,15,98.6517,0,10000320,US,DE,ADMIN,Shanghai
-596,2012-11-25,ABIN,31519,0,12,68.8974,0,10000726,CN,DE,ADMIN,Shanghai
-597,2013-04-17,FP-GTC,57013,0,14,49.4405,0,10000050,CN,CN,ADMIN,Shanghai
-598,2013-03-18,FP-non GTC,175750,0,13,2.3784,0,10000559,US,FR,ANALYST,Beijing
-599,2013-08-13,Others,62179,0,11,36.494,0,10000652,CN,JP,MODELER,Hongkong
-600,2012-10-15,ABIN,175750,0,14,42.7269,0,10000560,CN,DE,MODELER,Hongkong
-601,2012-01-08,Others,6762,0,5,58.5212,0,10000829,FR,CN,ADMIN,Shanghai
-602,2013-05-20,FP-non GTC,216,0,14,16.9644,0,10000104,CN,DE,ADMIN,Shanghai
-603,2012-12-07,Others,51582,0,5,6.0617,0,10000228,DE,CN,ANALYST,Beijing
-604,2013-12-04,ABIN,57013,0,16,48.2921,0,10000347,UK,UK,MODELER,Hongkong
-605,2013-12-17,ABIN,759,0,14,12.8117,0,10000161,CN,DE,ANALYST,Beijing
-606,2012-02-08,FP-non GTC,24760,0,13,11.319,0,10000938,CN,US,MODELER,Hongkong
-607,2013-01-03,FP-non GTC,15568,15,14,0.3508,0,10000964,UK,CN,ADMIN,Shanghai
-608,2013-04-21,Others,87118,0,12,9.5385,0,10000949,UK,JP,ADMIN,Shanghai
-609,2013-11-13,Others,36250,0,12,0.9029,0,10000532,UK,JP,ANALYST,Beijing
-610,2013-03-18,FP-GTC,164,0,12,3.7727,0,10000237,FR,FR,ADMIN,Shanghai
-611,2013-06-29,Others,15868,0,11,23.3421,0,10000671,FR,FR,MODELER,Hongkong
-612,2013-12-25,Others,25147,0,5,99.4168,0,10000708,JP,JP,ANALYST,Beijing
-613,2012-10-06,FP-GTC,533,0,12,40.5069,0,10000295,JP,US,MODELER,Hongkong
-614,2013-02-21,ABIN,216,0,14,84.3818,0,10000637,UK,DE,ADMIN,Shanghai
-615,2012-08-27,FP-non GTC,67698,0,16,7.0299,0,10000671,DE,CN,MODELER,Hongkong
-616,2013-04-13,Others,51582,0,13,67.973,0,10000778,CN,FR,ANALYST,Beijing
-617,2013-07-10,FP-GTC,139973,0,13,24.8423,0,10000253,UK,US,ADMIN,Shanghai
-618,2013-11-02,ABIN,1349,0,14,56.631,0,10000922,DE,DE,ANALYST,Beijing
-619,2013-05-11,Others,26249,0,16,49.7279,0,10000610,CN,FR,ANALYST,Beijing
-620,2013-01-17,FP-non GTC,118687,3,11,34.6913,0,10000002,JP,UK,MODELER,Hongkong
-621,2012-07-02,Others,148324,15,-99,90.6687,0,10000613,US,UK,ANALYST,Beijing
-622,2013-07-14,ABIN,41940,0,-99,60.2733,0,10000145,FR,FR,MODELER,Hongkong
-623,2013-06-13,Auction,63889,0,11,87.9644,0,10000920,CN,JP,MODELER,Hongkong
-624,2013-05-05,FP-non GTC,1120,3,16,99.4791,0,10000720,CN,FR,ADMIN,Shanghai
-625,2012-03-30,Others,4943,0,14,56.9643,0,10000105,US,CN,ANALYST,Beijing
-626,2012-01-13,FP-non GTC,139973,3,-99,15.838,0,10000587,CN,GB,ANALYST,Beijing
-627,2012-09-02,ABIN,12688,0,13,69.4513,0,10000756,CN,CN,ANALYST,Beijing
-628,2013-08-08,Others,121153,0,11,29.2318,0,10000289,JP,CN,ADMIN,Shanghai
-629,2013-03-09,Others,99985,0,15,14.7661,0,10000453,US,FR,MODELER,Hongkong
-630,2012-10-17,FP-non GTC,25147,0,11,96.1909,0,10000054,UK,US,ADMIN,Shanghai
-631,2012-01-20,Others,73506,0,14,56.0157,0,10000909,US,CN,ADMIN,Shanghai
-632,2012-04-15,FP-GTC,16145,0,12,94.6354,0,10000551,UK,US,ANALYST,Beijing
-633,2013-08-08,Others,3838,0,13,85.7695,0,10000464,CN,US,ANALYST,Beijing
-634,2013-01-21,Others,139973,0,13,64.5021,0,10000107,US,FR,MODELER,Hongkong
-635,2012-06-17,Others,139973,0,-99,72.8065,0,10000495,CN,FR,MODELER,Hongkong
-636,2013-12-31,Others,20886,0,15,91.2326,0,10000746,DE,US,MODELER,Hongkong
-637,2012-02-11,FP-non GTC,4943,0,14,28.3146,0,10000649,CN,JP,ANALYST,Beijing
-638,2013-10-01,Auction,155226,0,15,72.2196,0,10000576,DE,DE,MODELER,Hongkong
-639,2012-08-25,ABIN,223,0,15,17.6408,0,10000661,CN,US,ANALYST,Beijing
-640,2012-07-10,Others,67698,0,15,70.9451,0,10000115,CN,DE,ANALYST,Beijing
-641,2013-12-27,FP-non GTC,26262,0,13,64.5694,0,10000740,FR,DE,MODELER,Hongkong
-642,2012-02-05,Auction,32996,15,12,9.9521,0,10000833,UK,US,ANALYST,Beijing
-643,2012-11-07,FP-non GTC,25147,0,16,83.3886,0,10000494,FR,DE,MODELER,Hongkong
-644,2014-01-01,Others,45333,0,16,19.2606,0,10000869,US,JP,ADMIN,Shanghai
-645,2013-10-05,Auction,50508,0,14,39.1465,0,10000904,UK,UK,ADMIN,Shanghai
-646,2012-02-25,FP-GTC,37831,0,11,12.4616,0,10000680,US,FR,ANALYST,Beijing
-647,2012-05-16,Others,31519,0,12,66.9268,0,10000548,US,DE,ADMIN,Shanghai
-648,2013-11-08,FP-GTC,100847,0,11,47.751,0,10000699,JP,JP,ADMIN,Shanghai
-649,2012-03-15,ABIN,106246,0,16,96.1592,0,10000922,DE,DE,ADMIN,Shanghai
-650,2013-05-15,Auction,64076,0,15,37.743,0,10000148,CN,UK,ADMIN,Shanghai
-651,2012-11-03,Auction,61323,0,15,14.3851,0,10000353,CN,US,ADMIN,Shanghai
-652,2012-07-28,ABIN,175750,0,-99,45.5415,0,10000869,CN,CN,ANALYST,Beijing
-653,2013-06-26,FP-non GTC,95672,0,14,90.732,0,10000507,JP,FR,ANALYST,Beijing
-654,2012-09-11,Others,223,0,15,10.5185,0,10000705,UK,US,ANALYST,Beijing
-655,2012-09-24,FP-GTC,43479,0,16,3.4611,0,10000991,CN,CN,ANALYST,Beijing
-656,2013-09-13,FP-GTC,100847,0,-99,51.0213,0,10000871,JP,FR,ANALYST,Beijing
-657,2013-07-01,Others,132939,0,16,14.6584,0,10000859,DE,FR,ADMIN,Shanghai
-658,2012-06-13,Others,26262,0,-99,46.0446,0,10000041,CN,US,ANALYST,Beijing
-659,2013-10-26,Auction,3838,0,12,24.8196,0,10000561,CN,GB,ADMIN,Shanghai
-660,2012-03-05,Others,26262,0,13,66.7748,0,10000859,DE,FR,ANALYST,Beijing
-661,2012-12-10,FP-GTC,20485,0,12,52.3473,0,10000171,DE,JP,ANALYST,Beijing
-662,2013-11-15,Others,2023,0,5,84.6955,0,10000451,UK,CN,ADMIN,Shanghai
-663,2013-12-12,ABIN,80135,0,11,95.0112,0,10000718,FR,FR,ANALYST,Beijing
-664,2013-04-16,ABIN,106340,15,16,14.6367,0,10000822,US,UK,ANALYST,Beijing
-665,2012-03-10,Auction,161567,15,16,5.28,0,10000081,JP,JP,ANALYST,Beijing
-666,2013-09-19,Auction,11848,0,11,27.6166,0,10000388,FR,JP,ADMIN,Shanghai
-667,2013-12-28,FP-GTC,216,0,15,90.1257,0,10000122,CN,US,MODELER,Hongkong
-668,2012-01-26,ABIN,314,0,12,3.5634,0,10000912,CN,CN,ANALYST,Beijing
-669,2013-12-13,FP-GTC,158798,0,5,41.6652,0,10000485,CN,DE,MODELER,Hongkong
-670,2013-09-26,FP-non GTC,80135,0,11,20.5746,0,10000138,US,FR,ANALYST,Beijing
-671,2012-01-20,Auction,62179,0,14,94.2904,0,10000310,US,CN,ANALYST,Beijing
-672,2013-04-20,ABIN,963,0,5,68.3604,0,10000571,US,US,ADMIN,Shanghai
-673,2012-02-03,FP-GTC,164,0,11,94.0501,0,10000120,CN,CN,MODELER,Hongkong
-674,2013-10-14,Others,2023,0,12,84.9792,0,10000371,US,UK,MODELER,Hongkong
-675,2013-08-19,FP-GTC,57990,3,14,57.0343,0,10000171,JP,JP,MODELER,Hongkong
-676,2012-06-18,Others,963,0,12,44.4186,0,10000343,JP,DE,MODELER,Hongkong
-677,2012-10-18,Others,31519,0,12,50.6753,0,10000875,FR,DE,MODELER,Hongkong
-678,2013-07-09,Others,36250,0,16,99.5251,0,10000446,FR,UK,MODELER,Hongkong
-679,2012-01-19,ABIN,38238,0,14,0.1259,0,10000242,CN,JP,ADMIN,Shanghai
-680,2013-05-19,FP-non GTC,13836,0,16,92.7668,0,10000895,CN,JP,MODELER,Hongkong
-681,2013-01-29,Others,95672,0,5,13.6012,0,10000843,FR,DE,ADMIN,Shanghai
-682,2013-01-30,ABIN,67698,0,5,13.9968,0,10000424,CN,FR,MODELER,Hongkong
-683,2012-09-02,FP-GTC,223,0,-99,38.0021,0,10000803,UK,UK,ADMIN,Shanghai
-684,2012-05-11,Auction,11848,0,13,7.191,0,10000964,US,CN,ADMIN,Shanghai
-685,2013-05-14,FP-non GTC,759,0,16,43.172,0,10000123,JP,JP,ANALYST,Beijing
-686,2013-03-06,ABIN,3838,0,16,59.8866,0,10000653,DE,UK,ADMIN,Shanghai
-687,2012-08-23,FP-GTC,20213,0,16,91.3442,0,10000615,JP,JP,ANALYST,Beijing
-688,2013-03-24,FP-GTC,4943,0,-99,18.4365,0,10000718,DE,UK,MODELER,Hongkong
-689,2012-01-26,FP-non GTC,11554,0,14,41.6729,0,10000174,JP,FR,ADMIN,Shanghai
-690,2013-10-07,Others,108782,15,14,14.913,0,10000643,CN,GB,ADMIN,Shanghai
-691,2013-07-25,Others,13987,0,-99,28.7021,0,10000154,UK,US,MODELER,Hongkong
-692,2013-11-08,FP-non GTC,31673,0,13,83.8884,0,10000059,JP,FR,MODELER,Hongkong
-693,2013-01-06,ABIN,82494,15,12,74.4582,0,10000023,FR,FR,MODELER,Hongkong
-694,2013-01-21,FP-non GTC,13987,0,11,72.4276,0,10000541,FR,US,ANALYST,Beijing
-695,2013-11-22,Auction,67698,0,11,53.1908,0,10000553,FR,JP,ANALYST,Beijing
-696,2012-02-02,FP-non GTC,175750,0,12,79.116,0,10000121,JP,CN,ANALYST,Beijing
-697,2012-10-14,FP-GTC,24760,0,11,70.5894,0,10000677,DE,CN,ADMIN,Shanghai
-698,2012-10-20,ABIN,73506,0,11,3.7214,0,10000076,CN,JP,ANALYST,Beijing
-699,2013-12-01,Others,16145,0,15,54.8674,0,10000092,DE,CN,MODELER,Hongkong
-700,2012-11-07,FP-non GTC,53064,0,14,53.4385,0,10000267,CN,JP,MODELER,Hongkong
-701,2012-04-20,FP-non GTC,53064,0,15,76.5825,0,10000551,JP,DE,ANALYST,Beijing
-702,2012-11-18,Auction,50677,0,11,44.5149,0,10000924,CN,CN,MODELER,Hongkong
-703,2013-07-31,FP-GTC,51582,0,14,86.5325,0,10000652,UK,CN,MODELER,Hongkong
-704,2012-12-25,Others,31387,3,13,79.7624,0,10000495,JP,CN,ADMIN,Shanghai
-705,2012-08-10,ABIN,2635,0,12,54.5975,0,10000453,FR,CN,ADMIN,Shanghai
-706,2012-06-09,Others,31519,0,15,20.1828,0,10000226,CN,US,ANALYST,Beijing
-707,2012-09-16,Others,44079,0,16,46.0626,0,10000193,FR,US,ANALYST,Beijing
-708,2012-04-23,Others,16145,0,11,60.2064,0,10000058,US,CN,ANALYST,Beijing
-709,2012-01-03,Others,164,0,15,94.1749,0,10000509,FR,UK,ADMIN,Shanghai
-710,2012-05-16,FP-GTC,13987,0,15,5.6834,0,10000052,UK,DE,ADMIN,Shanghai
-711,2012-07-24,FP-non GTC,16145,0,15,8.1625,0,10000036,UK,UK,ADMIN,Shanghai
-712,2013-01-01,FP-non GTC,32996,15,15,75.5057,0,10000393,CN,US,ADMIN,Shanghai
-713,2013-01-28,FP-non GTC,155226,0,16,49.2395,0,10000966,JP,US,ANALYST,Beijing
-714,2013-03-19,FP-GTC,20485,0,5,57.4433,0,10000138,US,CN,ADMIN,Shanghai
-715,2013-02-03,FP-GTC,57784,0,5,11.8152,0,10000731,US,DE,ADMIN,Shanghai
-716,2012-01-02,Others,53064,0,-99,45.8162,0,10000785,CN,UK,ANALYST,Beijing
-717,2013-02-23,ABIN,16145,0,14,25.1092,0,10000582,CN,FR,ANALYST,Beijing
-718,2013-04-19,Auction,159184,0,12,85.351,0,10000422,CN,DE,ANALYST,Beijing
-719,2012-10-12,FP-non GTC,139973,0,11,95.7331,0,10000791,FR,UK,ANALYST,Beijing
-720,2012-04-20,FP-non GTC,87118,0,16,15.2148,0,10000621,CN,UK,ADMIN,Shanghai
-721,2013-04-30,Auction,53064,0,5,89.5035,0,10000377,JP,DE,MODELER,Hongkong
-722,2013-10-24,FP-non GTC,73506,0,12,84.0199,0,10000503,UK,UK,ANALYST,Beijing
-723,2012-01-08,Auction,63861,0,13,74.886,0,10000589,JP,DE,ADMIN,Shanghai
-724,2012-05-21,Others,63861,3,16,63.7247,0,10000730,FR,DE,ANALYST,Beijing
-725,2013-01-29,Auction,24541,0,15,22.2189,0,10000167,DE,CN,ANALYST,Beijing
-726,2012-02-08,Auction,533,0,5,53.5867,0,10000564,UK,FR,MODELER,Hongkong
-727,2012-08-11,FP-non GTC,174053,3,13,0.5792,0,10000568,FR,JP,ANALYST,Beijing
-728,2012-07-05,FP-GTC,20886,0,12,31.1464,0,10000806,JP,UK,MODELER,Hongkong
-729,2012-01-08,Auction,156356,0,5,7.4293,0,10000638,DE,US,MODELER,Hongkong
-730,2012-02-14,Others,65,0,14,62.6745,0,10000253,FR,UK,ADMIN,Shanghai
-731,2013-03-25,FP-non GTC,48027,0,-99,53.489,0,10000899,CN,GB,MODELER,Hongkong
-732,2012-09-30,Others,57013,0,12,75.9109,0,10000274,US,JP,MODELER,Hongkong
-733,2013-09-01,ABIN,32876,0,12,75.0498,0,10000086,FR,UK,ADMIN,Shanghai
-734,2013-11-11,FP-GTC,164,0,12,36.6051,0,10000521,JP,US,MODELER,Hongkong
-735,2012-02-18,FP-GTC,57013,0,12,14.3117,0,10000763,FR,UK,ANALYST,Beijing
-736,2013-11-23,ABIN,139973,0,15,58.3344,0,10000787,DE,FR,ANALYST,Beijing
-737,2012-03-21,FP-non GTC,41940,0,15,38.2502,0,10000342,US,DE,MODELER,Hongkong
-738,2012-02-24,Auction,57990,0,5,73.5795,0,10000175,CN,FR,ANALYST,Beijing
-739,2013-07-20,Auction,88750,0,-99,58.2532,0,10000122,FR,UK,ANALYST,Beijing
-740,2012-10-17,Others,43479,0,13,67.9585,0,10000516,CN,DE,ANALYST,Beijing
-741,2012-11-03,FP-non GTC,118687,3,13,90.9173,0,10000957,DE,DE,ADMIN,Shanghai
-742,2012-10-22,FP-GTC,60340,0,14,68.5877,0,10000502,FR,CN,ANALYST,Beijing
-743,2013-04-25,FP-GTC,11554,0,5,60.828,0,10000074,FR,FR,ADMIN,Shanghai
-744,2012-10-23,FP-GTC,24760,0,13,6.0732,0,10000138,US,CN,ANALYST,Beijing
-745,2013-04-29,ABIN,175750,0,11,8.0341,0,10000077,UK,CN,ADMIN,Shanghai
-746,2012-05-06,Auction,106340,15,14,8.0384,0,10000234,CN,CN,MODELER,Hongkong
-747,2012-07-15,ABIN,57784,0,14,27.1882,0,10000729,UK,FR,MODELER,Hongkong
-748,2012-10-15,Auction,67698,0,-99,63.179,0,10000226,UK,US,MODELER,Hongkong
-749,2013-03-22,FP-non GTC,57013,0,14,27.7251,0,10000658,FR,CN,ADMIN,Shanghai
-750,2013-08-12,FP-non GTC,11848,0,13,38.5597,0,10000020,FR,CN,ANALYST,Beijing
-751,2012-07-14,FP-GTC,80053,0,-99,81.0002,0,10000850,UK,JP,ADMIN,Shanghai
-752,2012-08-12,FP-non GTC,38238,0,11,58.6987,0,10000377,DE,FR,ANALYST,Beijing
-753,2013-09-09,FP-GTC,106246,0,11,5.0352,0,10000594,CN,JP,MODELER,Hongkong
-754,2013-02-11,FP-non GTC,57784,0,12,80.842,0,10000657,CN,CN,ADMIN,Shanghai
-755,2013-03-09,Others,175750,0,14,87.9884,0,10000152,CN,FR,MODELER,Hongkong
-756,2013-12-18,FP-non GTC,20485,0,14,74.4627,0,10000272,JP,US,MODELER,Hongkong
-757,2013-03-16,FP-non GTC,106246,0,5,64.1494,0,10000459,FR,FR,MODELER,Hongkong
-758,2012-09-06,FP-non GTC,80135,0,13,30.1506,0,10000269,CN,JP,MODELER,Hongkong
-759,2013-06-18,FP-non GTC,80135,0,-99,75.5826,0,10000706,JP,CN,ANALYST,Beijing
-760,2013-03-16,FP-GTC,156614,0,15,36.7157,0,10000761,US,UK,MODELER,Hongkong
-761,2012-11-07,Others,1349,0,15,96.7575,0,10000612,UK,CN,ANALYST,Beijing
-762,2012-11-25,ABIN,95173,0,5,61.1186,0,10000364,CN,UK,MODELER,Hongkong
-763,2013-08-10,ABIN,31673,0,11,63.4095,0,10000421,CN,JP,ADMIN,Shanghai
-764,2012-02-17,ABIN,88750,0,14,87.8624,0,10000306,DE,CN,MODELER,Hongkong
-765,2013-03-29,ABIN,139973,0,16,9.1685,0,10000089,UK,US,MODELER,Hongkong
-766,2012-12-27,ABIN,121153,0,12,88.7482,0,10000726,DE,FR,MODELER,Hongkong
-767,2013-10-13,FP-GTC,67698,0,16,20.8225,0,10000504,US,UK,MODELER,Hongkong
-768,2012-09-26,ABIN,2023,0,11,83.6499,0,10000641,CN,UK,MODELER,Hongkong
-769,2012-08-01,Others,139973,0,-99,1.0542,0,10000202,CN,GB,MODELER,Hongkong
-770,2012-03-02,ABIN,113802,15,5,23.4439,0,10000068,FR,UK,MODELER,Hongkong
-771,2012-10-25,Others,10866,0,11,60.7453,0,10000770,FR,US,ADMIN,Shanghai
-772,2013-01-07,FP-non GTC,20485,0,16,16.4177,0,10000698,US,JP,ADMIN,Shanghai
-773,2012-03-18,FP-non GTC,46575,0,15,55.2768,0,10000962,JP,FR,MODELER,Hongkong
-774,2012-07-09,FP-GTC,106246,0,14,99.6346,0,10000563,DE,DE,MODELER,Hongkong
-775,2012-09-26,FP-GTC,314,0,13,45.8302,0,10000769,DE,UK,ANALYST,Beijing
-776,2012-01-14,FP-non GTC,15868,0,16,47.5496,0,10000869,CN,CN,ADMIN,Shanghai
-777,2012-01-01,ABIN,63861,0,13,76.1393,0,10000028,JP,UK,ADMIN,Shanghai
-778,2012-08-15,FP-GTC,26249,0,5,40.7448,0,10000419,JP,UK,ADMIN,Shanghai
-779,2012-09-24,FP-non GTC,43398,0,12,54.0864,0,10000688,CN,US,MODELER,Hongkong
-780,2013-10-13,Others,533,0,-99,58.0312,0,10000658,UK,JP,MODELER,Hongkong
-781,2012-07-13,FP-GTC,148324,15,11,71.6067,0,10000006,UK,FR,ADMIN,Shanghai
-782,2012-01-21,FP-GTC,32996,15,11,86.8077,0,10000928,DE,FR,ADMIN,Shanghai
-783,2012-04-04,Others,63861,3,16,85.2671,0,10000648,JP,UK,MODELER,Hongkong
-784,2012-10-20,FP-non GTC,31519,0,13,99.193,0,10000469,UK,CN,ANALYST,Beijing
-785,2013-12-28,FP-GTC,1504,0,5,14.9821,0,10000342,DE,FR,ANALYST,Beijing
-786,2013-06-19,Others,25147,0,5,77.4671,0,10000212,FR,CN,ANALYST,Beijing
-787,2013-05-04,Others,13836,0,11,13.4655,0,10000331,CN,DE,MODELER,Hongkong
-788,2012-06-28,ABIN,145970,0,15,74.3304,0,10000900,US,CN,ANALYST,Beijing
-789,2012-12-24,FP-non GTC,156614,0,12,79.4508,0,10000743,JP,DE,MODELER,Hongkong
-790,2013-07-17,FP-GTC,139973,0,13,30.0555,0,10000949,CN,DE,ADMIN,Shanghai
-791,2013-11-23,Others,31519,3,11,9.0845,0,10000883,US,UK,MODELER,Hongkong
-792,2013-01-15,Others,164261,0,12,31.5563,0,10000029,FR,US,ANALYST,Beijing
-793,2012-04-15,FP-non GTC,20213,0,-99,55.6917,0,10000513,US,DE,ANALYST,Beijing
-794,2013-01-20,FP-non GTC,24760,0,15,71.7279,0,10000274,UK,CN,ANALYST,Beijing
-795,2012-11-13,FP-GTC,95173,0,5,95.7043,0,10000418,UK,CN,ANALYST,Beijing
-796,2012-09-03,ABIN,57990,0,12,1.5845,0,10000703,CN,JP,ANALYST,Beijing
-797,2012-01-02,FP-non GTC,57990,3,16,12.3706,0,10000090,UK,UK,ADMIN,Shanghai
-798,2013-12-06,Auction,67703,3,5,70.4134,0,10000753,CN,CN,MODELER,Hongkong
-799,2012-06-09,Others,50677,0,12,88.7297,0,10000990,FR,UK,MODELER,Hongkong
-800,2012-05-17,ABIN,31519,0,14,37.707,0,10000326,JP,CN,ADMIN,Shanghai
-801,2012-03-03,Others,156356,0,16,50.7844,0,10000693,CN,DE,MODELER,Hongkong
-802,2012-05-21,Auction,44079,0,5,86.759,0,10000795,UK,US,ADMIN,Shanghai
-803,2013-10-15,Auction,26262,0,15,56.3327,0,10000077,US,JP,MODELER,Hongkong
-804,2013-12-27,ABIN,20213,0,13,96.1815,0,10000195,JP,FR,ADMIN,Shanghai
-805,2012-02-15,ABIN,314,0,5,6.5869,0,10000753,UK,CN,ANALYST,Beijing
-806,2012-08-17,FP-non GTC,6762,0,15,41.0782,0,10000797,US,FR,ADMIN,Shanghai
-807,2013-04-29,Auction,145970,0,15,11.0876,0,10000208,UK,DE,ADMIN,Shanghai
-808,2013-10-27,Auction,80053,0,14,81.7435,0,10000310,UK,CN,ADMIN,Shanghai
-809,2012-12-21,FP-GTC,63861,0,13,57.9491,0,10000971,CN,JP,MODELER,Hongkong
-810,2013-07-29,ABIN,45333,0,12,33.5405,0,10000475,FR,UK,ADMIN,Shanghai
-811,2012-04-18,Auction,64076,0,13,83.3621,0,10000697,UK,US,ANALYST,Beijing
-812,2012-12-14,FP-non GTC,63889,0,16,0.964,0,10000607,JP,CN,ANALYST,Beijing
-813,2013-06-07,Auction,145970,0,13,6.1316,0,10000450,DE,JP,ANALYST,Beijing
-814,2012-11-27,Others,164,0,15,53.7718,0,10000305,CN,CN,ADMIN,Shanghai
-815,2012-08-03,ABIN,61323,0,15,62.2945,0,10000214,JP,CN,ANALYST,Beijing
-816,2012-07-07,ABIN,75665,0,15,77.0551,0,10000094,CN,GB,ANALYST,Beijing
-817,2012-08-06,FP-non GTC,99985,0,13,39.1253,0,10000609,US,DE,MODELER,Hongkong
-818,2012-10-01,Auction,759,0,15,69.179,0,10000199,US,DE,MODELER,Hongkong
-819,2012-03-22,Auction,20865,0,5,40.78,0,10000392,FR,US,MODELER,Hongkong
-820,2012-02-06,FP-non GTC,132939,0,11,66.8705,0,10000207,US,CN,ADMIN,Shanghai
-821,2012-10-17,FP-non GTC,63861,3,14,35.8018,0,10000096,DE,CN,ANALYST,Beijing
-822,2013-02-21,Others,41940,0,5,87.4721,0,10000858,CN,CN,MODELER,Hongkong
-823,2012-11-16,FP-GTC,57784,0,15,96.2799,0,10000537,FR,CN,ADMIN,Shanghai
-824,2012-01-10,Others,158666,15,11,7.4077,0,10000190,FR,JP,ADMIN,Shanghai
-825,2013-10-29,FP-non GTC,65,0,-99,21.3439,0,10000182,UK,FR,ADMIN,Shanghai
-826,2013-07-30,Auction,103178,15,-99,94.8679,0,10000943,DE,US,ADMIN,Shanghai
-827,2012-08-05,FP-non GTC,175750,0,14,10.098,0,10000212,CN,JP,ADMIN,Shanghai
-828,2012-07-29,ABIN,24760,0,13,44.527,0,10000168,JP,FR,ANALYST,Beijing
-829,2013-04-27,FP-GTC,57990,3,-99,75.6664,0,10000878,US,JP,ANALYST,Beijing
-830,2012-12-06,FP-GTC,37831,0,11,57.1191,0,10000090,CN,US,ADMIN,Shanghai
-831,2013-08-29,FP-GTC,164,0,16,75.2345,0,10000689,US,FR,ANALYST,Beijing
-832,2012-08-30,FP-GTC,63861,0,12,21.2099,0,10000643,US,UK,MODELER,Hongkong
-833,2013-05-30,ABIN,57013,0,15,28.841,0,10000290,US,JP,ADMIN,Shanghai
-834,2013-10-17,ABIN,50677,0,-99,16.3836,0,10000705,CN,DE,ANALYST,Beijing
-835,2012-04-25,FP-GTC,67698,0,11,52.3502,0,10000257,CN,FR,ANALYST,Beijing
-836,2013-08-18,ABIN,759,0,16,23.8258,0,10000911,US,UK,ADMIN,Shanghai
-837,2013-02-24,Others,13836,0,-99,40.6826,0,10000399,CN,CN,ADMIN,Shanghai
-838,2012-07-22,FP-non GTC,158798,0,15,94.1886,0,10000705,FR,CN,ANALYST,Beijing
-839,2012-08-05,FP-GTC,100847,0,-99,74.15,0,10000797,CN,GB,ADMIN,Shanghai
-840,2012-12-30,Auction,31519,0,11,69.1499,0,10000576,UK,US,ANALYST,Beijing
-841,2013-05-18,Auction,63864,3,14,44.5408,0,10000232,DE,CN,ANALYST,Beijing
-842,2012-11-01,ABIN,53064,0,15,19.9817,0,10000065,US,CN,ADMIN,Shanghai
-843,2013-06-25,FP-GTC,156614,0,16,91.4652,0,10000326,UK,DE,ANALYST,Beijing
-844,2013-07-11,FP-non GTC,10058,3,5,92.6671,0,10000787,FR,CN,ADMIN,Shanghai
-845,2013-01-19,FP-non GTC,13987,0,12,49.0313,0,10000556,FR,CN,ADMIN,Shanghai
-846,2013-06-17,Auction,94847,0,13,95.5377,0,10000047,JP,CN,ANALYST,Beijing
-847,2012-12-23,Others,64076,0,-99,50.1173,0,10000063,JP,CN,MODELER,Hongkong
-848,2013-01-07,FP-non GTC,67698,0,5,6.5979,0,10000412,UK,CN,ADMIN,Shanghai
-849,2012-01-20,ABIN,175750,3,16,79.6931,0,10000270,FR,JP,ADMIN,Shanghai
-850,2013-01-14,Others,95672,3,12,31.7315,0,10000044,CN,US,ANALYST,Beijing
-851,2013-02-22,Others,26249,0,16,43.7969,0,10000634,CN,DE,ADMIN,Shanghai
-852,2013-04-13,FP-non GTC,156614,0,16,79.3213,0,10000321,CN,JP,ADMIN,Shanghai
-853,2013-09-18,FP-non GTC,43398,0,5,17.3702,0,10000621,US,CN,MODELER,Hongkong
-854,2013-09-05,ABIN,67703,3,11,55.6439,0,10000784,DE,UK,ANALYST,Beijing
-855,2013-05-25,FP-GTC,43398,0,15,73.0548,0,10000921,CN,DE,MODELER,Hongkong
-856,2012-10-06,Others,100847,0,16,77.0258,0,10000606,US,US,MODELER,Hongkong
-857,2013-11-08,Others,164261,0,11,16.6959,0,10000136,UK,UK,ANALYST,Beijing
-858,2012-04-13,FP-non GTC,3838,0,5,68.4347,0,10000451,JP,CN,ADMIN,Shanghai
-859,2012-07-03,FP-GTC,63861,0,-99,89.5962,0,10000893,DE,JP,ADMIN,Shanghai
-860,2012-02-21,Auction,40059,3,-99,84.7455,0,10000613,DE,CN,MODELER,Hongkong
-861,2012-02-25,FP-non GTC,759,0,13,23.9258,0,10000826,JP,DE,MODELER,Hongkong
-862,2013-10-30,FP-non GTC,57990,0,16,88.4859,0,10000782,FR,FR,MODELER,Hongkong
-863,2013-10-12,FP-non GTC,57784,0,14,85.8986,0,10000220,CN,CN,MODELER,Hongkong
-864,2012-08-15,FP-GTC,223,0,12,32.1974,0,10000316,CN,JP,ANALYST,Beijing
-865,2012-10-18,FP-GTC,15115,0,16,52.5243,0,10000353,CN,US,ADMIN,Shanghai
-866,2013-11-09,FP-GTC,20886,0,13,94.0128,0,10000690,CN,US,MODELER,Hongkong
-867,2012-09-27,ABIN,31673,0,12,24.1332,0,10000404,US,CN,ANALYST,Beijing
-868,2012-05-08,FP-GTC,95672,0,16,88.8853,0,10000984,DE,JP,ADMIN,Shanghai
-869,2013-01-14,Others,95173,0,12,86.3987,0,10000501,CN,UK,ADMIN,Shanghai
-870,2012-12-02,Auction,152801,0,-99,71.8799,0,10000021,FR,DE,ADMIN,Shanghai
-871,2012-03-13,ABIN,31519,3,15,96.8998,0,10000188,FR,UK,ADMIN,Shanghai
-872,2013-06-15,Auction,121153,0,14,66.2729,0,10000101,DE,DE,ANALYST,Beijing
-873,2012-04-11,Others,63861,0,15,88.0097,0,10000536,DE,US,ADMIN,Shanghai
-874,2013-01-25,FP-non GTC,1357,0,16,49.8991,0,10000637,JP,US,ANALYST,Beijing
-875,2013-10-17,FP-GTC,106246,0,15,92.2507,0,10000046,JP,UK,MODELER,Hongkong
-876,2012-10-04,Others,4943,0,13,88.8498,0,10000215,US,UK,ADMIN,Shanghai
-877,2013-05-14,Auction,314,0,13,77.3441,0,10000900,FR,JP,ANALYST,Beijing
-878,2013-03-17,FP-non GTC,759,0,16,17.9719,0,10000759,UK,CN,MODELER,Hongkong
-879,2013-01-20,FP-non GTC,51582,0,16,3.6644,0,10000961,FR,CN,ADMIN,Shanghai
-880,2013-03-20,FP-non GTC,57990,0,12,9.368,0,10000846,CN,US,ANALYST,Beijing
-881,2012-06-06,FP-non GTC,165888,0,16,9.4584,0,10000312,UK,FR,ADMIN,Shanghai
-882,2012-12-13,Auction,43479,0,16,84.0632,0,10000349,FR,CN,MODELER,Hongkong
-883,2012-05-13,FP-non GTC,139973,3,5,37.351,0,10000790,CN,US,MODELER,Hongkong
-884,2013-03-27,Auction,48027,0,5,75.7167,0,10000424,FR,DE,MODELER,Hongkong
-885,2013-08-08,FP-GTC,37831,0,-99,27.78,0,10000637,CN,UK,ANALYST,Beijing
-886,2012-04-06,ABIN,314,0,12,18.3274,0,10000749,DE,CN,MODELER,Hongkong
-887,2013-10-11,FP-non GTC,57013,0,14,39.1868,0,10000599,JP,FR,MODELER,Hongkong
-888,2012-01-18,FP-GTC,41940,0,12,94.5165,0,10000102,FR,FR,MODELER,Hongkong
-889,2013-05-20,ABIN,50677,0,16,25.6136,0,10000423,CN,UK,ANALYST,Beijing
-890,2012-06-24,FP-GTC,963,0,13,20.9442,0,10000069,UK,CN,ADMIN,Shanghai
-891,2013-04-01,ABIN,75665,0,12,65.3307,0,10000543,UK,CN,MODELER,Hongkong
-892,2012-02-05,ABIN,50508,0,-99,69.4516,0,10000623,FR,UK,ANALYST,Beijing
-893,2012-06-24,ABIN,314,0,5,13.8612,0,10000347,CN,CN,ANALYST,Beijing
-894,2013-12-07,Others,1349,0,16,10.8619,0,10000784,UK,CN,MODELER,Hongkong
-895,2013-03-01,ABIN,73506,0,11,7.7818,0,10000882,UK,CN,ADMIN,Shanghai
-896,2012-08-24,Auction,45333,0,16,94.9583,0,10000495,UK,CN,ANALYST,Beijing
-897,2012-08-04,FP-non GTC,20886,0,15,69.73,0,10000428,CN,CN,MODELER,Hongkong
-898,2012-11-08,Auction,31519,0,15,41.4423,0,10000431,FR,CN,MODELER,Hongkong
-899,2013-08-07,FP-GTC,156356,0,13,44.6853,0,10000305,DE,JP,ANALYST,Beijing
-900,2013-09-25,ABIN,43398,0,16,46.5536,0,10000387,UK,CN,ANALYST,Beijing
-901,2012-04-26,FP-non GTC,95672,0,13,30.9202,0,10000591,CN,FR,MODELER,Hongkong
-902,2012-03-30,Others,159184,0,12,58.3993,0,10000381,CN,UK,ADMIN,Shanghai
-903,2012-04-25,Auction,73506,0,14,46.1501,0,10000010,FR,US,ANALYST,Beijing
-904,2012-09-13,FP-GTC,20485,0,5,2.4112,0,10000959,FR,FR,ADMIN,Shanghai
-905,2013-09-27,ABIN,80287,0,-99,64.4361,0,10000770,US,DE,ANALYST,Beijing
-906,2013-01-17,Others,36250,0,11,73.5773,0,10000948,JP,US,ADMIN,Shanghai
-907,2012-07-07,Auction,103324,15,16,4.8559,0,10000123,UK,CN,MODELER,Hongkong
-908,2013-08-15,ABIN,13836,0,16,25.8438,0,10000851,US,DE,ANALYST,Beijing
-909,2012-07-07,Others,12688,0,5,3.4934,0,10000096,DE,FR,MODELER,Hongkong
-910,2012-06-03,Auction,80135,0,12,17.8258,0,10000144,JP,CN,ANALYST,Beijing
-911,2013-12-24,FP-GTC,60340,0,11,22.6409,0,10000341,FR,DE,MODELER,Hongkong
-912,2012-10-06,ABIN,36250,0,5,7.4322,0,10000384,US,CN,ADMIN,Shanghai
-913,2013-01-03,Auction,175750,0,15,14.1976,0,10000366,DE,UK,ADMIN,Shanghai
-914,2013-04-15,Auction,88750,0,11,98.6057,0,10000584,US,JP,MODELER,Hongkong
-915,2013-06-03,FP-non GTC,1349,0,15,11.0419,0,10000866,CN,FR,ANALYST,Beijing
-916,2013-01-30,FP-GTC,2635,0,14,54.5175,0,10000959,UK,CN,ANALYST,Beijing
-917,2013-08-27,FP-non GTC,26249,0,13,50.0365,0,10000289,US,DE,MODELER,Hongkong
-918,2012-05-14,FP-non GTC,9426,3,5,60.0224,0,10000730,FR,US,ADMIN,Shanghai
-919,2012-10-16,FP-non GTC,16145,0,-99,53.0896,0,10000195,JP,JP,MODELER,Hongkong
-920,2012-09-24,FP-GTC,57013,0,14,81.7929,0,10000498,JP,US,MODELER,Hongkong
-921,2012-08-20,FP-non GTC,25147,0,-99,30.7605,0,10000242,CN,FR,ADMIN,Shanghai
-922,2012-02-19,Others,94847,0,13,20.6395,0,10000707,JP,US,ANALYST,Beijing
-923,2013-11-14,ABIN,50508,0,15,91.9217,0,10000208,CN,FR,ADMIN,Shanghai
-924,2012-07-10,Others,95672,0,13,21.1886,0,10000762,UK,DE,ADMIN,Shanghai
-925,2012-03-14,ABIN,44079,0,13,89.6838,0,10000227,DE,DE,ADMIN,Shanghai
-926,2012-01-17,FP-non GTC,314,0,14,44.3066,0,10000251,CN,CN,ADMIN,Shanghai
-927,2012-02-27,FP-non GTC,15868,0,14,80.8427,0,10000741,FR,DE,ANALYST,Beijing
-928,2012-10-23,Others,1161,3,5,54.0104,0,10000289,DE,DE,ADMIN,Shanghai
-929,2013-06-12,Others,53064,0,12,87.1336,0,10000871,CN,DE,ANALYST,Beijing
-930,2013-03-06,Auction,34273,100,15,6.7044,0,10000269,FR,DE,MODELER,Hongkong
-931,2012-06-16,Others,533,0,14,43.7104,0,10000172,CN,UK,ADMIN,Shanghai
-932,2012-12-19,FP-GTC,15868,0,15,94.1727,0,10000366,JP,DE,ANALYST,Beijing
-933,2012-07-21,ABIN,121153,0,12,42.3437,0,10000688,CN,US,ANALYST,Beijing
-934,2013-08-19,FP-non GTC,65,0,11,3.0275,0,10000055,US,DE,MODELER,Hongkong
-935,2012-08-08,Auction,15687,0,14,72.9505,0,10000725,US,FR,ADMIN,Shanghai
-936,2013-12-13,FP-GTC,4943,0,-99,37.7781,0,10000932,CN,DE,MODELER,Hongkong
-937,2013-05-18,FP-GTC,3838,0,14,45.1642,0,10000370,CN,US,ANALYST,Beijing
-938,2012-01-31,Auction,25147,0,14,78.658,0,10000721,US,JP,MODELER,Hongkong
-939,2013-12-05,FP-GTC,62179,0,12,76.4692,0,10000423,DE,UK,MODELER,Hongkong
-940,2012-03-07,Auction,175750,0,15,19.5357,0,10000053,FR,FR,MODELER,Hongkong
-941,2013-09-30,Others,100847,0,12,38.2299,0,10000772,US,FR,ADMIN,Shanghai
-942,2013-01-07,Others,15115,0,12,82.5099,0,10000864,FR,FR,ADMIN,Shanghai
-943,2013-11-04,FP-non GTC,57013,0,-99,61.3731,0,10000335,CN,DE,MODELER,Hongkong
-944,2012-08-11,FP-non GTC,31673,0,12,12.7001,0,10000303,FR,DE,MODELER,Hongkong
-945,2012-12-30,ABIN,156356,0,14,99.3562,0,10000427,UK,JP,ANALYST,Beijing
-946,2013-05-19,Others,113593,15,11,31.1634,0,10000361,FR,FR,ADMIN,Shanghai
-947,2013-06-03,ABIN,314,0,13,32.6968,0,10000373,US,CN,ANALYST,Beijing
-948,2013-06-29,Auction,15808,15,12,18.3967,0,10000484,US,CN,ADMIN,Shanghai
-949,2012-06-10,FP-GTC,61323,0,13,10.8143,0,10000403,UK,UK,ADMIN,Shanghai
-950,2012-01-29,FP-GTC,1349,0,15,0.4358,0,10000639,US,CN,ANALYST,Beijing
-951,2013-05-30,FP-non GTC,759,0,14,19.592,0,10000116,JP,DE,MODELER,Hongkong
-952,2013-07-28,ABIN,15868,0,14,59.7123,0,10000668,FR,FR,ADMIN,Shanghai
-953,2013-03-23,Auction,165888,0,12,67.1201,0,10000063,DE,US,MODELER,Hongkong
-954,2012-05-29,FP-GTC,118687,3,-99,98.203,0,10000886,CN,DE,ADMIN,Shanghai
-955,2013-09-15,FP-non GTC,156614,0,5,43.1795,0,10000302,FR,UK,ANALYST,Beijing
-956,2012-05-21,FP-GTC,223,0,-99,66.285,0,10000118,UK,FR,ANALYST,Beijing
-957,2012-04-03,FP-non GTC,26262,0,12,17.5241,0,10000440,CN,US,ADMIN,Shanghai
-958,2012-05-22,ABIN,44079,0,16,25.2405,0,10000183,FR,UK,ANALYST,Beijing
-959,2014-01-01,Others,50508,0,16,92.5843,0,10000911,CN,FR,MODELER,Hongkong
-960,2012-04-08,Others,16145,3,12,90.2242,0,10000825,JP,JP,ADMIN,Shanghai
-961,2013-11-27,ABIN,95672,0,16,90.938,0,10000829,CN,CN,ANALYST,Beijing
-962,2013-03-22,Auction,41940,0,15,72.0648,0,10000795,DE,UK,ANALYST,Beijing
-963,2012-04-06,Others,64076,0,14,86.9185,0,10000356,JP,DE,ADMIN,Shanghai
-964,2013-05-02,Others,156356,0,11,41.6871,0,10000122,UK,JP,ANALYST,Beijing
-965,2012-04-02,Others,132939,0,12,75.8372,0,10000395,JP,UK,ADMIN,Shanghai
-966,2012-06-29,ABIN,26262,0,13,83.6043,0,10000120,CN,UK,ADMIN,Shanghai
-967,2012-07-06,FP-GTC,11554,0,16,38.4329,0,10000352,FR,UK,MODELER,Hongkong
-968,2012-06-10,ABIN,156614,0,-99,94.1812,0,10000280,US,UK,ANALYST,Beijing
-969,2012-04-05,Auction,164261,0,12,0.9691,0,10000480,UK,CN,ANALYST,Beijing
-970,2012-11-02,Auction,31519,0,11,98.0505,0,10000893,JP,DE,MODELER,Hongkong
-971,2012-05-20,Auction,50508,0,15,61.9439,0,10000363,CN,US,ANALYST,Beijing
-972,2012-03-31,Auction,80135,0,16,11.1543,0,10000692,FR,CN,MODELER,Hongkong
-973,2012-07-01,FP-non GTC,63889,0,-99,86.6557,0,10000402,JP,DE,MODELER,Hongkong
-974,2013-01-04,ABIN,41940,0,-99,48.4505,0,10000296,DE,FR,ANALYST,Beijing
-975,2013-02-10,FP-non GTC,63861,0,13,60.3535,0,10000214,US,UK,ADMIN,Shanghai
-976,2012-03-25,Others,73506,0,11,74.2113,0,10000801,JP,JP,ANALYST,Beijing
-977,2013-07-23,FP-non GTC,53064,0,16,69.2368,0,10000542,US,US,MODELER,Hongkong
-978,2012-04-09,Auction,11848,0,15,76.172,0,10000555,CN,FR,ADMIN,Shanghai
-979,2012-08-01,Others,12688,0,5,85.4589,0,10000130,DE,CN,ADMIN,Shanghai
-980,2012-07-15,FP-non GTC,31519,3,-99,17.8485,0,10000045,CN,GB,MODELER,Hongkong
-981,2013-10-06,ABIN,1349,0,11,52.278,0,10000622,US,CN,MODELER,Hongkong
-982,2012-10-26,Auction,25147,0,13,60.9912,0,10000379,CN,FR,ADMIN,Shanghai
-983,2013-01-04,FP-non GTC,37831,0,15,61.0134,0,10000785,CN,JP,ADMIN,Shanghai
-984,2012-03-24,Auction,139973,0,-99,8.1533,0,10000487,CN,JP,ADMIN,Shanghai
-985,2012-12-30,Auction,6762,0,-99,77.8179,0,10000645,DE,CN,ADMIN,Shanghai
-986,2012-10-03,ABIN,103178,15,12,15.2201,0,10000824,DE,JP,MODELER,Hongkong
-987,2012-04-18,FP-non GTC,50508,0,14,11.9715,0,10000012,FR,CN,MODELER,Hongkong
-988,2012-09-30,FP-non GTC,31673,0,12,90.0864,0,10000281,CN,US,MODELER,Hongkong
-989,2013-03-24,ABIN,4943,0,12,11.0294,0,10000785,CN,US,ADMIN,Shanghai
-990,2012-05-02,Others,87118,0,14,52.6549,0,10000924,CN,DE,ADMIN,Shanghai
-991,2013-05-03,FP-GTC,24760,0,5,8.9077,0,10000758,CN,CN,MODELER,Hongkong
-992,2012-08-05,FP-non GTC,38238,0,11,0.8463,0,10000066,FR,CN,MODELER,Hongkong
-993,2012-08-15,Auction,67698,0,11,23.6278,0,10000085,CN,CN,ADMIN,Shanghai
-994,2013-04-04,Auction,164261,0,13,52.3718,0,10000965,JP,DE,ADMIN,Shanghai
-995,2013-04-18,ABIN,25147,0,14,71.7687,0,10000157,FR,CN,ANALYST,Beijing
-996,2013-11-09,ABIN,43479,0,12,39.9269,0,10000024,US,DE,ANALYST,Beijing
-997,2013-05-02,FP-non GTC,175750,0,12,17.6149,0,10000862,JP,US,ANALYST,Beijing
-998,2012-09-01,ABIN,50508,0,14,99.1518,0,10000746,DE,UK,ANALYST,Beijing
-999,2013-02-23,FP-non GTC,139255,15,-99,74.6182,0,10000447,FR,JP,ANALYST,Beijing
-1000,2012-01-19,FP-non GTC,6762,0,13,0.1297,0,10000189,UK,FR,MODELER,Hongkong
-1001,2012-11-10,Auction,113802,15,-99,11.9583,0,10000596,JP,UK,ANALYST,Beijing
-1002,2012-12-11,FP-non GTC,95173,0,5,67.9416,0,10000582,CN,CN,ADMIN,Shanghai
-1003,2012-01-05,ABIN,2635,0,11,72.3723,0,10000375,UK,JP,ANALYST,Beijing
-1004,2012-05-04,FP-non GTC,43479,0,13,79.1307,0,10000944,FR,UK,ADMIN,Shanghai
-1005,2013-07-20,ABIN,63861,0,13,86.7155,0,10000274,JP,UK,MODELER,Hongkong
-1006,2013-08-25,FP-GTC,80053,0,5,11.7108,0,10000059,CN,JP,MODELER,Hongkong
-1007,2012-10-24,ABIN,38238,0,5,79.2964,0,10000388,FR,CN,MODELER,Hongkong
-1008,2012-01-10,ABIN,73506,0,5,3.8878,0,10000895,JP,DE,ANALYST,Beijing
-1009,2013-07-11,Auction,174106,3,12,46.7087,0,10000683,FR,CN,ANALYST,Beijing
-1010,2013-06-16,FP-GTC,61323,0,15,64.0224,0,10000121,US,DE,ADMIN,Shanghai
-1011,2013-06-23,ABIN,63889,0,5,59.2766,0,10000466,JP,US,MODELER,Hongkong
-1012,2013-01-25,Others,156356,0,14,79.9992,0,10000611,FR,DE,ANALYST,Beijing
-1013,2013-10-09,Auction,50508,0,16,70.9682,0,10000009,US,FR,MODELER,Hongkong
-1014,2013-05-19,Auction,25147,0,16,8.0973,0,10000731,FR,JP,ADMIN,Shanghai
-1015,2012-10-14,Auction,67698,0,5,49.0497,0,10000605,CN,FR,ADMIN,Shanghai
-1016,2012-06-08,Auction,65,0,14,76.5149,0,10000165,UK,US,MODELER,Hongkong
-1017,2012-12-21,FP-GTC,15868,0,5,45.9224,0,10000215,JP,US,ADMIN,Shanghai
-1018,2012-12-02,Auction,1357,0,15,8.9337,0,10000730,JP,DE,ANALYST,Beijing
-1019,2013-11-27,ABIN,24760,0,11,62.4675,0,10000932,FR,US,ADMIN,Shanghai
-1020,2013-11-24,Auction,95173,0,16,16.7927,0,10000795,CN,JP,ANALYST,Beijing
-1021,2013-11-30,ABIN,164,0,11,10.8897,0,10000914,US,DE,MODELER,Hongkong
-1022,2012-12-22,Auction,15868,0,15,7.4153,0,10000540,JP,CN,MODELER,Hongkong
-1023,2012-05-13,FP-non GTC,26249,0,5,58.8647,0,10000898,UK,JP,MODELER,Hongkong
-1024,2013-06-22,ABIN,95672,3,12,71.8628,0,10000026,UK,FR,ANALYST,Beijing
-1025,2013-05-22,ABIN,63861,0,11,71.8836,0,10000650,UK,CN,ADMIN,Shanghai
-1026,2012-11-17,FP-GTC,32876,0,12,35.6019,0,10000959,US,CN,ADMIN,Shanghai
-1027,2013-04-13,Auction,38238,0,5,21.7443,0,10000036,FR,CN,ANALYST,Beijing
-1028,2012-09-09,FP-non GTC,87118,0,14,46.2858,0,10000512,CN,DE,MODELER,Hongkong
-1029,2013-08-23,FP-non GTC,41940,0,11,98.4779,0,10000844,JP,US,MODELER,Hongkong
-1030,2012-09-02,FP-non GTC,43398,0,12,91.8543,0,10000917,JP,CN,ANALYST,Beijing
-1031,2012-11-23,Others,145970,0,11,43.8631,0,10000997,FR,US,ANALYST,Beijing
-1032,2012-01-22,FP-non GTC,164261,0,16,74.9772,0,10000091,DE,DE,MODELER,Hongkong
-1033,2012-05-02,ABIN,152801,0,13,15.2794,0,10000649,JP,UK,ADMIN,Shanghai
-1034,2013-08-14,Others,50677,0,13,26.5329,0,10000275,JP,DE,MODELER,Hongkong
-1035,2012-06-04,Others,156614,0,14,42.781,0,10000347,CN,JP,ANALYST,Beijing
-1036,2012-08-04,FP-non GTC,145970,0,15,92.6953,0,10000622,US,CN,ADMIN,Shanghai
-1037,2012-08-15,Auction,32876,0,16,35.2015,0,10000533,DE,DE,MODELER,Hongkong
-1038,2012-11-05,Auction,95672,0,11,0.5233,0,10000965,CN,US,MODELER,Hongkong
-1039,2013-10-12,ABIN,165888,0,11,27.3564,0,10000036,UK,DE,ADMIN,Shanghai
-1040,2012-10-22,ABIN,20485,0,14,28.6742,0,10000448,DE,UK,ANALYST,Beijing
-1041,2012-07-02,Auction,20213,0,15,81.3332,0,10000484,CN,UK,ADMIN,Shanghai
-1042,2013-01-13,Others,152801,0,11,3.3406,0,10000124,CN,DE,MODELER,Hongkong
-1043,2012-05-17,Others,132939,0,15,55.3503,0,10000086,US,DE,MODELER,Hongkong
-1044,2013-02-05,Others,48904,0,15,25.234,0,10000246,UK,JP,MODELER,Hongkong
-1045,2013-09-13,FP-GTC,32876,0,-99,10.7672,0,10000729,CN,FR,ANALYST,Beijing
-1046,2012-06-14,ABIN,20865,0,14,47.9841,0,10000406,CN,CN,ADMIN,Shanghai
-1047,2013-02-11,Auction,152801,0,5,86.4355,0,10000050,FR,US,ANALYST,Beijing
-1048,2012-12-13,Auction,50508,0,5,72.5114,0,10000715,JP,FR,ANALYST,Beijing
-1049,2013-11-30,Auction,3838,0,-99,6.8517,0,10000706,FR,JP,ADMIN,Shanghai
-1050,2013-03-15,FP-non GTC,20865,0,14,93.304,0,10000926,FR,US,MODELER,Hongkong
-1051,2012-08-12,FP-GTC,57784,0,-99,18.4045,0,10000231,UK,UK,ADMIN,Shanghai
-1052,2013-10-19,Others,1161,3,14,79.1353,0,10000561,US,JP,ADMIN,Shanghai
-1053,2013-02-17,Others,31519,0,-99,49.0409,0,10000904,FR,UK,ADMIN,Shanghai
-1054,2012-03-30,FP-non GTC,314,0,11,93.6413,0,10000041,CN,US,ANALYST,Beijing
-1055,2012-08-26,Others,175750,0,-99,46.6557,0,10000719,CN,UK,ANALYST,Beijing
-1056,2012-12-26,Auction,139973,0,15,6.9809,0,10000044,US,JP,MODELER,Hongkong
-1057,2013-01-30,Others,159184,0,13,45.4599,0,10000131,DE,CN,ANALYST,Beijing
-1058,2012-04-27,Auction,48904,0,5,16.3472,0,10000157,US,UK,MODELER,Hongkong
-1059,2012-11-10,ABIN,16145,0,13,0.4693,0,10000668,UK,CN,ANALYST,Beijing
-1060,2012-05-13,Auction,121153,0,5,24.642,0,10000314,US,DE,MODELER,Hongkong
-1061,2012-02-01,FP-GTC,6762,0,13,5.9417,0,10000851,JP,UK,MODELER,Hongkong
-1062,2012-07-16,Auction,95672,0,-99,11.9934,0,10000828,FR,US,MODELER,Hongkong
-1063,2012-09-21,Auction,106246,0,5,39.0057,0,10000826,FR,FR,ANALYST,Beijing
-1064,2012-10-20,FP-GTC,314,0,16,5.5339,0,10000011,UK,UK,ANALYST,Beijing
-1065,2013-04-29,FP-non GTC,100847,0,13,3.3872,0,10000840,US,CN,ADMIN,Shanghai
-1066,2013-02-16,FP-non GTC,43398,0,13,3.4171,0,10000106,JP,UK,ANALYST,Beijing
-1067,2013-09-11,FP-GTC,63889,0,12,15.4662,0,10000004,US,CN,MODELER,Hongkong
-1068,2012-12-18,Others,80287,0,15,71.309,0,10000722,CN,JP,MODELER,Hongkong
-1069,2013-09-14,FP-non GTC,20865,0,-99,98.0987,0,10000591,JP,JP,ADMIN,Shanghai
-1070,2013-01-27,FP-non GTC,57990,3,14,84.9483,0,10000304,CN,DE,ANALYST,Beijing
-1071,2013-02-07,Others,95672,0,14,9.7034,0,10000894,US,FR,ANALYST,Beijing
-1072,2013-03-24,FP-non GTC,38238,0,-99,98.5328,0,10000286,CN,CN,MODELER,Hongkong
-1073,2012-01-05,Others,164,0,15,21.3336,0,10000151,CN,JP,ADMIN,Shanghai
-1074,2012-11-14,Auction,57990,0,12,91.1773,0,10000087,CN,GB,ADMIN,Shanghai
-1075,2013-03-24,Auction,13987,0,13,5.9953,0,10000551,FR,CN,ADMIN,Shanghai
-1076,2013-12-26,FP-GTC,32876,0,11,86.1904,0,10000012,UK,DE,MODELER,Hongkong
-1077,2013-06-23,FP-non GTC,1504,0,12,9.4689,0,10000720,JP,CN,ANALYST,Beijing
-1078,2012-01-11,FP-GTC,57013,0,13,96.6705,0,10000790,JP,CN,ADMIN,Shanghai
-1079,2012-08-30,FP-non GTC,62179,0,5,22.4244,0,10000403,FR,CN,MODELER,Hongkong
-1080,2012-09-01,ABIN,61323,0,12,30.1513,0,10000283,FR,CN,ANALYST,Beijing
-1081,2013-04-20,Auction,16145,3,12,77.9327,0,10000043,CN,UK,MODELER,Hongkong
-1082,2013-10-03,Others,963,0,14,82.3371,0,10000002,DE,FR,MODELER,Hongkong
-1083,2012-10-23,ABIN,45333,0,16,86.7017,0,10000180,UK,DE,MODELER,Hongkong
-1084,2013-10-09,ABIN,63889,0,-99,33.9942,0,10000485,CN,US,ADMIN,Shanghai
-1085,2013-04-28,ABIN,73506,0,5,22.6563,0,10000070,DE,CN,ANALYST,Beijing
-1086,2013-07-07,ABIN,57990,0,11,11.7659,0,10000834,CN,DE,ADMIN,Shanghai
-1087,2012-01-04,FP-non GTC,165888,0,16,78.7873,0,10000743,CN,FR,ANALYST,Beijing
-1088,2012-05-05,ABIN,158666,15,16,55.1036,0,10000372,DE,US,ANALYST,Beijing
-1089,2012-08-04,ABIN,43479,0,13,61.3469,0,10000425,FR,UK,MODELER,Hongkong
-1090,2013-02-18,Others,161567,15,5,43.5109,0,10000315,JP,JP,MODELER,Hongkong
-1091,2013-08-09,FP-GTC,161567,15,14,0.9756,0,10000420,CN,FR,MODELER,Hongkong
-1092,2012-04-30,Others,65,0,12,31.5741,0,10000858,US,CN,ANALYST,Beijing
-1093,2013-10-16,FP-non GTC,16509,0,15,8.4323,0,10000376,US,UK,MODELER,Hongkong
-1094,2013-11-06,FP-GTC,15115,0,12,91.7204,0,10000495,UK,CN,ADMIN,Shanghai
-1095,2012-04-13,ABIN,37831,0,16,86.915,0,10000947,US,JP,MODELER,Hongkong
-1096,2013-04-21,FP-GTC,95672,0,-99,60.6048,0,10000052,JP,DE,ANALYST,Beijing
-1097,2013-12-04,FP-GTC,121153,0,13,9.3639,0,10000384,US,US,ADMIN,Shanghai
-1098,2012-08-03,Others,50508,0,13,74.2474,0,10000103,US,DE,ANALYST,Beijing
-1099,2013-12-29,FP-GTC,1357,0,5,46.9382,0,10000987,CN,JP,ANALYST,Beijing
-1100,2012-11-01,Others,46575,0,13,54.452,0,10000247,JP,US,ANALYST,Beijing
-1101,2012-06-30,FP-non GTC,26249,0,5,74.2755,0,10000641,FR,CN,MODELER,Hongkong
-1102,2012-06-20,Auction,26262,0,11,11.9173,0,10000418,CN,FR,MODELER,Hongkong
-1103,2012-07-24,Others,80287,0,14,49.7794,0,10000590,UK,CN,MODELER,Hongkong
-1104,2012-08-11,FP-GTC,26262,0,11,16.291,0,10000698,UK,JP,MODELER,Hongkong
-1105,2013-06-27,ABIN,63861,0,-99,5.4086,0,10000810,UK,CN,ANALYST,Beijing
-1106,2012-04-28,FP-GTC,15115,0,15,91.7724,0,10000414,CN,CN,MODELER,Hongkong
-1107,2013-04-19,ABIN,73506,0,13,14.8212,0,10000728,CN,UK,ADMIN,Shanghai
-1108,2012-01-25,Others,32876,0,13,23.4295,0,10000673,CN,UK,MODELER,Hongkong
-1109,2013-11-06,ABIN,139973,3,12,41.9681,0,10000902,DE,FR,MODELER,Hongkong
-1110,2013-03-09,Auction,61323,0,15,53.8594,0,10000323,US,UK,ANALYST,Beijing
-1111,2012-03-01,ABIN,60340,0,11,78.1279,0,10000947,JP,DE,ANALYST,Beijing
-1112,2012-07-28,FP-non GTC,279,15,16,80.8961,0,10000691,FR,JP,MODELER,Hongkong
-1113,2013-02-04,Others,24541,0,11,40.4673,0,10000727,JP,DE,MODELER,Hongkong
-1114,2012-02-24,Auction,152801,0,13,9.8921,0,10000661,US,CN,ADMIN,Shanghai
-1115,2013-08-13,ABIN,156614,0,15,61.6452,0,10000734,DE,FR,ANALYST,Beijing
-1116,2012-04-02,ABIN,63861,3,11,18.4644,0,10000446,CN,CN,ANALYST,Beijing
-1117,2013-09-28,Others,95173,0,-99,51.2096,0,10000049,CN,FR,ANALYST,Beijing
-1118,2013-08-13,FP-non GTC,64076,0,12,85.3952,0,10000581,UK,US,ANALYST,Beijing
-1119,2012-09-09,ABIN,20485,0,15,0.5979,0,10000081,CN,GB,ADMIN,Shanghai
-1120,2012-04-26,Auction,145970,0,11,40.5874,0,10000401,US,JP,ANALYST,Beijing
-1121,2013-05-10,Others,26262,0,11,83.8561,0,10000545,FR,CN,MODELER,Hongkong
-1122,2013-05-23,FP-GTC,80287,0,11,27.6123,0,10000529,DE,FR,ADMIN,Shanghai
-1123,2012-01-08,Others,175750,0,14,45.4268,0,10000408,DE,CN,ANALYST,Beijing
-1124,2012-07-20,ABIN,38238,0,16,44.1376,0,10000796,CN,US,ADMIN,Shanghai
-1125,2012-09-15,Auction,65,0,-99,97.3146,0,10000792,CN,FR,MODELER,Hongkong
-1126,2012-10-17,ABIN,65,0,12,68.5363,0,10000569,FR,JP,MODELER,Hongkong
-1127,2013-08-03,FP-GTC,40059,3,15,87.3892,0,10000317,UK,UK,ADMIN,Shanghai
-1128,2013-08-28,FP-non GTC,80287,0,14,61.9829,0,10000015,UK,US,ADMIN,Shanghai
-1129,2012-07-21,Auction,25147,0,11,24.7662,0,10000333,FR,DE,MODELER,Hongkong
-1130,2013-10-17,Auction,132939,0,11,11.6038,0,10000926,US,DE,MODELER,Hongkong
-1131,2012-09-04,Auction,44079,0,15,75.1095,0,10000435,FR,UK,ANALYST,Beijing
-1132,2012-11-01,Auction,67698,0,5,8.8273,0,10000834,JP,DE,ADMIN,Shanghai
-1133,2013-12-02,Auction,24541,0,15,95.2195,0,10000259,DE,FR,ANALYST,Beijing
-1134,2012-02-06,Auction,533,0,12,66.4962,0,10000007,FR,US,MODELER,Hongkong
-1135,2013-12-03,FP-non GTC,99985,0,12,0.0537,0,10000244,DE,CN,ANALYST,Beijing
-1136,2013-05-11,FP-non GTC,60606,3,12,76.6828,0,10000002,CN,FR,ANALYST,Beijing
-1137,2012-09-18,Auction,156614,0,16,72.1912,0,10000174,DE,FR,ANALYST,Beijing
-1138,2012-05-19,Auction,60340,0,16,42.9975,0,10000813,FR,CN,MODELER,Hongkong
-1139,2012-04-26,FP-GTC,165888,0,14,88.3875,0,10000919,DE,CN,ADMIN,Shanghai
-1140,2013-08-30,Others,963,0,15,6.6424,0,10000994,CN,UK,ADMIN,Shanghai
-1141,2013-08-28,ABIN,10866,0,12,79.7295,0,10000832,CN,DE,ADMIN,Shanghai
-1142,2012-01-16,ABIN,145970,0,11,72.394,0,10000008,CN,FR,ANALYST,Beijing
-1143,2013-12-23,Others,65,0,15,94.0397,0,10000812,FR,CN,ADMIN,Shanghai
-1144,2013-09-06,FP-GTC,25147,0,11,72.9395,0,10000956,CN,JP,ANALYST,Beijing
-1145,2012-05-07,FP-GTC,16509,0,11,9.829,0,10000940,CN,UK,MODELER,Hongkong
-1146,2013-09-07,Others,139973,0,11,22.9964,0,10000563,CN,UK,ANALYST,Beijing
-1147,2012-11-17,Others,65,0,-99,62.7316,0,10000227,US,CN,MODELER,Hongkong
-1148,2012-05-14,ABIN,57013,0,13,19.0969,0,10000844,CN,CN,ANALYST,Beijing
-1149,2012-06-01,FP-non GTC,30059,3,16,26.9105,0,10000516,US,FR,ADMIN,Shanghai
-1150,2012-11-24,Auction,75665,0,15,26.5788,0,10000717,DE,UK,ADMIN,Shanghai
-1151,2013-10-23,FP-non GTC,75708,3,12,10.5298,0,10000238,JP,CN,ANALYST,Beijing
-1152,2012-10-19,Auction,64076,0,11,49.0746,0,10000389,UK,CN,ANALYST,Beijing
-1153,2012-02-17,ABIN,65,0,12,58.3004,0,10000219,CN,CN,MODELER,Hongkong
-1154,2012-01-08,FP-GTC,15115,0,16,3.238,0,10000435,UK,FR,ANALYST,Beijing
-1155,2012-02-04,Others,164261,0,14,45.9304,0,10000748,FR,CN,ANALYST,Beijing
-1156,2012-08-25,FP-GTC,156356,0,5,70.3654,0,10000864,CN,US,ADMIN,Shanghai
-1157,2013-04-10,FP-GTC,38238,0,5,72.9872,0,10000638,DE,UK,ADMIN,Shanghai
-1158,2013-05-16,FP-GTC,175750,0,5,95.3008,0,10000366,CN,JP,ADMIN,Shanghai
-1159,2012-06-27,FP-GTC,100847,0,14,64.8447,0,10000686,CN,JP,MODELER,Hongkong
-1160,2013-03-06,Auction,139255,15,12,20.7811,0,10000261,CN,CN,ANALYST,Beijing
-1161,2013-08-04,Auction,67698,0,14,20.1344,0,10000142,CN,DE,MODELER,Hongkong
-1162,2013-08-15,Auction,20865,0,12,64.6842,0,10000235,DE,CN,ANALYST,Beijing
-1163,2012-04-17,Auction,24760,0,-99,88.4159,0,10000039,CN,CN,ADMIN,Shanghai
-1164,2013-03-14,Others,75665,0,13,36.977,0,10000480,CN,JP,ANALYST,Beijing
-1165,2012-12-09,Others,139973,0,15,85.1056,0,10000801,US,CN,MODELER,Hongkong
-1166,2012-07-02,Others,36250,0,11,71.5548,0,10000926,US,US,MODELER,Hongkong
-1167,2013-08-04,FP-non GTC,31519,0,16,23.9847,0,10000680,JP,JP,ADMIN,Shanghai
-1168,2013-02-15,FP-non GTC,80053,0,15,68.4438,0,10000487,JP,CN,MODELER,Hongkong
-1169,2012-07-12,FP-non GTC,64076,0,5,54.574,0,10000628,CN,UK,ANALYST,Beijing
-1170,2012-11-07,Others,164,0,12,45.1637,0,10000846,FR,FR,ADMIN,Shanghai
-1171,2013-09-14,FP-GTC,106340,15,15,38.1572,0,10000201,US,CN,ANALYST,Beijing
-1172,2013-12-19,Auction,759,0,-99,53.0086,0,10000250,CN,US,ANALYST,Beijing
-1173,2012-05-11,Others,16509,0,-99,68.3043,0,10000809,FR,CN,ADMIN,Shanghai
-1174,2013-07-20,Others,45333,0,15,58.5554,0,10000981,CN,JP,MODELER,Hongkong
-1175,2013-03-24,FP-GTC,20485,0,5,62.1012,0,10000877,FR,CN,ADMIN,Shanghai
-1176,2013-08-09,ABIN,63861,0,5,44.1975,0,10000554,US,US,MODELER,Hongkong
-1177,2013-12-04,FP-GTC,53064,0,14,29.8358,0,10000491,US,CN,MODELER,Hongkong
-1178,2013-07-06,ABIN,139973,0,5,26.8986,0,10000978,UK,CN,ANALYST,Beijing
-1179,2013-04-29,FP-non GTC,67698,0,11,46.9203,0,10000148,CN,US,ANAL
<TRUNCATED>
[38/50] [abbrv] kylin git commit: KYLIN-2245 code review
Posted by li...@apache.org.
KYLIN-2245 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fc1e11aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fc1e11aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fc1e11aa
Branch: refs/heads/master-cdh5.7
Commit: fc1e11aa8a2d20add72e382396787d9cb09771da
Parents: 0a441c3
Author: Yang Li <li...@apache.org>
Authored: Mon Dec 5 21:20:21 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Dec 5 21:20:21 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/cube/CubeInstance.java | 4 ----
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../main/java/org/apache/kylin/metadata/model/ISegment.java | 4 ++--
.../main/java/org/apache/kylin/metadata/model/Segments.java | 8 ++++----
4 files changed, 7 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index ecbb437..a3665f5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -371,10 +371,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return segments.calculateToBeSegments(newSegment, getModel().getPartitionDesc().isPartitioned());
}
- public Pair<CubeSegment, CubeSegment> findMergeOffsetsByDateRange(Segments<CubeSegment> segs, long startDate, long endDate, long skipSegDateRangeCap) {
- return this.segments.findMergeOffsetsByDateRange(segs, startDate, endDate, skipSegDateRangeCap);
- }
-
public CubeSegment getLastSegment() {
List<CubeSegment> existing = getSegments();
if (existing.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 296a4e7..0c80c07 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -519,7 +519,7 @@ public class CubeManager implements IRealizationProvider {
if (isOffsetsOn) {
// offset cube, merge by date range?
if (startOffset == endOffset) {
- Pair<CubeSegment, CubeSegment> pair = cube.findMergeOffsetsByDateRange(cube.getSegments(SegmentStatusEnum.READY), startDate, endDate, Long.MAX_VALUE);
+ Pair<CubeSegment, CubeSegment> pair = cube.getSegments(SegmentStatusEnum.READY).findMergeOffsetsByDateRange(startDate, endDate, Long.MAX_VALUE);
if (pair == null)
throw new IllegalArgumentException("Find no segments to merge by date range " + startDate + "-" + endDate + " for cube " + cube);
startOffset = pair.getFirst().getSourceOffsetStart();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index 9d26927..f006613 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -26,6 +26,8 @@ public interface ISegment {
public long getDateRangeEnd();
+ public boolean isSourceOffsetsOn();
+
public long getSourceOffsetStart();
public long getSourceOffsetEnd();
@@ -35,6 +37,4 @@ public interface ISegment {
public SegmentStatusEnum getStatus();
public long getLastBuildTime();
-
- public boolean isSourceOffsetsOn();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fc1e11aa/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index bc115cc..5198dc7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -187,8 +187,8 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
for (int s = 0; s < readySegs.size(); s++) {
ISegment seg = readySegs.get(s);
- Pair<T, T> p = findMergeOffsetsByDateRange(readySegs.getSubList(s, readySegs.size()), //
- seg.getDateRangeStart(), seg.getDateRangeStart() + toMergeRange, toMergeRange);
+ Pair<T, T> p = readySegs.getSubList(s, readySegs.size()) //
+ .findMergeOffsetsByDateRange(seg.getDateRangeStart(), seg.getDateRangeStart() + toMergeRange, toMergeRange);
if (p != null && p.getSecond().getDateRangeEnd() - p.getFirst().getDateRangeStart() >= toMergeRange)
return Pair.newPair(p.getFirst().getSourceOffsetStart(), p.getSecond().getSourceOffsetEnd());
}
@@ -197,10 +197,10 @@ public class Segments<T extends ISegment> extends ArrayList<T> {
return null;
}
- public Pair<T, T> findMergeOffsetsByDateRange(Segments<T> segments, long startDate, long endDate, long skipSegDateRangeCap) {
+ public Pair<T, T> findMergeOffsetsByDateRange(long startDate, long endDate, long skipSegDateRangeCap) {
// must be offset cube
Segments result = new Segments();
- for (ISegment seg : segments) {
+ for (ISegment seg : this) {
// include if date range overlaps
if (startDate < seg.getDateRangeEnd() && seg.getDateRangeStart() < endDate) {
[16/50] [abbrv] kylin git commit: KYLIN-2240 Add a toggle to ignore
all cube signature inconsistency temporally
Posted by li...@apache.org.
KYLIN-2240 Add a toggle to ignore all cube signature inconsistency temporally
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/28ba1eae
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/28ba1eae
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/28ba1eae
Branch: refs/heads/master-cdh5.7
Commit: 28ba1eaeac8fd29ff3872f6d14604813a89b3a6e
Parents: 93bf0d0
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Dec 1 11:35:34 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Dec 1 11:46:00 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/KylinConfigBase.java | 12 ++++++++----
.../main/java/org/apache/kylin/cube/model/CubeDesc.java | 6 +++++-
2 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/28ba1eae/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 766c04d..3c10826 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -79,7 +79,7 @@ abstract public class KylinConfigBase implements Serializable {
protected KylinConfigBase(Properties props, boolean force) {
this.properties = force ? props : BCC.check(props);
}
-
+
final protected String getOptional(String prop) {
return getOptional(prop, null);
}
@@ -161,7 +161,7 @@ abstract public class KylinConfigBase implements Serializable {
public boolean isDevEnv() {
return "DEV".equals(getOptional("kylin.env", "DEV"));
}
-
+
public String getDeployEnv() {
return getOptional("kylin.env", "DEV");
}
@@ -211,7 +211,7 @@ abstract public class KylinConfigBase implements Serializable {
public Map<String, String> getCubeCustomMeasureTypes() {
return getPropertiesByPrefix("kylin.metadata.custom-measure-types.");
}
-
+
// ============================================================================
// DICTIONARY & SNAPSHOT
// ============================================================================
@@ -278,6 +278,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.cube.algorithm.inmem-split-limit", "500"));
}
+ public boolean isIgnoreCubeSignatureInconsistency() {
+ return Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", "false"));
+ }
+
@Deprecated
public int getCubeAggrGroupMaxSize() {
return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-size", "12"));
@@ -298,7 +302,7 @@ abstract public class KylinConfigBase implements Serializable {
public void setMaxBuildingSegments(int maxBuildingSegments) {
setProperty("kylin.cube.max-building-segments", String.valueOf(maxBuildingSegments));
}
-
+
// ============================================================================
// JOB
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/28ba1eae/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 47063d0..327ce57 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -466,6 +466,11 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
* @return
*/
public boolean checkSignature() {
+ if (this.getConfig().isIgnoreCubeSignatureInconsistency()) {
+ logger.info("Skip checking cube signature");
+ return true;
+ }
+
if (KylinVersion.getCurrentVersion().isCompatibleWith(new KylinVersion(getVersion())) && !KylinVersion.getCurrentVersion().isSignatureCompatibleWith(new KylinVersion(getVersion()))) {
logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), getVersion());
return true;
@@ -1102,7 +1107,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return newCubeDesc;
}
-
private Collection ensureOrder(Collection c) {
TreeSet set = new TreeSet();
for (Object o : c)
[29/50] [abbrv] kylin git commit: KYLIN-2212 bug fix in
BuiltInFunctionTupleFilter serialization
Posted by li...@apache.org.
KYLIN-2212 bug fix in BuiltInFunctionTupleFilter serialization
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb686a94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb686a94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb686a94
Branch: refs/heads/master-cdh5.7
Commit: eb686a9428d930a60ec87883a2c253fb07f80a72
Parents: 0fd1ed6
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 2 18:34:29 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 2 18:34:29 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb686a94/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 1e836b7..f6e687b 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -151,17 +151,21 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
@Override
public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
BytesUtil.writeUTFString(name, buffer);
+ buffer.put((byte) (isReversed ? 1 : 0));
}
@Override
public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
this.name = BytesUtil.readUTFString(buffer);
+ this.isReversed = buffer.get() != 0;
this.initMethod();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
+ if (isReversed)
+ sb.append("NOT ");
sb.append(name);
sb.append("(");
for (int i = 0; i < methodParams.size(); i++) {
[46/50] [abbrv] kylin git commit: KYLIN-2252,
Enhance project/model/cube name check
Posted by li...@apache.org.
KYLIN-2252, Enhance project/model/cube name check
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e787165
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e787165
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e787165
Branch: refs/heads/master-cdh5.7
Commit: 1e787165edee713a59db8627e70f79edabfd7d9d
Parents: ea13af4
Author: Billy Liu <bi...@apache.org>
Authored: Wed Dec 7 09:42:58 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Dec 7 09:42:58 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller/CubeController.java | 10 ++++++++++
.../apache/kylin/rest/controller/CubeDescController.java | 4 ++--
.../org/apache/kylin/rest/controller/ModelController.java | 10 ++++++++++
.../apache/kylin/rest/controller/ProjectController.java | 10 +++++++++-
4 files changed, 31 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e787165/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 3846d28..f537231 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -83,6 +83,8 @@ import com.google.common.collect.Maps;
public class CubeController extends BasicController {
private static final Logger logger = LoggerFactory.getLogger(CubeController.class);
+ private static final char[] VALID_CUBENAME = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_".toCharArray();
+
@Autowired
private CubeService cubeService;
@@ -343,6 +345,10 @@ public class CubeController extends BasicController {
if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
throw new BadRequestException("Broken cube can't be cloned");
}
+ if (!StringUtils.containsOnly(newCubeName, VALID_CUBENAME)) {
+ logger.info("Invalid Cube name {}, only letters, numbers and underline supported.", newCubeName);
+ throw new BadRequestException("Invalid Cube name, only letters, numbers and underline supported.");
+ }
CubeDesc cubeDesc = cube.getDescriptor();
CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
@@ -421,6 +427,10 @@ public class CubeController extends BasicController {
logger.info("Cube name should not be empty.");
throw new BadRequestException("Cube name should not be empty.");
}
+ if (!StringUtils.containsOnly(name, VALID_CUBENAME)) {
+ logger.info("Invalid Cube name {}, only letters, numbers and underline supported.", name);
+ throw new BadRequestException("Invalid Cube name, only letters, numbers and underline supported.");
+ }
try {
desc.setUuid(UUID.randomUUID().toString());
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e787165/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
index 5a8eeec..0c8f487 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
@@ -44,8 +44,8 @@ public class CubeDescController extends BasicController {
/**
* Get detail information of the "Cube ID"
*
- * @param cubeDescName
- * Cube ID
+ * @param cubeName
+ * Cube Name
* @return
* @throws IOException
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e787165/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
index 5f6a91b..df9ecfb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
@@ -61,6 +61,8 @@ import com.fasterxml.jackson.databind.JsonMappingException;
public class ModelController extends BasicController {
private static final Logger logger = LoggerFactory.getLogger(ModelController.class);
+ private static final char[] VALID_MODELNAME = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_".toCharArray();
+
@Autowired
private ModelService modelService;
@@ -93,6 +95,10 @@ public class ModelController extends BasicController {
logger.info("Model name should not be empty.");
throw new BadRequestException("Model name should not be empty.");
}
+ if (!StringUtils.containsOnly(modelDesc.getName(), VALID_MODELNAME)) {
+ logger.info("Invalid Model name {}, only letters, numbers and underline supported.", modelDesc.getName());
+ throw new BadRequestException("Invalid Model name, only letters, numbers and underline supported.");
+ }
try {
modelDesc.setUuid(UUID.randomUUID().toString());
@@ -174,6 +180,10 @@ public class ModelController extends BasicController {
logger.info("New model name is empty.");
throw new BadRequestException("New model name is empty.");
}
+ if (!StringUtils.containsOnly(newModelName, VALID_MODELNAME)) {
+ logger.info("Invalid Model name {}, only letters, numbers and underline supported.", newModelName);
+ throw new BadRequestException("Invalid Model name, only letters, numbers and underline supported.");
+ }
DataModelDesc newModelDesc = DataModelDesc.getCopyOf(modelDesc);
newModelDesc.setName(newModelName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e787165/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
index 496e44a..05af82c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
@@ -27,6 +27,7 @@ import org.apache.kylin.common.persistence.AclEntity;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.request.CreateProjectRequest;
import org.apache.kylin.rest.request.UpdateProjectRequest;
@@ -60,6 +61,8 @@ import org.springframework.web.bind.annotation.ResponseBody;
public class ProjectController extends BasicController {
private static final Logger logger = LoggerFactory.getLogger(ProjectController.class);
+ private static final char[] VALID_PROJECTNAME = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_".toCharArray();
+
@Autowired
private ProjectService projectService;
@Autowired
@@ -197,7 +200,12 @@ public class ProjectController extends BasicController {
@ResponseBody
public ProjectInstance saveProject(@RequestBody CreateProjectRequest projectRequest) {
if (StringUtils.isEmpty(projectRequest.getName())) {
- throw new InternalErrorException("A project name must be given to create a project");
+ logger.info("Project name should not be empty.");
+ throw new BadRequestException("Project name should not be empty.");
+ }
+ if (!StringUtils.containsOnly(projectRequest.getName(), VALID_PROJECTNAME)) {
+ logger.info("Invalid Project name {}, only letters, numbers and underline supported.", projectRequest.getName());
+ throw new BadRequestException("Invalid Project name, only letters, numbers and underline supported.");
}
ProjectInstance createdProj = null;
[26/50] [abbrv] kylin git commit: KYLIN-2245 refine CubeSegments
Posted by li...@apache.org.
KYLIN-2245 refine CubeSegments
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ca53954
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ca53954
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ca53954
Branch: refs/heads/master-cdh5.7
Commit: 3ca53954415c62c2a91473c6d7bf831504b5e9db
Parents: 4a0ee79
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Dec 2 15:35:52 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 2 17:26:07 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 115 +++------------
.../java/org/apache/kylin/cube/CubeManager.java | 10 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 8 +-
.../apache/kylin/metadata/model/ISegment.java | 6 +-
.../apache/kylin/metadata/model/Segments.java | 140 +++++++++++++++++++
.../hbase/util/ExtendCubeToHybridCLI.java | 4 +-
.../kylin/tool/ExtendCubeToHybridCLI.java | 4 +-
7 files changed, 176 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ca53954/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 061ab23..8b12c2e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -18,8 +18,6 @@
package org.apache.kylin.cube;
-import java.util.ArrayList;
-import java.util.LinkedList;
import java.util.List;
import java.util.Set;
@@ -34,6 +32,7 @@ import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
@@ -64,7 +63,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
cubeInstance.setName(cubeName);
cubeInstance.setDescName(cubeDesc.getName());
cubeInstance.setCreateTimeUTC(System.currentTimeMillis());
- cubeInstance.setSegments(new ArrayList<CubeSegment>());
+ cubeInstance.setSegments(new Segments<CubeSegment>());
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
cubeInstance.updateRandomUuid();
@@ -87,7 +86,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
@JsonManagedReference
@JsonProperty("segments")
- private List<CubeSegment> segments = new ArrayList<CubeSegment>();
+ private Segments<CubeSegment> segments = new Segments<CubeSegment>();
@JsonProperty("create_time_utc")
private long createTimeUTC;
@@ -97,39 +96,11 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public List<CubeSegment> getBuildingSegments() {
- List<CubeSegment> buildingSegments = new ArrayList<CubeSegment>();
- if (null != segments) {
- for (CubeSegment segment : segments) {
- if (SegmentStatusEnum.NEW == segment.getStatus() || SegmentStatusEnum.READY_PENDING == segment.getStatus()) {
- buildingSegments.add(segment);
- }
- }
- }
-
- return buildingSegments;
+ return segments.getBuildingSegments();
}
public List<CubeSegment> getMergingSegments(CubeSegment mergedSegment) {
- LinkedList<CubeSegment> result = new LinkedList<CubeSegment>();
- if (mergedSegment == null)
- return result;
-
- for (CubeSegment seg : this.segments) {
- if (seg.getStatus() != SegmentStatusEnum.READY && seg.getStatus() != SegmentStatusEnum.READY_PENDING)
- continue;
-
- if (seg == mergedSegment)
- continue;
-
- if (mergedSegment.sourceOffsetContains(seg)) {
- // make sure no holes
- if (result.size() > 0 && result.getLast().getSourceOffsetEnd() != seg.getSourceOffsetStart())
- throw new IllegalStateException("Merging segments must not have holes between " + result.getLast() + " and " + seg);
-
- result.add(seg);
- }
- }
- return result;
+ return segments.getMergingSegments(mergedSegment);
}
public CubeDesc getDescriptor() {
@@ -154,7 +125,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
// in a temporary broken state, so that user can edit and fix it. Broken state is often due to
// schema changes at source.
public boolean allowBrokenDescriptor() {
- return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN) && segments.isEmpty();
+ return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN) && segments.isEmpty();
}
public String getResourcePath() {
@@ -269,65 +240,30 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public CubeSegment getFirstSegment() {
- if (this.segments == null || this.segments.size() == 0) {
- return null;
- } else {
- return this.segments.get(0);
- }
+ return segments.getFirstSegment();
}
public CubeSegment getLatestReadySegment() {
- CubeSegment latest = null;
- for (int i = segments.size() - 1; i >= 0; i--) {
- CubeSegment seg = segments.get(i);
- if (seg.getStatus() != SegmentStatusEnum.READY)
- continue;
- if (latest == null || latest.getDateRangeEnd() < seg.getDateRangeEnd()) {
- latest = seg;
- }
- }
- return latest;
+ return segments.getLatestReadySegment();
}
public CubeSegment getLatestBuiltSegment() {
- CubeSegment latest = null;
- for (int i = segments.size() - 1; i >= 0; i--) {
- CubeSegment seg = segments.get(i);
- if (seg.getLastBuildTime() > 0) {
- if (latest == null || seg.getLastBuildTime() > latest.getLastBuildTime())
- latest = seg;
- }
- }
- return latest;
+ return segments.getLatestBuiltSegment();
}
-
- public List<CubeSegment> getSegments() {
+
+ public Segments<CubeSegment> getSegments() {
return segments;
}
- public List<CubeSegment> getSegments(SegmentStatusEnum status) {
- List<CubeSegment> result = new ArrayList<CubeSegment>();
-
- for (CubeSegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
-
- return result;
+ public Segments<CubeSegment> getSegments(SegmentStatusEnum status) {
+ return segments.getSegments(status);
}
public CubeSegment getSegment(String name, SegmentStatusEnum status) {
- for (CubeSegment segment : segments) {
- if ((null != segment.getName() && segment.getName().equals(name)) && (status == null || segment.getStatus() == status)) {
- return segment;
- }
- }
-
- return null;
+ return segments.getSegment(name, status);
}
- public void setSegments(List<CubeSegment> segments) {
+ public void setSegments(Segments segments) {
this.segments = segments;
}
@@ -393,30 +329,15 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
public Set<ColumnDesc> getAllColumnDescs() {
return getDescriptor().listAllColumnDescs();
}
-
+
@Override
public long getDateRangeStart() {
- List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long startTime = Long.MAX_VALUE;
- for (CubeSegment seg : readySegs) {
- startTime = Math.min(startTime, seg.getDateRangeStart());
- }
-
- return startTime;
+ return segments.getDateRangeStart();
}
@Override
public long getDateRangeEnd() {
-
- List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long endTime = Long.MIN_VALUE;
- for (CubeSegment seg : readySegs) {
- endTime = Math.max(endTime, seg.getDateRangeEnd());
- }
-
- return endTime;
+ return segments.getDateRangeEnd();
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ca53954/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 119a21a..4ba29af 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -56,6 +56,7 @@ import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -213,7 +214,6 @@ public class CubeManager implements IRealizationProvider {
return result;
}
-
public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, ReadableTable inpTable) throws IOException {
CubeDesc cubeDesc = cubeSeg.getCubeDesc();
if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
@@ -222,18 +222,17 @@ public class CubeManager implements IRealizationProvider {
String builderClass = cubeDesc.getDictionaryBuilderClass(col);
DictionaryInfo dictInfo = getDictionaryManager().buildDictionary(cubeDesc.getModel(), col, inpTable, builderClass);
-
saveDictionaryInfo(cubeSeg, col, dictInfo);
return dictInfo;
}
-
+
public DictionaryInfo saveDictionary(CubeSegment cubeSeg, TblColRef col, ReadableTable inpTable, Dictionary<String> dict) throws IOException {
CubeDesc cubeDesc = cubeSeg.getCubeDesc();
if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col))
return null;
DictionaryInfo dictInfo = getDictionaryManager().saveDictionary(cubeDesc.getModel(), col, inpTable, dict);
-
+
saveDictionaryInfo(cubeSeg, col, dictInfo);
return dictInfo;
}
@@ -366,7 +365,7 @@ public class CubeManager implements IRealizationProvider {
CubeInstance cube = update.getCubeInstance();
logger.info("Updating cube instance '" + cube.getName() + "'");
- List<CubeSegment> newSegs = Lists.newArrayList(cube.getSegments());
+ Segments<CubeSegment> newSegs = (Segments) cube.getSegments().clone();
if (update.getToAddSegs() != null)
newSegs.addAll(Arrays.asList(update.getToAddSegs()));
@@ -385,7 +384,6 @@ public class CubeManager implements IRealizationProvider {
}
}
}
-
}
if (update.getToUpdateSegs() != null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ca53954/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 1ec01a2..1fc28eb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -217,6 +217,7 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
this.inputRecordsSize = inputRecordsSize;
}
+ @Override
public long getLastBuildTime() {
return lastBuildTime;
}
@@ -366,11 +367,12 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
}
// date range is used in place of source offsets when offsets are missing
- public boolean sourceOffsetContains(CubeSegment seg) {
+ @Override
+ public boolean sourceOffsetContains(ISegment seg) {
if (isSourceOffsetsOn())
- return sourceOffsetStart <= seg.sourceOffsetStart && seg.sourceOffsetEnd <= sourceOffsetEnd;
+ return sourceOffsetStart <= ((CubeSegment) seg).sourceOffsetStart && ((CubeSegment) seg).sourceOffsetEnd <= sourceOffsetEnd;
else
- return dateRangeContains(seg);
+ return dateRangeContains(((CubeSegment) seg));
}
public void validate() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ca53954/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
index f69ae3f..e97f4f4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ISegment.java
@@ -18,7 +18,7 @@
package org.apache.kylin.metadata.model;
-public interface ISegment {
+public interface ISegment{
public String getName();
@@ -33,4 +33,8 @@ public interface ISegment {
public DataModelDesc getModel();
public SegmentStatusEnum getStatus();
+
+ public long getLastBuildTime();
+
+ public boolean sourceOffsetContains(ISegment seg);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ca53954/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
new file mode 100644
index 0000000..104c2af
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import java.util.ArrayList;
+
+public class Segments<T extends ISegment> extends ArrayList<T> {
+
+ public T getFirstSegment() {
+ if (this == null || this.size() == 0) {
+ return null;
+ } else {
+ return this.get(0);
+ }
+ }
+
+ public long getDateRangeStart() {
+ Segments<T> readySegs = getSegments(SegmentStatusEnum.READY);
+
+ long startTime = Long.MAX_VALUE;
+ for (T seg : readySegs) {
+ startTime = Math.min(startTime, seg.getDateRangeStart());
+ }
+
+ return startTime;
+ }
+
+ public long getDateRangeEnd() {
+ Segments<T> readySegs = getSegments(SegmentStatusEnum.READY);
+
+ long endTime = Long.MIN_VALUE;
+ for (T seg : readySegs) {
+ endTime = Math.max(endTime, seg.getDateRangeEnd());
+ }
+
+ return endTime;
+ }
+
+ public T getLatestReadySegment() {
+ T latest = null;
+ for (int i = this.size() - 1; i >= 0; i--) {
+ T seg = this.get(i);
+ if (seg.getStatus() != SegmentStatusEnum.READY)
+ continue;
+ if (latest == null || latest.getDateRangeEnd() < seg.getDateRangeEnd()) {
+ latest = seg;
+ }
+ }
+ return latest;
+ }
+
+ public T getLatestBuiltSegment() {
+ T latest = null;
+ for (int i = this.size() - 1; i >= 0; i--) {
+ T seg = this.get(i);
+ if (seg.getLastBuildTime() > 0) {
+ if (latest == null || seg.getLastBuildTime() > latest.getLastBuildTime())
+ latest = seg;
+ }
+ }
+ return latest;
+ }
+
+ public Segments getSegments(SegmentStatusEnum status) {
+ Segments<T> result = new Segments<>();
+
+ for (T segment : this) {
+ if (segment.getStatus() == status) {
+ result.add(segment);
+ }
+ }
+ return result;
+ }
+
+ public T getSegment(String name, SegmentStatusEnum status) {
+ for (T segment : this) {
+ if ((null != segment.getName() && segment.getName().equals(name)) && (status == null || segment.getStatus() == status)) {
+ return segment;
+ }
+ }
+ return null;
+ }
+
+ public Segments getBuildingSegments() {
+ Segments<T> buildingSegments = new Segments();
+ if (null != this) {
+ for (T segment : this) {
+ if (SegmentStatusEnum.NEW == segment.getStatus() || SegmentStatusEnum.READY_PENDING == segment.getStatus()) {
+ buildingSegments.add(segment);
+ }
+ }
+ }
+ return buildingSegments;
+ }
+
+ public Segments getMergingSegments(T mergedSegment) {
+ Segments<T> result = new Segments();
+ if (mergedSegment == null)
+ return result;
+
+ for (T seg : this) {
+ if (seg.getStatus() != SegmentStatusEnum.READY && seg.getStatus() != SegmentStatusEnum.READY_PENDING)
+ continue;
+
+ if (seg == mergedSegment)
+ continue;
+
+ if (mergedSegment.sourceOffsetContains(seg)) {
+ // make sure no holes
+ if (result.size() > 0 && result.getLast().getSourceOffsetEnd() != seg.getSourceOffsetStart())
+ throw new IllegalStateException("Merging segments must not have holes between " + result.getLast() + " and " + seg);
+
+ result.add(seg);
+ }
+ }
+ return result;
+ }
+
+ private T getLast() {
+ assert this.size() != 0;
+ return this.get(this.size() - 1);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ca53954/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index 8d0cb82..61c73d5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -18,7 +18,6 @@
package org.apache.kylin.storage.hbase.util;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -45,6 +44,7 @@ import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
@@ -189,7 +189,7 @@ public class ExtendCubeToHybridCLI {
logger.info("CubeDesc was saved at: " + cubeDesc.getResourcePath());
// clear segments for old cube
- cubeInstance.setSegments(new ArrayList<CubeSegment>());
+ cubeInstance.setSegments(new Segments<CubeSegment>());
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
store.putResource(cubeInstance.getResourcePath(), cubeInstance, CubeManager.CUBE_SERIALIZER);
logger.info("CubeInstance was saved at: " + cubeInstance.getResourcePath());
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ca53954/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index 8bd5766..19e5db0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -18,7 +18,6 @@
package org.apache.kylin.tool;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -45,6 +44,7 @@ import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
@@ -185,7 +185,7 @@ public class ExtendCubeToHybridCLI {
logger.info("CubeDesc was saved at: " + cubeDesc.getResourcePath());
// clear segments for old cube
- cubeInstance.setSegments(new ArrayList<CubeSegment>());
+ cubeInstance.setSegments(new Segments());
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
store.putResource(cubeInstance.getResourcePath(), cubeInstance, CubeManager.CUBE_SERIALIZER);
logger.info("CubeInstance was saved at: " + cubeInstance.getResourcePath());
[47/50] [abbrv] kylin git commit: minor, chmod +x on scripts
Posted by li...@apache.org.
minor, chmod +x on scripts
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/081ed0d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/081ed0d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/081ed0d5
Branch: refs/heads/master-cdh5.7
Commit: 081ed0d505e5638b1294c59e57ed651a0f9dffa1
Parents: 1e78716
Author: lidongsjtu <li...@apache.org>
Authored: Wed Dec 7 11:10:49 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Dec 7 11:10:49 2016 +0800
----------------------------------------------------------------------
dev-support/sync_hbase_cdh_branches.sh | 0
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/081ed0d5/dev-support/sync_hbase_cdh_branches.sh
----------------------------------------------------------------------
diff --git a/dev-support/sync_hbase_cdh_branches.sh b/dev-support/sync_hbase_cdh_branches.sh
old mode 100644
new mode 100755
[23/50] [abbrv] kylin git commit: KYLIN-2192 More Robust Global
Dictionary
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
index caa8754..949599d 100644
--- a/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
+++ b/examples/test_case_data/localmeta/data/flatten_data_for_without_slr_left_join.csv
@@ -1,402 +1,402 @@
-2013-03-31,48028,0,\N,\N,\N,Auction,12,184.21,10000001,1,Ebay
-2013-11-12,164262,0,\N,\N,\N,Others,5,172.03,10000002,1,Ebay
-2013-04-06,82494,15,BookMagazines,NULL,Comic Books,Auction,14,66.6,10000003,1,Ebay
-2013-05-17,66767,15,Home & Garden,NULL,Dogs,Auction,12,92.98,10000004,1,Ebay
-2013-05-20,152801,0,Jewelry & Watches,NULL,Earrings,FP-GTC,5,132.33,10000005,1,Ebay
-2013-06-16,43398,0,Home & Garden,NULL,Cheese & Crackers,FP-GTC,13,7.12,10000006,1,Ebay
-2013-06-14,95173,0,Health & Beauty,Bath & Body,Bath Sets & Kits,Auction,14,204.28,10000007,1,Ebay
-2013-03-22,158666,15,ToyHobbies,Action Figures,Anime & Manga,Auction,13,35.72,10000008,1,Ebay
-2013-03-10,12688,0,eBay Premier,Books & Manuscripts,Books: Other,Auction,12,4.13,10000009,1,Ebay
-2013-11-01,103324,15,ClothinShoeAccessories,Women's Shoes,Mixed Items,FP-GTC,5,27.48,10000010,1,Ebay
-2013-06-16,108782,15,Vehicle Parts & Accessories,CaTruck Parts,Car Care & Cleaning,FP-GTC,14,9.26,10000011,1,Ebay
-2013-09-12,80287,0,Computers/Tablets & Networking,Software,Office & Business,Auction,12,3.18,10000012,1,Ebay
-2013-09-28,140746,100,eBay Motors,Parts & Accessories,Vintage Car & Truck Parts,Others,13,3.18,10000013,1,Ebay
-2013-06-15,87118,0,Sporting Goods,Outdoor Sports,Paintball,ABIN,14,377.94,10000014,1,Ebay
-2013-03-14,25147,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Baseball-MLB,Auction,12,146.33,10000015,1,Ebay
-2013-09-01,170302,15,Crafts,Embroidery,Design CDs,FP-GTC,5,51.23,10000016,1,Ebay
-2013-05-29,53064,0,Business & Industrial,Heavy Equipment,Antique & Vintage Farm Equip,FP-non GTC,13,72.65,10000017,1,Ebay
-2013-05-31,132939,0,Jewelry & Watches,Fashion Jewelry,Other,Auction,13,66.6,10000018,1,Ebay
-2013-03-18,113593,15,Phones,Mobile Phones,Mobile Phones,Auction,12,9.26,10000019,1,Ebay
-2013-07-19,34273,100,eBay Motors,Parts & Accessories,Motorcycle,Auction,14,583.44,10000020,1,Ebay
-2013-06-23,106340,15,Home & Garden,Gardening,Hand Tools,FP-GTC,14,638.72,10000021,1,Ebay
-2013-05-20,150265,15,Baby,Baby Clothing,Boys,FP-GTC,14,4.54,10000022,1,Ebay
-2013-05-17,24760,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Hockey-NHL,FP-GTC,12,319.79,10000023,1,Ebay
-2013-03-11,37831,0,Collectibles,Advertising,Merchandise & Memorabilia,Auction,12,20.35,10000024,1,Ebay
-2013-01-30,1120,3,Books,First Editions,Other,FP-non GTC,5,223.63,10000025,1,Ebay
-2013-01-26,43972,100,eBay Motors,Parts & Accessories,ATV Parts,FP-GTC,13,204.28,10000026,1,Ebay
-2013-03-22,166013,15,Computers,Computer Components & Parts,Video Capture & TV Tuner Cards,Auction,14,5.48,10000027,1,Ebay
-2013-07-23,15568,15,Baby,Baby Clothing,Unisex,Auction,14,27.48,10000028,1,Ebay
-2013-07-27,103178,15,ClothinShoeAccessories,Women's Bags,Women's Bags,FP-GTC,5,21.72,10000029,1,Ebay
-2013-10-29,2023,0,Sporting Goods,Team Sports,Basketball,ABIN,12,3.18,10000030,1,Ebay
-2013-10-08,94847,0,Consumer Electronics,Vehicle Electronics & GPS,Car Video,FP-GTC,11,491.32,10000031,1,Ebay
-2013-04-26,15868,0,Real Estate,Land,Land,Auction,14,448.8,10000032,1,Ebay
-2013-01-01,32876,0,Home & Garden,Home Improvement,Plumbing & Fixtures,Auction,13,415.73,10000033,1,Ebay
-2013-01-15,62179,0,ClothinShoes & Accessories,Women's Clothing,Athletic Apparel,Auction,13,377.94,10000034,1,Ebay
-2013-05-27,33038,15,Musical Instruments,Instruments,Guitars (Electric),FP-GTC,14,146.33,10000035,1,Ebay
-2013-11-11,156614,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,FP-GTC,5,7.12,10000036,1,Ebay
-2013-03-08,106246,0,Health & Beauty,Hair Care & Styling,Shampoo & Conditioning,Auction,13,42.99,10000037,1,Ebay
-2013-03-25,20865,0,ClothinShoes & Accessories,Men's Clothing,Athletic Apparel,Auction,13,12.85,10000038,1,Ebay
-2013-08-20,15115,0,Video Games & Consoles,Video Games,Video Games,FP-GTC,13,55.89,10000039,1,Ebay
-2013-05-17,3838,0,Jewelry & Watches,Fashion Jewelry,Charms & Charm Bracelets,FP-GTC,14,73.26,10000040,1,Ebay
-2013-06-05,759,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,Auction,11,112.56,10000041,1,Ebay
-2013-10-08,61323,0,Consumer Electronics,TVideo & Home Audio,TVideo & Audio Accessories,FP-non GTC,11,3.49,10000042,1,Ebay
-2013-08-14,121153,0,Baby,Nursery Decor,Night Lights,Auction,13,184.21,10000043,1,Ebay
-2013-08-14,88750,0,Consumer Electronics,Vehicle Electronics & GPS,Radar & Laser Detectors,Auction,13,157.14,10000044,1,Ebay
-2013-05-17,161567,15,Computers,Laptop & Desktop Accessories,Laptop Batteries,FP-GTC,14,72.65,10000045,1,Ebay
-2013-08-09,113802,15,Lots More...,Metaphysical,Herbs,FP-GTC,14,51.23,10000046,1,Ebay
-2013-06-30,15808,15,ClothinShoeAccessories,Women's Clothing,Tops & Blouses,FP-non GTC,14,15.85,10000047,1,Ebay
-2013-06-03,174053,3,Vehicle Parts & Accessories,Car Parts,External & Body Parts,FP-GTC,13,7.12,10000048,1,Ebay
-2013-12-31,2635,0,Toys & Hobbies,Toy Soldiers,1970-Now,Auction,14,12.04,10000049,1,Ebay
-2013-12-25,1161,3,DVFilm & TV,Other Formats,Videos: NTSC (US),Auction,13,73.26,10000050,1,Ebay
-2013-03-28,64076,0,Computers/Tablets & Networking,Enterprise NetworkinServers,Switches & Hubs,FP-non GTC,5,184.21,10000051,1,Ebay
-2013-01-30,33977,15,Crafts,Scrapbooking,Albums,FP-GTC,13,172.03,10000052,1,Ebay
-2013-12-05,31673,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Racing-NASCAR,FP-GTC,14,122.78,10000053,1,Ebay
-2013-10-08,174106,3,Vehicle Parts & Accessories,Car Parts,Transmission & Drivetrain,Auction,14,92.98,10000054,1,Ebay
-2013-12-27,26249,0,Business & Industrial,Printing & Graphic Arts,Commercial Printing Presses,Auction,13,12.19,10000055,1,Ebay
-2013-12-16,159184,0,Sporting Goods,Winter Sports,Snowboarding,FP-GTC,5,15.65,10000056,1,Ebay
-2013-10-17,10058,3,Events Tickets,Other Tickets,Other Tickets,FP-GTC,11,101.79,10000057,1,Ebay
-2013-11-17,48904,0,ClothinShoes & Accessories,Vintage,Women's Vintage Clothing,ABIN,12,7.12,10000058,1,Ebay
-2013-09-18,145970,0,Toys & Hobbies,Models & Kits,Automotive,FP-non GTC,14,12.85,10000059,1,Ebay
-2013-06-30,963,0,ClothinShoes & Accessories,Vintage,Women's Vintage Shoes,FP-GTC,13,12.19,10000060,1,Ebay
-2013-10-12,118687,3,Health & Beauty,Fragrances,Women's Fragrances,FP-GTC,13,92.98,10000061,1,Ebay
-2013-08-20,20886,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,FP-GTC,14,42.99,10000062,1,Ebay
-2013-08-29,148324,15,Phones,Mobile Accessories,CaseCoverSkins,Auction,13,1.88,10000063,1,Ebay
-2013-07-17,139255,15,Jewellery & Watches,Fine Jewellery,Earrings,Auction,14,21.14,10000064,1,Ebay
-2013-07-23,20213,0,Collectibles,Postcards,US StateCities & Towns,FP-GTC,5,21.14,10000065,1,Ebay
-2013-01-06,32996,15,Movies,Television Memorabilia,Clippings,Auction,13,132.33,10000066,1,Ebay
-2013-08-14,99985,0,Collectibles,Trading Cards,Sci-FFantasy,FP-GTC,14,120.87,10000067,1,Ebay
-2013-08-10,67703,3,Jewellery & Watches,Jewellery Boxes & Supplies,Jewellery Display,Auction,14,120.87,10000068,1,Ebay
-2013-09-28,65,0,Collectibles,Comics,Platinum Age (1897-1937),FP-non GTC,11,9.26,10000069,1,Ebay
-2013-08-21,130,0,Collectibles,Transportation,Railroadiana & Trains,FP-non GTC,14,16.26,10000070,1,Ebay
-2013-03-11,164,0,Computers/Tablets & Networking,Computer Components & Parts,CPUProcessors,FP-GTC,14,157.14,10000071,1,Ebay
-2013-05-06,216,0,Sports MeCards & Fan Shop,Cards,Hockey,FP-GTC,11,1.88,10000072,1,Ebay
-2013-05-17,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,12.04,10000073,1,Ebay
-2013-01-10,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,14,189.23,10000074,1,Ebay
-2013-05-05,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,73.26,10000075,1,Ebay
-2013-02-03,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,4.13,10000076,1,Ebay
-2013-11-26,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,290.72,10000077,1,Ebay
-2013-08-30,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,265.56,10000078,1,Ebay
-2013-04-26,279,15,BookMagazines,Children's Books,Children's Books,FP-GTC,5,5.91,10000079,1,Ebay
-2013-06-30,314,0,ClothinShoes & Accessories,Women's Clothing,Other,Auction,5,319.79,10000080,1,Ebay
-2013-06-30,314,211,ClothinShoes & Accessories,Womens' Clothing,Other,Auction,5,246,10000081,1,Ebay
-2013-12-16,314,211,ClothinShoes & Accessories,Womens' Clothing,Other,Auction,5,20.35,10000082,1,Ebay
-2013-12-15,314,0,ClothinShoes & Accessories,Women's Clothing,Other,Auction,5,36.7,10000083,1,Ebay
-2013-08-17,533,0,Coins & Paper Money,Coins: World,Africa,Auction,13,101.79,10000084,1,Ebay
-2013-12-15,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,5,47.71,10000085,1,Ebay
-2013-02-04,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,5,3.49,10000086,1,Ebay
-2013-01-11,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,13,46.44,10000087,1,Ebay
-2013-02-04,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,13,4.54,10000088,1,Ebay
-2013-05-17,1357,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,14,3.18,10000089,1,Ebay
-2013-11-12,1504,0,Business & Industrial,Electrical & Test Equipment,Test Equipment,FP-GTC,14,86.58,10000090,1,Ebay
-2013-08-21,4943,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-GTC,13,12.85,10000091,1,Ebay
-2013-05-09,6762,0,Unknown,Unknown,Unknown,ABIN,13,16.26,10000092,1,Ebay
-2013-09-19,9426,3,Mobile Phones & Communication,Home Phones & Accessories,Phone Accessories,Auction,13,21.14,10000093,1,Ebay
-2013-02-06,10866,0,Collectibles,Animals,Farm & Countryside,FP-non GTC,14,20.6,10000094,1,Ebay
-2013-02-02,11554,0,ClothinShoes & Accessories,Women's Clothing,Jeans,Auction,13,246,10000095,1,Ebay
-2013-08-23,11848,0,Health & Beauty,Fragrances,Women,FP-GTC,14,109,10000096,1,Ebay
-2013-08-03,13836,0,Collectibles,Decorative Collectibles,Spoons,Auction,13,39.41,10000097,1,Ebay
-2013-05-17,13836,0,Collectibles,Decorative Collectibles,Spoons,Auction,14,16.26,10000098,1,Ebay
-2013-06-06,13987,0,Collectibles,Paper,Booklets,FP-GTC,13,112.56,10000099,1,Ebay
-2013-07-02,15687,0,ClothinShoes & Accessories,Men's Clothing,T-Shirts,Auction,14,184.21,10000100,1,Ebay
-2013-10-25,15687,0,ClothinShoes & Accessories,Men's Clothing,T-Shirts,Auction,11,27.48,10000001,1,Ebay
-2013-04-20,16145,3,Computers/Tablets & Networking,Computer Components & Parts,Other Components & Parts,FP-non GTC,12,26.45,10000002,1,Ebay
-2013-03-12,16145,0,Computers/Tablets & Networking,Computer Components & Parts,Other,FP-non GTC,13,415.73,10000003,1,Ebay
-2013-03-28,16509,0,Toys & Hobbies,Model Railroads & Trains,S Scale,ABIN,5,56.36,10000004,1,Ebay
-2013-10-29,16509,0,Toys & Hobbies,Model Railroads & Trains,S Scale,ABIN,5,2.44,10000005,1,Ebay
-2013-05-22,20485,0,Home & Garden,Furniture,Other,FP-GTC,14,269.76,10000006,1,Ebay
-2013-01-25,20485,101,CasArredamento e Bricolage,Cucina,Altro per cucina,FP-GTC,12,109,10000007,1,Ebay
-2013-06-12,20485,101,CasArredamento e Bricolage,Cucina,Altro per cucina,FP-GTC,12,101.79,10000008,1,Ebay
-2013-12-26,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,246,10000009,1,Ebay
-2013-12-26,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,189.23,10000010,1,Ebay
-2013-12-31,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,15.65,10000011,1,Ebay
-2013-10-04,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,28.23,10000012,1,Ebay
-2013-03-16,24541,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,College-NCAA,FP-GTC,5,16.26,10000013,1,Ebay
-2013-05-21,26262,0,Collectibles,Advertising,Food & Beverage,FP-GTC,5,122.78,10000014,1,Ebay
-2013-01-28,30059,3,Cameras & Photography,Lenses & Filters,Lens AdapterMounts & Tubes,FP-GTC,14,172.03,10000015,1,Ebay
-2013-04-26,31387,3,Jewellery & Watches,Watches,Wristwatches,Auction,14,42.99,10000016,1,Ebay
-2013-10-06,31387,3,Jewellery & Watches,Watches,Wristwatches,Auction,14,207.5,10000017,1,Ebay
-2013-11-06,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,5.91,10000018,1,Ebay
-2013-10-06,31519,3,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,39.41,10000019,1,Ebay
-2013-12-28,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,16.26,10000020,1,Ebay
-2013-11-06,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,16.26,10000021,1,Ebay
-2013-11-06,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,78.48,10000022,1,Ebay
-2013-12-28,31519,3,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,190.22,10000023,1,Ebay
-2013-06-11,35570,100,eBay Motors,Parts & Accessories,Motorcycle Parts,FP-GTC,12,2.44,10000024,1,Ebay
-2013-01-10,36250,0,Sporting Goods,Hunting,Decoys,Auction,5,7.12,10000025,1,Ebay
-2013-09-17,38238,0,Home & Garden,Home Decor,Other,FP-non GTC,14,36.7,10000026,1,Ebay
-2013-08-14,40059,3,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,FP-GTC,14,35.72,10000027,1,Ebay
-2013-08-09,40059,3,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,FP-GTC,14,3.49,10000028,1,Ebay
-2013-12-02,41940,0,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,FP-GTC,13,223.63,10000029,1,Ebay
-2013-02-01,41940,0,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,FP-GTC,13,265.56,10000030,1,Ebay
-2013-07-28,43479,0,Cameras & Photo,Film Photography,Other,FP-non GTC,13,62.02,10000031,1,Ebay
-2013-06-16,44079,0,Sporting Goods,Exercise & Fitness,GyWorkout & Yoga,FP-GTC,12,46.44,10000032,1,Ebay
-2013-08-23,45238,101,Abbigliamento e accessori,Donna: Accessori,SciarpFoulard e Scialli,Auction,14,132.33,10000033,1,Ebay
-2013-06-15,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,Auction,13,448.8,10000034,1,Ebay
-2013-06-15,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,FP-non GTC,14,207.5,10000035,1,Ebay
-2013-06-01,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,FP-non GTC,14,190.22,10000036,1,Ebay
-2013-08-10,46575,0,Business & Industrial,Light Equipment & Tools,Air Tools,FP-GTC,14,16.71,10000037,1,Ebay
-2013-03-22,50508,0,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,FP-non GTC,13,4.13,10000038,1,Ebay
-2013-12-25,50508,0,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,FP-non GTC,13,1.88,10000039,1,Ebay
-2013-07-22,50677,0,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,FP-GTC,13,491.32,10000040,1,Ebay
-2013-04-13,50677,0,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,FP-GTC,5,2.44,10000041,1,Ebay
-2013-04-16,51582,0,ClothinShoes & Accessories,Kids' ClothinShoes & Accs,Girls' Clothing (Sizes 4 & Up),Auction,14,56.36,10000042,1,Ebay
-2013-08-21,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-GTC,13,15.85,10000043,1,Ebay
-2013-04-22,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-non GTC,14,2.44,10000044,1,Ebay
-2013-08-29,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-GTC,14,7.12,10000045,1,Ebay
-2013-05-16,57784,0,ClothinShoes & Accessories,Baby & Toddler Clothing,Boys' Clothing (Newborn-5T),Auction,14,35.72,10000046,1,Ebay
-2013-08-23,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,11,9.26,10000047,1,Ebay
-2013-07-10,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,3.18,10000048,1,Ebay
-2013-08-10,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,638.72,10000049,1,Ebay
-2013-08-23,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,141.7,10000050,1,Ebay
-2013-04-18,57990,0,ClothinShoes & Accessories,Men's Clothing,Casual Shirts,ABIN,13,12.19,10000051,1,Ebay
-2013-07-10,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,132.33,10000052,1,Ebay
-2013-06-16,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,ABIN,5,5.48,10000053,1,Ebay
-2013-07-15,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,11,1.88,10000054,1,Ebay
-2013-11-06,60340,0,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,FP-GTC,14,12.85,10000055,1,Ebay
-2013-12-27,60340,0,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,FP-GTC,14,62.02,10000056,1,Ebay
-2013-07-29,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,15.85,10000057,1,Ebay
-2013-11-17,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,9.26,10000058,1,Ebay
-2013-07-27,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,16.71,10000059,1,Ebay
-2013-07-29,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,20.6,10000060,1,Ebay
-2013-01-09,63861,3,ClotheShoes & Accessories,Women's Clothing,Dresses,Auction,5,1.88,10000061,1,Ebay
-2013-06-11,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,5,141.7,10000062,1,Ebay
-2013-01-10,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,5,1.88,10000063,1,Ebay
-2013-09-16,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Others,11,112.56,10000064,1,Ebay
-2013-01-14,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Others,11,94.45,10000065,1,Ebay
-2013-05-17,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Auction,14,78.48,10000066,1,Ebay
-2013-06-05,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,13,5.48,10000067,1,Ebay
-2013-05-24,63864,3,ClotheShoes & Accessories,Women's Clothing,Skirts,Auction,14,28.23,10000068,1,Ebay
-2013-05-15,63889,0,ClothinShoes & Accessories,Women's Shoes,Mixed Items & Lots,Others,13,3.49,10000069,1,Ebay
-2013-03-25,67698,2,Business & Industrial,Retail & Services,Jewellery Packaging & Display,FP-GTC,11,15.65,10000070,1,Ebay
-2013-03-09,67698,0,Business & Industrial,Retail & Services,Jewelry Packaging & Display,FP-GTC,11,5.48,10000071,1,Ebay
-2013-12-05,67698,0,Business & Industrial,Retail & Services,Jewelry Packaging & Display,FP-GTC,11,246,10000072,1,Ebay
-2013-04-18,73506,0,Collectibles,Decorative Collectibles,Tea PotSets,FP-non GTC,13,122.78,10000073,1,Ebay
-2013-11-01,75665,0,Home & Garden,YarGarden & Outdoor Living,Gardening Supplies,FP-GTC,14,223.63,10000074,1,Ebay
-2013-05-03,75708,3,Toys & Games,Action Figures,TMovies & Video Games,ABIN,5,141.7,10000075,1,Ebay
-2013-04-21,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,21.14,10000076,1,Ebay
-2013-03-12,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,55.89,10000077,1,Ebay
-2013-05-19,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,51.23,10000078,1,Ebay
-2013-11-23,80135,0,Computers/Tablets & Networking,DriveStorage & Blank Media,Blank Media & Accessories,Auction,14,21.72,10000079,1,Ebay
-2013-10-19,95672,3,ClotheShoes & Accessories,Women's Shoes,Trainers,Auction,14,204.28,10000080,1,Ebay
-2013-05-18,95672,0,ClothinShoes & Accessories,Women's Shoes,Athletic,Others,11,21.14,10000081,1,Ebay
-2013-02-01,100847,0,Half Books,Half Books,Half Books,Others,5,204.28,10000082,1,Ebay
-2013-01-14,100847,0,Half Books,Half Books,Half Books,Others,5,122.78,10000083,1,Ebay
-2013-08-05,139973,3,Video Games & Consoles,Games,Games,ABIN,14,94.45,10000084,1,Ebay
-2013-05-19,139973,0,Video Games & Consoles,Video Games,Video Games,ABIN,11,86.58,10000085,1,Ebay
-2013-12-01,150047,3,Crafts,Jewellery Making,Findings,Auction,14,56.36,10000086,1,Ebay
-2013-12-02,150047,3,Crafts,Jewellery Making,Findings,Auction,14,290.72,10000087,1,Ebay
-2013-01-11,155226,0,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,FP-GTC,13,60.37,10000088,1,Ebay
-2013-05-27,155226,0,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,FP-GTC,13,112.56,10000089,1,Ebay
-2013-09-01,156356,0,Collectibles,Postcards,BuildingArchitecture,FP-GTC,13,265.56,10000090,1,Ebay
-2013-04-11,158798,0,Toys & Hobbies,Vintage & Antique Toys,Spinning Tops,FP-GTC,11,35.72,10000091,1,Ebay
-2013-05-05,165888,0,Jewelry & Watches,Vintage & Antique Jewelry,Costume,FP-non GTC,13,92.98,10000092,1,Ebay
-2013-11-21,170083,3,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),Auction,11,28.23,10000093,1,Ebay
-2013-10-07,170083,3,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),Auction,11,27.48,10000094,1,Ebay
-2013-07-12,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,9.26,10000095,1,Ebay
-2013-06-07,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,3.18,10000096,1,Ebay
-2013-05-22,175750,0,Home & Garden,Bedding,Blankets & Throws,FP-GTC,14,12.04,10000097,1,Ebay
-2013-11-28,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,13,20.6,10000098,1,Ebay
-2013-07-12,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,13,12.04,10000099,1,Ebay
-2013-06-07,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,4.13,10000100,1,Ebay
-2013-12-01,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,73.26,10000201,1,Ebay
-2012-10-12,48027,0,Coins & Paper Money,Paper Money: World,Asia,Auction,12,184.21,10000001,1,Ebay
-2012-08-20,164261,0,Jewelry & Watches,Fashion Jewelry,Earrings,Others,5,172.03,10000002,1,Ebay
-2012-09-18,82494,15,BookMagazines,NULL,Comic Books,Auction,14,66.6,10000003,1,Ebay
-2012-06-30,66767,15,Home & Garden,NULL,Dogs,Auction,12,92.98,10000004,1,Ebay
-2012-08-29,152801,0,Jewelry & Watches,NULL,Earrings,FP-GTC,5,132.33,10000005,1,Ebay
-2012-01-06,43398,0,Home & Garden,NULL,Cheese & Crackers,FP-GTC,13,7.12,10000006,1,Ebay
-2012-08-14,95173,0,Health & Beauty,Bath & Body,Bath Sets & Kits,Auction,14,204.28,10000007,1,Ebay
-2012-07-17,158666,15,ToyHobbies,Action Figures,Anime & Manga,Auction,13,35.72,10000008,1,Ebay
-2012-07-23,12688,0,eBay Premier,Books & Manuscripts,Books: Other,Auction,12,4.13,10000009,1,Ebay
-2012-07-27,103324,15,ClothinShoeAccessories,Women's Shoes,Mixed Items,FP-GTC,5,27.48,10000010,1,Ebay
-2012-07-29,108782,15,Vehicle Parts & Accessories,CaTruck Parts,Car Care & Cleaning,FP-GTC,14,9.26,10000011,1,Ebay
-2012-11-17,80287,0,Computers/Tablets & Networking,Software,Office & Business,Auction,12,3.18,10000012,1,Ebay
-2012-10-29,140746,100,eBay Motors,Parts & Accessories,Vintage Car & Truck Parts,Others,13,3.18,10000013,1,Ebay
-2012-03-28,87118,0,Sporting Goods,Outdoor Sports,Paintball,ABIN,14,377.94,10000014,1,Ebay
-2012-01-30,25147,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Baseball-MLB,Auction,12,146.33,10000015,1,Ebay
-2012-10-07,170302,15,Crafts,Embroidery,Design CDs,FP-GTC,5,51.23,10000016,1,Ebay
-2012-11-28,53064,0,Business & Industrial,Heavy Equipment,Antique & Vintage Farm Equip,FP-non GTC,13,72.65,10000017,1,Ebay
-2012-07-12,132939,0,Jewelry & Watches,Fashion Jewelry,Other,Auction,13,66.6,10000018,1,Ebay
-2012-06-07,113593,15,Phones,Mobile Phones,Mobile Phones,Auction,12,9.26,10000019,1,Ebay
-2012-12-01,34273,100,eBay Motors,Parts & Accessories,Motorcycle,Auction,14,583.44,10000020,1,Ebay
-2012-12-02,106340,15,Home & Garden,Gardening,Hand Tools,FP-GTC,14,638.72,10000021,1,Ebay
-2012-02-01,150265,15,Baby,Baby Clothing,Boys,FP-GTC,14,4.54,10000022,1,Ebay
-2012-01-14,24760,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Hockey-NHL,FP-GTC,12,319.79,10000023,1,Ebay
-2012-08-14,37831,0,Collectibles,Advertising,Merchandise & Memorabilia,Auction,12,20.35,10000024,1,Ebay
-2012-10-04,1120,3,Books,First Editions,Other,FP-non GTC,5,223.63,10000025,1,Ebay
-2012-12-26,43972,100,eBay Motors,Parts & Accessories,ATV Parts,FP-GTC,13,204.28,10000026,1,Ebay
-2012-12-31,166013,15,Computers,Computer Components & Parts,Video Capture & TV Tuner Cards,Auction,14,5.48,10000027,1,Ebay
-2012-03-22,15568,15,Baby,Baby Clothing,Unisex,Auction,14,27.48,10000028,1,Ebay
-2012-12-25,103178,15,ClothinShoeAccessories,Women's Bags,Women's Bags,FP-GTC,5,21.72,10000029,1,Ebay
-2012-12-05,2023,0,Sporting Goods,Team Sports,Basketball,ABIN,12,3.18,10000030,1,Ebay
-2012-05-27,94847,0,Consumer Electronics,Vehicle Electronics & GPS,Car Video,FP-GTC,11,491.32,10000031,1,Ebay
-2012-01-11,15868,0,Real Estate,Land,Land,Auction,14,448.8,10000032,1,Ebay
-2012-02-04,32876,0,Home & Garden,Home Improvement,Plumbing & Fixtures,Auction,13,415.73,10000033,1,Ebay
-2012-12-15,62179,0,ClothinShoes & Accessories,Women's Clothing,Athletic Apparel,Auction,13,377.94,10000034,1,Ebay
-2012-06-30,33038,15,Musical Instruments,Instruments,Guitars (Electric),FP-GTC,14,146.33,10000035,1,Ebay
-2012-12-16,156614,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,FP-GTC,5,7.12,10000036,1,Ebay
-2012-10-17,106246,0,Health & Beauty,Hair Care & Styling,Shampoo & Conditioning,Auction,13,42.99,10000037,1,Ebay
-2012-10-08,20865,0,ClothinShoes & Accessories,Men's Clothing,Athletic Apparel,Auction,13,12.85,10000038,1,Ebay
-2012-04-26,15115,0,Video Games & Consoles,Video Games,Video Games,FP-GTC,13,55.89,10000039,1,Ebay
-2012-10-06,3838,0,Jewelry & Watches,Fashion Jewelry,Charms & Charm Bracelets,FP-GTC,14,73.26,10000040,1,Ebay
-2012-12-28,759,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,Auction,11,112.56,10000041,1,Ebay
-2012-11-06,61323,0,Consumer Electronics,TVideo & Home Audio,TVideo & Audio Accessories,FP-non GTC,11,3.49,10000042,1,Ebay
-2012-11-06,121153,0,Baby,Nursery Decor,Night Lights,Auction,13,184.21,10000043,1,Ebay
-2012-12-27,88750,0,Consumer Electronics,Vehicle Electronics & GPS,Radar & Laser Detectors,Auction,13,157.14,10000044,1,Ebay
-2012-07-15,161567,15,Computers,Laptop & Desktop Accessories,Laptop Batteries,FP-GTC,14,72.65,10000045,1,Ebay
-2012-08-23,113802,15,Lots More...,Metaphysical,Herbs,FP-GTC,14,51.23,10000046,1,Ebay
-2012-07-10,15808,15,ClothinShoeAccessories,Women's Clothing,Tops & Blouses,FP-non GTC,14,15.85,10000047,1,Ebay
-2012-08-10,174053,3,Vehicle Parts & Accessories,Car Parts,External & Body Parts,FP-GTC,13,7.12,10000048,1,Ebay
-2012-09-01,2635,0,Toys & Hobbies,Toy Soldiers,1970-Now,Auction,14,12.04,10000049,1,Ebay
-2012-09-19,1161,3,DVFilm & TV,Other Formats,Videos: NTSC (US),Auction,13,73.26,10000050,1,Ebay
-2012-08-10,64076,0,Computers/Tablets & Networking,Enterprise NetworkinServers,Switches & Hubs,FP-non GTC,5,184.21,10000051,1,Ebay
-2012-08-21,33977,15,Crafts,Scrapbooking,Albums,FP-GTC,13,172.03,10000052,1,Ebay
-2012-08-29,31673,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Racing-NASCAR,FP-GTC,14,122.78,10000053,1,Ebay
-2012-04-22,174106,3,Vehicle Parts & Accessories,Car Parts,Transmission & Drivetrain,Auction,14,92.98,10000054,1,Ebay
-2012-07-02,26249,0,Business & Industrial,Printing & Graphic Arts,Commercial Printing Presses,Auction,13,12.19,10000055,1,Ebay
-2012-01-28,159184,0,Sporting Goods,Winter Sports,Snowboarding,FP-GTC,5,15.65,10000056,1,Ebay
-2012-03-11,10058,3,Events Tickets,Other Tickets,Other Tickets,FP-GTC,11,101.79,10000057,1,Ebay
-2012-05-29,48904,0,ClothinShoes & Accessories,Vintage,Women's Vintage Clothing,ABIN,12,7.12,10000058,1,Ebay
-2012-05-31,145970,0,Toys & Hobbies,Models & Kits,Automotive,FP-non GTC,14,12.85,10000059,1,Ebay
-2012-10-25,963,0,ClothinShoes & Accessories,Vintage,Women's Vintage Shoes,FP-GTC,13,12.19,10000060,1,Ebay
-2012-11-23,118687,3,Health & Beauty,Fragrances,Women's Fragrances,FP-GTC,13,92.98,10000061,1,Ebay
-2012-09-28,20886,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,FP-GTC,14,42.99,10000062,1,Ebay
-2012-09-28,148324,15,Phones,Mobile Accessories,CaseCoverSkins,Auction,13,1.88,10000063,1,Ebay
-2012-10-19,139255,15,Jewellery & Watches,Fine Jewellery,Earrings,Auction,14,21.14,10000064,1,Ebay
-2012-08-05,20213,0,Collectibles,Postcards,US StateCities & Towns,FP-GTC,5,21.14,10000065,1,Ebay
-2012-05-19,32996,15,Movies,Television Memorabilia,Clippings,Auction,13,132.33,10000066,1,Ebay
-2012-04-11,99985,0,Collectibles,Trading Cards,Sci-FFantasy,FP-GTC,14,120.87,10000067,1,Ebay
-2012-05-16,67703,3,Jewellery & Watches,Jewellery Boxes & Supplies,Jewellery Display,Auction,14,120.87,10000068,1,Ebay
-2012-05-24,65,0,Collectibles,Comics,Platinum Age (1897-1937),FP-non GTC,11,9.26,10000069,1,Ebay
-2012-01-10,130,0,Collectibles,Transportation,Railroadiana & Trains,FP-non GTC,14,16.26,10000070,1,Ebay
-2012-05-17,164,0,Computers/Tablets & Networking,Computer Components & Parts,CPUProcessors,FP-GTC,14,157.14,10000071,1,Ebay
-2012-02-03,216,0,Sports MeCards & Fan Shop,Cards,Hockey,FP-GTC,11,1.88,10000072,1,Ebay
-2012-05-05,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,12.04,10000073,1,Ebay
-2012-11-26,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,14,189.23,10000074,1,Ebay
-2012-08-30,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,73.26,10000075,1,Ebay
-2012-05-21,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,4.13,10000076,1,Ebay
-2012-06-06,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,290.72,10000077,1,Ebay
-2012-06-16,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,265.56,10000078,1,Ebay
-2012-05-15,279,15,BookMagazines,Children's Books,Children's Books,FP-GTC,5,5.91,10000079,1,Ebay
-2012-12-05,314,0,ClothinShoes & Accessories,Women's Clothing,Other,Auction,5,319.79,10000080,1,Ebay
-2012-03-25,314,211,ClothinShoes & Accessories,Womens' Clothing,Other,Auction,5,246,10000081,1,Ebay
-2012-03-09,314,211,ClothinShoes & Accessories,Womens' Clothing,Other,Auction,5,20.35,10000082,1,Ebay
-2012-05-06,314,0,ClothinShoes & Accessories,Women's Clothing,Other,Auction,5,36.7,10000083,1,Ebay
-2012-06-15,533,0,Coins & Paper Money,Coins: World,Africa,Auction,13,101.79,10000084,1,Ebay
-2012-03-14,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,5,47.71,10000085,1,Ebay
-2012-05-20,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,5,3.49,10000086,1,Ebay
-2012-05-17,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,13,46.44,10000087,1,Ebay
-2012-03-11,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,13,4.54,10000088,1,Ebay
-2012-01-10,1357,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,14,3.18,10000089,1,Ebay
-2012-04-13,1504,0,Business & Industrial,Electrical & Test Equipment,Test Equipment,FP-GTC,14,86.58,10000090,1,Ebay
-2012-07-22,4943,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-GTC,13,12.85,10000091,1,Ebay
-2012-06-15,6762,0,Unknown,Unknown,Unknown,ABIN,13,16.26,10000092,1,Ebay
-2012-06-15,9426,3,Mobile Phones & Communication,Home Phones & Accessories,Phone Accessories,Auction,13,21.14,10000093,1,Ebay
-2012-06-01,10866,0,Collectibles,Animals,Farm & Countryside,FP-non GTC,14,20.6,10000094,1,Ebay
-2012-05-17,11554,0,ClothinShoes & Accessories,Women's Clothing,Jeans,Auction,13,246,10000095,1,Ebay
-2012-04-26,11848,0,Health & Beauty,Fragrances,Women,FP-GTC,14,109,10000096,1,Ebay
-2012-07-28,13836,0,Collectibles,Decorative Collectibles,Spoons,Auction,13,39.41,10000097,1,Ebay
-2012-03-12,13836,0,Collectibles,Decorative Collectibles,Spoons,Auction,14,16.26,10000098,1,Ebay
-2012-04-20,13987,0,Collectibles,Paper,Booklets,FP-GTC,13,112.56,10000099,1,Ebay
-2012-03-18,15687,0,ClothinShoes & Accessories,Men's Clothing,T-Shirts,Auction,14,184.21,10000100,1,Ebay
-2012-05-17,15687,0,ClothinShoes & Accessories,Men's Clothing,T-Shirts,Auction,11,27.48,10000001,1,Ebay
-2012-06-23,16145,3,Computers/Tablets & Networking,Computer Components & Parts,Other Components & Parts,FP-non GTC,12,26.45,10000002,1,Ebay
-2012-05-22,16145,0,Computers/Tablets & Networking,Computer Components & Parts,Other,FP-non GTC,13,415.73,10000003,1,Ebay
-2012-01-25,16509,0,Toys & Hobbies,Model Railroads & Trains,S Scale,ABIN,5,56.36,10000004,1,Ebay
-2012-06-12,16509,0,Toys & Hobbies,Model Railroads & Trains,S Scale,ABIN,5,2.44,10000005,1,Ebay
-2012-05-17,20485,0,Home & Garden,Furniture,Other,FP-GTC,14,269.76,10000006,1,Ebay
-2012-08-03,20485,101,CasArredamento e Bricolage,Cucina,Altro per cucina,FP-GTC,12,109,10000007,1,Ebay
-2012-05-17,20485,101,CasArredamento e Bricolage,Cucina,Altro per cucina,FP-GTC,12,101.79,10000008,1,Ebay
-2012-08-21,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,246,10000009,1,Ebay
-2012-08-21,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,189.23,10000010,1,Ebay
-2012-04-18,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,15.65,10000011,1,Ebay
-2012-06-16,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,28.23,10000012,1,Ebay
-2012-01-09,24541,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,College-NCAA,FP-GTC,5,16.26,10000013,1,Ebay
-2012-05-03,26262,0,Collectibles,Advertising,Food & Beverage,FP-GTC,5,122.78,10000014,1,Ebay
-2012-05-20,30059,3,Cameras & Photography,Lenses & Filters,Lens AdapterMounts & Tubes,FP-GTC,14,172.03,10000015,1,Ebay
-2012-03-12,31387,3,Jewellery & Watches,Watches,Wristwatches,Auction,14,42.99,10000016,1,Ebay
-2012-05-19,31387,3,Jewellery & Watches,Watches,Wristwatches,Auction,14,207.5,10000017,1,Ebay
-2012-04-21,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,5.91,10000018,1,Ebay
-2012-05-18,31519,3,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,39.41,10000019,1,Ebay
-2012-06-16,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,16.26,10000020,1,Ebay
-2012-06-11,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,16.26,10000021,1,Ebay
-2012-03-31,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,78.48,10000022,1,Ebay
-2012-11-12,31519,3,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,190.22,10000023,1,Ebay
-2012-04-06,35570,100,eBay Motors,Parts & Accessories,Motorcycle Parts,FP-GTC,12,2.44,10000024,1,Ebay
-2012-11-01,36250,0,Sporting Goods,Hunting,Decoys,Auction,5,7.12,10000025,1,Ebay
-2012-02-06,38238,0,Home & Garden,Home Decor,Other,FP-non GTC,14,36.7,10000026,1,Ebay
-2012-06-16,40059,3,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,FP-GTC,14,35.72,10000027,1,Ebay
-2012-09-12,40059,3,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,FP-GTC,14,3.49,10000028,1,Ebay
-2012-04-16,41940,0,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,FP-GTC,13,223.63,10000029,1,Ebay
-2012-11-01,41940,0,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,FP-GTC,13,265.56,10000030,1,Ebay
-2012-06-14,43479,0,Cameras & Photo,Film Photography,Other,FP-non GTC,13,62.02,10000031,1,Ebay
-2012-11-12,44079,0,Sporting Goods,Exercise & Fitness,GyWorkout & Yoga,FP-GTC,12,46.44,10000032,1,Ebay
-2012-03-22,45238,101,Abbigliamento e accessori,Donna: Accessori,SciarpFoulard e Scialli,Auction,14,132.33,10000033,1,Ebay
-2012-05-22,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,Auction,13,448.8,10000034,1,Ebay
-2012-03-10,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,FP-non GTC,14,207.5,10000035,1,Ebay
-2012-05-17,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,FP-non GTC,14,190.22,10000036,1,Ebay
-2012-09-01,46575,0,Business & Industrial,Light Equipment & Tools,Air Tools,FP-GTC,14,16.71,10000037,1,Ebay
-2012-04-18,50508,0,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,FP-non GTC,13,4.13,10000038,1,Ebay
-2012-06-05,50508,0,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,FP-non GTC,13,1.88,10000039,1,Ebay
-2012-10-08,50677,0,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,FP-GTC,13,491.32,10000040,1,Ebay
-2012-09-17,50677,0,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,FP-GTC,5,2.44,10000041,1,Ebay
-2012-02-02,51582,0,ClothinShoes & Accessories,Kids' ClothinShoes & Accs,Girls' Clothing (Sizes 4 & Up),Auction,14,56.36,10000042,1,Ebay
-2012-08-23,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-GTC,13,15.85,10000043,1,Ebay
-2012-08-17,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-non GTC,14,2.44,10000044,1,Ebay
-2012-05-05,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-GTC,14,7.12,10000045,1,Ebay
-2012-03-08,57784,0,ClothinShoes & Accessories,Baby & Toddler Clothing,Boys' Clothing (Newborn-5T),Auction,14,35.72,10000046,1,Ebay
-2012-03-16,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,11,9.26,10000047,1,Ebay
-2012-05-09,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,3.18,10000048,1,Ebay
-2012-03-25,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,638.72,10000049,1,Ebay
-2012-06-05,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,141.7,10000050,1,Ebay
-2012-01-10,57990,0,ClothinShoes & Accessories,Men's Clothing,Casual Shirts,ABIN,13,12.19,10000051,1,Ebay
-2012-06-11,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,132.33,10000052,1,Ebay
-2012-08-23,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,ABIN,5,5.48,10000053,1,Ebay
-2012-08-20,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,11,1.88,10000054,1,Ebay
-2012-08-09,60340,0,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,FP-GTC,14,12.85,10000055,1,Ebay
-2012-06-30,60340,0,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,FP-GTC,14,62.02,10000056,1,Ebay
-2012-06-03,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,15.85,10000057,1,Ebay
-2012-08-14,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,9.26,10000058,1,Ebay
-2012-08-14,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,16.71,10000059,1,Ebay
-2012-05-17,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,20.6,10000060,1,Ebay
-2012-07-23,63861,3,ClotheShoes & Accessories,Women's Clothing,Dresses,Auction,5,1.88,10000061,1,Ebay
-2012-07-27,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,5,141.7,10000062,1,Ebay
-2012-07-29,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,5,1.88,10000063,1,Ebay
-2012-11-17,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Others,11,112.56,10000064,1,Ebay
-2012-10-29,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Others,11,94.45,10000065,1,Ebay
-2012-03-28,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Auction,14,78.48,10000066,1,Ebay
-2012-01-30,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,13,5.48,10000067,1,Ebay
-2012-01-26,63864,3,ClotheShoes & Accessories,Women's Clothing,Skirts,Auction,14,28.23,10000068,1,Ebay
-2012-11-21,63889,0,ClothinShoes & Accessories,Women's Shoes,Mixed Items & Lots,Others,13,3.49,10000069,1,Ebay
-2012-07-12,67698,2,Business & Industrial,Retail & Services,Jewellery Packaging & Display,FP-GTC,11,15.65,10000070,1,Ebay
-2012-06-07,67698,0,Business & Industrial,Retail & Services,Jewelry Packaging & Display,FP-GTC,11,5.48,10000071,1,Ebay
-2012-12-01,67698,0,Business & Industrial,Retail & Services,Jewelry Packaging & Display,FP-GTC,11,246,10000072,1,Ebay
-2012-12-02,73506,0,Collectibles,Decorative Collectibles,Tea PotSets,FP-non GTC,13,122.78,10000073,1,Ebay
-2012-02-01,75665,0,Home & Garden,YarGarden & Outdoor Living,Gardening Supplies,FP-GTC,14,223.63,10000074,1,Ebay
-2012-01-14,75708,3,Toys & Games,Action Figures,TMovies & Video Games,ABIN,5,141.7,10000075,1,Ebay
-2012-09-16,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,21.14,10000076,1,Ebay
-2012-08-09,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,55.89,10000077,1,Ebay
-2012-12-26,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,51.23,10000078,1,Ebay
-2012-12-31,80135,0,Computers/Tablets & Networking,DriveStorage & Blank Media,Blank Media & Accessories,Auction,14,21.72,10000079,1,Ebay
-2012-03-22,95672,3,ClotheShoes & Accessories,Women's Shoes,Trainers,Auction,14,204.28,10000080,1,Ebay
-2012-12-25,95672,0,ClothinShoes & Accessories,Women's Shoes,Athletic,Others,11,21.14,10000081,1,Ebay
-2012-01-15,100847,0,Half Books,Half Books,Half Books,Others,5,204.28,10000082,1,Ebay
-2012-05-27,100847,0,Half Books,Half Books,Half Books,Others,5,122.78,10000083,1,Ebay
-2012-01-11,139973,3,Video Games & Consoles,Games,Games,ABIN,14,94.45,10000084,1,Ebay
-2012-02-04,139973,0,Video Games & Consoles,Video Games,Video Games,ABIN,11,86.58,10000085,1,Ebay
-2012-12-15,150047,3,Crafts,Jewellery Making,Findings,Auction,14,56.36,10000086,1,Ebay
-2012-06-30,150047,3,Crafts,Jewellery Making,Findings,Auction,14,290.72,10000087,1,Ebay
-2012-12-16,155226,0,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,FP-GTC,13,60.37,10000088,1,Ebay
-2012-11-11,155226,0,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,FP-GTC,13,112.56,10000089,1,Ebay
-2012-10-08,156356,0,Collectibles,Postcards,BuildingArchitecture,FP-GTC,13,265.56,10000090,1,Ebay
-2012-04-26,158798,0,Toys & Hobbies,Vintage & Antique Toys,Spinning Tops,FP-GTC,11,35.72,10000091,1,Ebay
-2012-10-06,165888,0,Jewelry & Watches,Vintage & Antique Jewelry,Costume,FP-non GTC,13,92.98,10000092,1,Ebay
-2012-12-28,170083,3,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),Auction,11,28.23,10000093,1,Ebay
-2012-11-06,170083,3,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),Auction,11,27.48,10000094,1,Ebay
-2012-11-06,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,9.26,10000095,1,Ebay
-2012-12-27,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,3.18,10000096,1,Ebay
-2012-01-01,175750,0,Home & Garden,Bedding,Blankets & Throws,FP-GTC,14,12.04,10000097,1,Ebay
-2012-08-23,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,13,20.6,10000098,1,Ebay
-2012-07-10,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,13,12.04,10000099,1,Ebay
-2012-08-10,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,4.13,10000100,1,Ebay
-2012-07-19,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,73.26,10000201,1,Ebay
+2013-03-31,48028,0,\N,\N,\N,Auction,12,184.21,10000001,1,eef
+2013-11-12,164262,0,\N,\N,\N,Others,5,172.03,10000002,1,gji
+2013-04-06,82494,15,BookMagazines,NULL,Comic Books,Auction,14,66.6,10000003,1,jjc
+2013-05-17,66767,15,Home & Garden,NULL,Dogs,Auction,12,92.98,10000004,1,add
+2013-05-20,152801,0,Jewelry & Watches,NULL,Earrings,FP-GTC,5,132.33,10000005,1,ife
+2013-06-16,43398,0,Home & Garden,NULL,Cheese & Crackers,FP-GTC,13,7.12,10000006,1,hce
+2013-06-14,95173,0,Health & Beauty,Bath & Body,Bath Sets & Kits,Auction,14,204.28,10000007,1,bei
+2013-03-22,158666,15,ToyHobbies,Action Figures,Anime & Manga,Auction,13,35.72,10000008,1,bjb
+2013-03-10,12688,0,eBay Premier,Books & Manuscripts,Books: Other,Auction,12,4.13,10000009,1,daf
+2013-11-01,103324,15,ClothinShoeAccessories,Women's Shoes,Mixed Items,FP-GTC,5,27.48,10000010,1,cji
+2013-06-16,108782,15,Vehicle Parts & Accessories,CaTruck Parts,Car Care & Cleaning,FP-GTC,14,9.26,10000011,1,hch
+2013-09-12,80287,0,Computers/Tablets & Networking,Software,Office & Business,Auction,12,3.18,10000012,1,edc
+2013-09-28,140746,100,eBay Motors,Parts & Accessories,Vintage Car & Truck Parts,Others,13,3.18,10000013,1,jhi
+2013-06-15,87118,0,Sporting Goods,Outdoor Sports,Paintball,ABIN,14,377.94,10000014,1,age
+2013-03-14,25147,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Baseball-MLB,Auction,12,146.33,10000015,1,afc
+2013-09-01,170302,15,Crafts,Embroidery,Design CDs,FP-GTC,5,51.23,10000016,1,jib
+2013-05-29,53064,0,Business & Industrial,Heavy Equipment,Antique & Vintage Farm Equip,FP-non GTC,13,72.65,10000017,1,bai
+2013-05-31,132939,0,Jewelry & Watches,Fashion Jewelry,Other,Auction,13,66.6,10000018,1,gii
+2013-03-18,113593,15,Phones,Mobile Phones,Mobile Phones,Auction,12,9.26,10000019,1,fcj
+2013-07-19,34273,100,eBay Motors,Parts & Accessories,Motorcycle,Auction,14,583.44,10000020,1,ifc
+2013-06-23,106340,15,Home & Garden,Gardening,Hand Tools,FP-GTC,14,638.72,10000021,1,jfe
+2013-05-20,150265,15,Baby,Baby Clothing,Boys,FP-GTC,14,4.54,10000022,1,ehd
+2013-05-17,24760,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Hockey-NHL,FP-GTC,12,319.79,10000023,1,hbc
+2013-03-11,37831,0,Collectibles,Advertising,Merchandise & Memorabilia,Auction,12,20.35,10000024,1,dbh
+2013-01-30,1120,3,Books,First Editions,Other,FP-non GTC,5,223.63,10000025,1,igh
+2013-01-26,43972,100,eBay Motors,Parts & Accessories,ATV Parts,FP-GTC,13,204.28,10000026,1,jbe
+2013-03-22,166013,15,Computers,Computer Components & Parts,Video Capture & TV Tuner Cards,Auction,14,5.48,10000027,1,deb
+2013-07-23,15568,15,Baby,Baby Clothing,Unisex,Auction,14,27.48,10000028,1,hai
+2013-07-27,103178,15,ClothinShoeAccessories,Women's Bags,Women's Bags,FP-GTC,5,21.72,10000029,1,cfj
+2013-10-29,2023,0,Sporting Goods,Team Sports,Basketball,ABIN,12,3.18,10000030,1,abc
+2013-10-08,94847,0,Consumer Electronics,Vehicle Electronics & GPS,Car Video,FP-GTC,11,491.32,10000031,1,eha
+2013-04-26,15868,0,Real Estate,Land,Land,Auction,14,448.8,10000032,1,fig
+2013-01-01,32876,0,Home & Garden,Home Improvement,Plumbing & Fixtures,Auction,13,415.73,10000033,1,ghj
+2013-01-15,62179,0,ClothinShoes & Accessories,Women's Clothing,Athletic Apparel,Auction,13,377.94,10000034,1,edj
+2013-05-27,33038,15,Musical Instruments,Instruments,Guitars (Electric),FP-GTC,14,146.33,10000035,1,jii
+2013-11-11,156614,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,FP-GTC,5,7.12,10000036,1,jbi
+2013-03-08,106246,0,Health & Beauty,Hair Care & Styling,Shampoo & Conditioning,Auction,13,42.99,10000037,1,hce
+2013-03-25,20865,0,ClothinShoes & Accessories,Men's Clothing,Athletic Apparel,Auction,13,12.85,10000038,1,jdc
+2013-08-20,15115,0,Video Games & Consoles,Video Games,Video Games,FP-GTC,13,55.89,10000039,1,ggc
+2013-05-17,3838,0,Jewelry & Watches,Fashion Jewelry,Charms & Charm Bracelets,FP-GTC,14,73.26,10000040,1,bca
+2013-06-05,759,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,Auction,11,112.56,10000041,1,hbh
+2013-10-08,61323,0,Consumer Electronics,TVideo & Home Audio,TVideo & Audio Accessories,FP-non GTC,11,3.49,10000042,1,hif
+2013-08-14,121153,0,Baby,Nursery Decor,Night Lights,Auction,13,184.21,10000043,1,hhd
+2013-08-14,88750,0,Consumer Electronics,Vehicle Electronics & GPS,Radar & Laser Detectors,Auction,13,157.14,10000044,1,eif
+2013-05-17,161567,15,Computers,Laptop & Desktop Accessories,Laptop Batteries,FP-GTC,14,72.65,10000045,1,jjg
+2013-08-09,113802,15,Lots More...,Metaphysical,Herbs,FP-GTC,14,51.23,10000046,1,igd
+2013-06-30,15808,15,ClothinShoeAccessories,Women's Clothing,Tops & Blouses,FP-non GTC,14,15.85,10000047,1,fii
+2013-06-03,174053,3,Vehicle Parts & Accessories,Car Parts,External & Body Parts,FP-GTC,13,7.12,10000048,1,bdj
+2013-12-31,2635,0,Toys & Hobbies,Toy Soldiers,1970-Now,Auction,14,12.04,10000049,1,hfi
+2013-12-25,1161,3,DVFilm & TV,Other Formats,Videos: NTSC (US),Auction,13,73.26,10000050,1,gaf
+2013-03-28,64076,0,Computers/Tablets & Networking,Enterprise NetworkinServers,Switches & Hubs,FP-non GTC,5,184.21,10000051,1,agf
+2013-01-30,33977,15,Crafts,Scrapbooking,Albums,FP-GTC,13,172.03,10000052,1,hic
+2013-12-05,31673,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Racing-NASCAR,FP-GTC,14,122.78,10000053,1,fae
+2013-10-08,174106,3,Vehicle Parts & Accessories,Car Parts,Transmission & Drivetrain,Auction,14,92.98,10000054,1,ica
+2013-12-27,26249,0,Business & Industrial,Printing & Graphic Arts,Commercial Printing Presses,Auction,13,12.19,10000055,1,jfa
+2013-12-16,159184,0,Sporting Goods,Winter Sports,Snowboarding,FP-GTC,5,15.65,10000056,1,hca
+2013-10-17,10058,3,Events Tickets,Other Tickets,Other Tickets,FP-GTC,11,101.79,10000057,1,ijc
+2013-11-17,48904,0,ClothinShoes & Accessories,Vintage,Women's Vintage Clothing,ABIN,12,7.12,10000058,1,jfd
+2013-09-18,145970,0,Toys & Hobbies,Models & Kits,Automotive,FP-non GTC,14,12.85,10000059,1,iab
+2013-06-30,963,0,ClothinShoes & Accessories,Vintage,Women's Vintage Shoes,FP-GTC,13,12.19,10000060,1,abb
+2013-10-12,118687,3,Health & Beauty,Fragrances,Women's Fragrances,FP-GTC,13,92.98,10000061,1,gja
+2013-08-20,20886,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,FP-GTC,14,42.99,10000062,1,iba
+2013-08-29,148324,15,Phones,Mobile Accessories,CaseCoverSkins,Auction,13,1.88,10000063,1,deh
+2013-07-17,139255,15,Jewellery & Watches,Fine Jewellery,Earrings,Auction,14,21.14,10000064,1,afb
+2013-07-23,20213,0,Collectibles,Postcards,US StateCities & Towns,FP-GTC,5,21.14,10000065,1,ijb
+2013-01-06,32996,15,Movies,Television Memorabilia,Clippings,Auction,13,132.33,10000066,1,ddi
+2013-08-14,99985,0,Collectibles,Trading Cards,Sci-FFantasy,FP-GTC,14,120.87,10000067,1,aej
+2013-08-10,67703,3,Jewellery & Watches,Jewellery Boxes & Supplies,Jewellery Display,Auction,14,120.87,10000068,1,jif
+2013-09-28,65,0,Collectibles,Comics,Platinum Age (1897-1937),FP-non GTC,11,9.26,10000069,1,fcb
+2013-08-21,130,0,Collectibles,Transportation,Railroadiana & Trains,FP-non GTC,14,16.26,10000070,1,dib
+2013-03-11,164,0,Computers/Tablets & Networking,Computer Components & Parts,CPUProcessors,FP-GTC,14,157.14,10000071,1,ejc
+2013-05-06,216,0,Sports MeCards & Fan Shop,Cards,Hockey,FP-GTC,11,1.88,10000072,1,jhb
+2013-05-17,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,12.04,10000073,1,jgg
+2013-01-10,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,14,189.23,10000074,1,bgb
+2013-05-05,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,73.26,10000075,1,efh
+2013-02-03,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,4.13,10000076,1,jja
+2013-11-26,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,290.72,10000077,1,cbg
+2013-08-30,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,265.56,10000078,1,gef
+2013-04-26,279,15,BookMagazines,Children's Books,Children's Books,FP-GTC,5,5.91,10000079,1,eji
+2013-06-30,314,0,ClothinShoes & Accessories,Women's Clothing,Other,Auction,5,319.79,10000080,1,jhf
+2013-06-30,314,211,ClothinShoes & Accessories,Womens' Clothing,Other,Auction,5,246,10000081,1,hgg
+2013-12-16,314,211,ClothinShoes & Accessories,Womens' Clothing,Other,Auction,5,20.35,10000082,1,jjg
+2013-12-15,314,0,ClothinShoes & Accessories,Women's Clothing,Other,Auction,5,36.7,10000083,1,ejf
+2013-08-17,533,0,Coins & Paper Money,Coins: World,Africa,Auction,13,101.79,10000084,1,cbc
+2013-12-15,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,5,47.71,10000085,1,aeh
+2013-02-04,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,5,3.49,10000086,1,jdb
+2013-01-11,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,13,46.44,10000087,1,eaj
+2013-02-04,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,13,4.54,10000088,1,jjh
+2013-05-17,1357,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,14,3.18,10000089,1,cga
+2013-11-12,1504,0,Business & Industrial,Electrical & Test Equipment,Test Equipment,FP-GTC,14,86.58,10000090,1,jjj
+2013-08-21,4943,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-GTC,13,12.85,10000091,1,egj
+2013-05-09,6762,0,Unknown,Unknown,Unknown,ABIN,13,16.26,10000092,1,ehg
+2013-09-19,9426,3,Mobile Phones & Communication,Home Phones & Accessories,Phone Accessories,Auction,13,21.14,10000093,1,fff
+2013-02-06,10866,0,Collectibles,Animals,Farm & Countryside,FP-non GTC,14,20.6,10000094,1,bha
+2013-02-02,11554,0,ClothinShoes & Accessories,Women's Clothing,Jeans,Auction,13,246,10000095,1,gfa
+2013-08-23,11848,0,Health & Beauty,Fragrances,Women,FP-GTC,14,109,10000096,1,jeb
+2013-08-03,13836,0,Collectibles,Decorative Collectibles,Spoons,Auction,13,39.41,10000097,1,baf
+2013-05-17,13836,0,Collectibles,Decorative Collectibles,Spoons,Auction,14,16.26,10000098,1,adh
+2013-06-06,13987,0,Collectibles,Paper,Booklets,FP-GTC,13,112.56,10000099,1,eii
+2013-07-02,15687,0,ClothinShoes & Accessories,Men's Clothing,T-Shirts,Auction,14,184.21,10000100,1,ijj
+2013-10-25,15687,0,ClothinShoes & Accessories,Men's Clothing,T-Shirts,Auction,11,27.48,10000001,1,jji
+2013-04-20,16145,3,Computers/Tablets & Networking,Computer Components & Parts,Other Components & Parts,FP-non GTC,12,26.45,10000002,1,jfd
+2013-03-12,16145,0,Computers/Tablets & Networking,Computer Components & Parts,Other,FP-non GTC,13,415.73,10000003,1,ibh
+2013-03-28,16509,0,Toys & Hobbies,Model Railroads & Trains,S Scale,ABIN,5,56.36,10000004,1,gaa
+2013-10-29,16509,0,Toys & Hobbies,Model Railroads & Trains,S Scale,ABIN,5,2.44,10000005,1,ece
+2013-05-22,20485,0,Home & Garden,Furniture,Other,FP-GTC,14,269.76,10000006,1,bjj
+2013-01-25,20485,101,CasArredamento e Bricolage,Cucina,Altro per cucina,FP-GTC,12,109,10000007,1,acd
+2013-06-12,20485,101,CasArredamento e Bricolage,Cucina,Altro per cucina,FP-GTC,12,101.79,10000008,1,ade
+2013-12-26,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,246,10000009,1,iai
+2013-12-26,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,189.23,10000010,1,jga
+2013-12-31,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,15.65,10000011,1,fed
+2013-10-04,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,28.23,10000012,1,gfh
+2013-03-16,24541,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,College-NCAA,FP-GTC,5,16.26,10000013,1,bjb
+2013-05-21,26262,0,Collectibles,Advertising,Food & Beverage,FP-GTC,5,122.78,10000014,1,jba
+2013-01-28,30059,3,Cameras & Photography,Lenses & Filters,Lens AdapterMounts & Tubes,FP-GTC,14,172.03,10000015,1,fjb
+2013-04-26,31387,3,Jewellery & Watches,Watches,Wristwatches,Auction,14,42.99,10000016,1,bdg
+2013-10-06,31387,3,Jewellery & Watches,Watches,Wristwatches,Auction,14,207.5,10000017,1,jja
+2013-11-06,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,5.91,10000018,1,ggd
+2013-10-06,31519,3,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,39.41,10000019,1,bdc
+2013-12-28,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,16.26,10000020,1,cid
+2013-11-06,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,16.26,10000021,1,jfg
+2013-11-06,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,78.48,10000022,1,gha
+2013-12-28,31519,3,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,190.22,10000023,1,fdh
+2013-06-11,35570,100,eBay Motors,Parts & Accessories,Motorcycle Parts,FP-GTC,12,2.44,10000024,1,fad
+2013-01-10,36250,0,Sporting Goods,Hunting,Decoys,Auction,5,7.12,10000025,1,hjf
+2013-09-17,38238,0,Home & Garden,Home Decor,Other,FP-non GTC,14,36.7,10000026,1,jdh
+2013-08-14,40059,3,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,FP-GTC,14,35.72,10000027,1,bhh
+2013-08-09,40059,3,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,FP-GTC,14,3.49,10000028,1,gjj
+2013-12-02,41940,0,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,FP-GTC,13,223.63,10000029,1,cjf
+2013-02-01,41940,0,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,FP-GTC,13,265.56,10000030,1,caj
+2013-07-28,43479,0,Cameras & Photo,Film Photography,Other,FP-non GTC,13,62.02,10000031,1,fcd
+2013-06-16,44079,0,Sporting Goods,Exercise & Fitness,GyWorkout & Yoga,FP-GTC,12,46.44,10000032,1,bhg
+2013-08-23,45238,101,Abbigliamento e accessori,Donna: Accessori,SciarpFoulard e Scialli,Auction,14,132.33,10000033,1,ddi
+2013-06-15,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,Auction,13,448.8,10000034,1,jce
+2013-06-15,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,FP-non GTC,14,207.5,10000035,1,daf
+2013-06-01,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,FP-non GTC,14,190.22,10000036,1,gfg
+2013-08-10,46575,0,Business & Industrial,Light Equipment & Tools,Air Tools,FP-GTC,14,16.71,10000037,1,cjj
+2013-03-22,50508,0,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,FP-non GTC,13,4.13,10000038,1,agg
+2013-12-25,50508,0,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,FP-non GTC,13,1.88,10000039,1,bad
+2013-07-22,50677,0,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,FP-GTC,13,491.32,10000040,1,fhb
+2013-04-13,50677,0,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,FP-GTC,5,2.44,10000041,1,cfc
+2013-04-16,51582,0,ClothinShoes & Accessories,Kids' ClothinShoes & Accs,Girls' Clothing (Sizes 4 & Up),Auction,14,56.36,10000042,1,eie
+2013-08-21,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-GTC,13,15.85,10000043,1,cjc
+2013-04-22,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-non GTC,14,2.44,10000044,1,hgc
+2013-08-29,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-GTC,14,7.12,10000045,1,fdd
+2013-05-16,57784,0,ClothinShoes & Accessories,Baby & Toddler Clothing,Boys' Clothing (Newborn-5T),Auction,14,35.72,10000046,1,heg
+2013-08-23,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,11,9.26,10000047,1,cdf
+2013-07-10,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,3.18,10000048,1,adi
+2013-08-10,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,638.72,10000049,1,egh
+2013-08-23,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,141.7,10000050,1,hge
+2013-04-18,57990,0,ClothinShoes & Accessories,Men's Clothing,Casual Shirts,ABIN,13,12.19,10000051,1,egj
+2013-07-10,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,132.33,10000052,1,fea
+2013-06-16,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,ABIN,5,5.48,10000053,1,abf
+2013-07-15,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,11,1.88,10000054,1,ecc
+2013-11-06,60340,0,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,FP-GTC,14,12.85,10000055,1,hda
+2013-12-27,60340,0,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,FP-GTC,14,62.02,10000056,1,cgh
+2013-07-29,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,15.85,10000057,1,gbf
+2013-11-17,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,9.26,10000058,1,bae
+2013-07-27,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,16.71,10000059,1,hag
+2013-07-29,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,20.6,10000060,1,ici
+2013-01-09,63861,3,ClotheShoes & Accessories,Women's Clothing,Dresses,Auction,5,1.88,10000061,1,bfa
+2013-06-11,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,5,141.7,10000062,1,dea
+2013-01-10,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,5,1.88,10000063,1,bce
+2013-09-16,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Others,11,112.56,10000064,1,cfi
+2013-01-14,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Others,11,94.45,10000065,1,ijh
+2013-05-17,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Auction,14,78.48,10000066,1,gag
+2013-06-05,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,13,5.48,10000067,1,bja
+2013-05-24,63864,3,ClotheShoes & Accessories,Women's Clothing,Skirts,Auction,14,28.23,10000068,1,gca
+2013-05-15,63889,0,ClothinShoes & Accessories,Women's Shoes,Mixed Items & Lots,Others,13,3.49,10000069,1,fcf
+2013-03-25,67698,2,Business & Industrial,Retail & Services,Jewellery Packaging & Display,FP-GTC,11,15.65,10000070,1,fae
+2013-03-09,67698,0,Business & Industrial,Retail & Services,Jewelry Packaging & Display,FP-GTC,11,5.48,10000071,1,cci
+2013-12-05,67698,0,Business & Industrial,Retail & Services,Jewelry Packaging & Display,FP-GTC,11,246,10000072,1,efc
+2013-04-18,73506,0,Collectibles,Decorative Collectibles,Tea PotSets,FP-non GTC,13,122.78,10000073,1,cig
+2013-11-01,75665,0,Home & Garden,YarGarden & Outdoor Living,Gardening Supplies,FP-GTC,14,223.63,10000074,1,big
+2013-05-03,75708,3,Toys & Games,Action Figures,TMovies & Video Games,ABIN,5,141.7,10000075,1,bgi
+2013-04-21,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,21.14,10000076,1,bdg
+2013-03-12,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,55.89,10000077,1,ffh
+2013-05-19,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,51.23,10000078,1,chb
+2013-11-23,80135,0,Computers/Tablets & Networking,DriveStorage & Blank Media,Blank Media & Accessories,Auction,14,21.72,10000079,1,dhd
+2013-10-19,95672,3,ClotheShoes & Accessories,Women's Shoes,Trainers,Auction,14,204.28,10000080,1,gid
+2013-05-18,95672,0,ClothinShoes & Accessories,Women's Shoes,Athletic,Others,11,21.14,10000081,1,jij
+2013-02-01,100847,0,Half Books,Half Books,Half Books,Others,5,204.28,10000082,1,ddj
+2013-01-14,100847,0,Half Books,Half Books,Half Books,Others,5,122.78,10000083,1,hab
+2013-08-05,139973,3,Video Games & Consoles,Games,Games,ABIN,14,94.45,10000084,1,fai
+2013-05-19,139973,0,Video Games & Consoles,Video Games,Video Games,ABIN,11,86.58,10000085,1,bei
+2013-12-01,150047,3,Crafts,Jewellery Making,Findings,Auction,14,56.36,10000086,1,jhh
+2013-12-02,150047,3,Crafts,Jewellery Making,Findings,Auction,14,290.72,10000087,1,bdg
+2013-01-11,155226,0,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,FP-GTC,13,60.37,10000088,1,heh
+2013-05-27,155226,0,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,FP-GTC,13,112.56,10000089,1,hja
+2013-09-01,156356,0,Collectibles,Postcards,BuildingArchitecture,FP-GTC,13,265.56,10000090,1,egc
+2013-04-11,158798,0,Toys & Hobbies,Vintage & Antique Toys,Spinning Tops,FP-GTC,11,35.72,10000091,1,dgb
+2013-05-05,165888,0,Jewelry & Watches,Vintage & Antique Jewelry,Costume,FP-non GTC,13,92.98,10000092,1,idc
+2013-11-21,170083,3,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),Auction,11,28.23,10000093,1,bgi
+2013-10-07,170083,3,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),Auction,11,27.48,10000094,1,ihd
+2013-07-12,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,9.26,10000095,1,gci
+2013-06-07,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,3.18,10000096,1,fba
+2013-05-22,175750,0,Home & Garden,Bedding,Blankets & Throws,FP-GTC,14,12.04,10000097,1,adb
+2013-11-28,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,13,20.6,10000098,1,hbg
+2013-07-12,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,13,12.04,10000099,1,ige
+2013-06-07,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,4.13,10000100,1,jjj
+2013-12-01,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,73.26,10000201,1,ijb
+2012-10-12,48027,0,Coins & Paper Money,Paper Money: World,Asia,Auction,12,184.21,10000001,1,ehb
+2012-08-20,164261,0,Jewelry & Watches,Fashion Jewelry,Earrings,Others,5,172.03,10000002,1,cai
+2012-09-18,82494,15,BookMagazines,NULL,Comic Books,Auction,14,66.6,10000003,1,bie
+2012-06-30,66767,15,Home & Garden,NULL,Dogs,Auction,12,92.98,10000004,1,iba
+2012-08-29,152801,0,Jewelry & Watches,NULL,Earrings,FP-GTC,5,132.33,10000005,1,hjf
+2012-01-06,43398,0,Home & Garden,NULL,Cheese & Crackers,FP-GTC,13,7.12,10000006,1,dbg
+2012-08-14,95173,0,Health & Beauty,Bath & Body,Bath Sets & Kits,Auction,14,204.28,10000007,1,fdb
+2012-07-17,158666,15,ToyHobbies,Action Figures,Anime & Manga,Auction,13,35.72,10000008,1,icf
+2012-07-23,12688,0,eBay Premier,Books & Manuscripts,Books: Other,Auction,12,4.13,10000009,1,iag
+2012-07-27,103324,15,ClothinShoeAccessories,Women's Shoes,Mixed Items,FP-GTC,5,27.48,10000010,1,egj
+2012-07-29,108782,15,Vehicle Parts & Accessories,CaTruck Parts,Car Care & Cleaning,FP-GTC,14,9.26,10000011,1,jda
+2012-11-17,80287,0,Computers/Tablets & Networking,Software,Office & Business,Auction,12,3.18,10000012,1,cdc
+2012-10-29,140746,100,eBay Motors,Parts & Accessories,Vintage Car & Truck Parts,Others,13,3.18,10000013,1,efa
+2012-03-28,87118,0,Sporting Goods,Outdoor Sports,Paintball,ABIN,14,377.94,10000014,1,ibd
+2012-01-30,25147,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Baseball-MLB,Auction,12,146.33,10000015,1,jbh
+2012-10-07,170302,15,Crafts,Embroidery,Design CDs,FP-GTC,5,51.23,10000016,1,bjj
+2012-11-28,53064,0,Business & Industrial,Heavy Equipment,Antique & Vintage Farm Equip,FP-non GTC,13,72.65,10000017,1,djd
+2012-07-12,132939,0,Jewelry & Watches,Fashion Jewelry,Other,Auction,13,66.6,10000018,1,gae
+2012-06-07,113593,15,Phones,Mobile Phones,Mobile Phones,Auction,12,9.26,10000019,1,adj
+2012-12-01,34273,100,eBay Motors,Parts & Accessories,Motorcycle,Auction,14,583.44,10000020,1,cdd
+2012-12-02,106340,15,Home & Garden,Gardening,Hand Tools,FP-GTC,14,638.72,10000021,1,cid
+2012-02-01,150265,15,Baby,Baby Clothing,Boys,FP-GTC,14,4.54,10000022,1,jfj
+2012-01-14,24760,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Hockey-NHL,FP-GTC,12,319.79,10000023,1,abi
+2012-08-14,37831,0,Collectibles,Advertising,Merchandise & Memorabilia,Auction,12,20.35,10000024,1,fag
+2012-10-04,1120,3,Books,First Editions,Other,FP-non GTC,5,223.63,10000025,1,gci
+2012-12-26,43972,100,eBay Motors,Parts & Accessories,ATV Parts,FP-GTC,13,204.28,10000026,1,gch
+2012-12-31,166013,15,Computers,Computer Components & Parts,Video Capture & TV Tuner Cards,Auction,14,5.48,10000027,1,ifh
+2012-03-22,15568,15,Baby,Baby Clothing,Unisex,Auction,14,27.48,10000028,1,hfh
+2012-12-25,103178,15,ClothinShoeAccessories,Women's Bags,Women's Bags,FP-GTC,5,21.72,10000029,1,ggc
+2012-12-05,2023,0,Sporting Goods,Team Sports,Basketball,ABIN,12,3.18,10000030,1,gde
+2012-05-27,94847,0,Consumer Electronics,Vehicle Electronics & GPS,Car Video,FP-GTC,11,491.32,10000031,1,hha
+2012-01-11,15868,0,Real Estate,Land,Land,Auction,14,448.8,10000032,1,hgf
+2012-02-04,32876,0,Home & Garden,Home Improvement,Plumbing & Fixtures,Auction,13,415.73,10000033,1,eia
+2012-12-15,62179,0,ClothinShoes & Accessories,Women's Clothing,Athletic Apparel,Auction,13,377.94,10000034,1,cfh
+2012-06-30,33038,15,Musical Instruments,Instruments,Guitars (Electric),FP-GTC,14,146.33,10000035,1,ide
+2012-12-16,156614,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,FP-GTC,5,7.12,10000036,1,gac
+2012-10-17,106246,0,Health & Beauty,Hair Care & Styling,Shampoo & Conditioning,Auction,13,42.99,10000037,1,afa
+2012-10-08,20865,0,ClothinShoes & Accessories,Men's Clothing,Athletic Apparel,Auction,13,12.85,10000038,1,hed
+2012-04-26,15115,0,Video Games & Consoles,Video Games,Video Games,FP-GTC,13,55.89,10000039,1,hif
+2012-10-06,3838,0,Jewelry & Watches,Fashion Jewelry,Charms & Charm Bracelets,FP-GTC,14,73.26,10000040,1,ggb
+2012-12-28,759,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,Auction,11,112.56,10000041,1,ffi
+2012-11-06,61323,0,Consumer Electronics,TVideo & Home Audio,TVideo & Audio Accessories,FP-non GTC,11,3.49,10000042,1,bdh
+2012-11-06,121153,0,Baby,Nursery Decor,Night Lights,Auction,13,184.21,10000043,1,bej
+2012-12-27,88750,0,Consumer Electronics,Vehicle Electronics & GPS,Radar & Laser Detectors,Auction,13,157.14,10000044,1,ihh
+2012-07-15,161567,15,Computers,Laptop & Desktop Accessories,Laptop Batteries,FP-GTC,14,72.65,10000045,1,eag
+2012-08-23,113802,15,Lots More...,Metaphysical,Herbs,FP-GTC,14,51.23,10000046,1,fcc
+2012-07-10,15808,15,ClothinShoeAccessories,Women's Clothing,Tops & Blouses,FP-non GTC,14,15.85,10000047,1,gbb
+2012-08-10,174053,3,Vehicle Parts & Accessories,Car Parts,External & Body Parts,FP-GTC,13,7.12,10000048,1,ihb
+2012-09-01,2635,0,Toys & Hobbies,Toy Soldiers,1970-Now,Auction,14,12.04,10000049,1,bie
+2012-09-19,1161,3,DVFilm & TV,Other Formats,Videos: NTSC (US),Auction,13,73.26,10000050,1,eah
+2012-08-10,64076,0,Computers/Tablets & Networking,Enterprise NetworkinServers,Switches & Hubs,FP-non GTC,5,184.21,10000051,1,iag
+2012-08-21,33977,15,Crafts,Scrapbooking,Albums,FP-GTC,13,172.03,10000052,1,aga
+2012-08-29,31673,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Racing-NASCAR,FP-GTC,14,122.78,10000053,1,ede
+2012-04-22,174106,3,Vehicle Parts & Accessories,Car Parts,Transmission & Drivetrain,Auction,14,92.98,10000054,1,gii
+2012-07-02,26249,0,Business & Industrial,Printing & Graphic Arts,Commercial Printing Presses,Auction,13,12.19,10000055,1,heb
+2012-01-28,159184,0,Sporting Goods,Winter Sports,Snowboarding,FP-GTC,5,15.65,10000056,1,fab
+2012-03-11,10058,3,Events Tickets,Other Tickets,Other Tickets,FP-GTC,11,101.79,10000057,1,dbf
+2012-05-29,48904,0,ClothinShoes & Accessories,Vintage,Women's Vintage Clothing,ABIN,12,7.12,10000058,1,fda
+2012-05-31,145970,0,Toys & Hobbies,Models & Kits,Automotive,FP-non GTC,14,12.85,10000059,1,gaf
+2012-10-25,963,0,ClothinShoes & Accessories,Vintage,Women's Vintage Shoes,FP-GTC,13,12.19,10000060,1,bfc
+2012-11-23,118687,3,Health & Beauty,Fragrances,Women's Fragrances,FP-GTC,13,92.98,10000061,1,eed
+2012-09-28,20886,0,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,FP-GTC,14,42.99,10000062,1,deh
+2012-09-28,148324,15,Phones,Mobile Accessories,CaseCoverSkins,Auction,13,1.88,10000063,1,dii
+2012-10-19,139255,15,Jewellery & Watches,Fine Jewellery,Earrings,Auction,14,21.14,10000064,1,ceh
+2012-08-05,20213,0,Collectibles,Postcards,US StateCities & Towns,FP-GTC,5,21.14,10000065,1,ejd
+2012-05-19,32996,15,Movies,Television Memorabilia,Clippings,Auction,13,132.33,10000066,1,fab
+2012-04-11,99985,0,Collectibles,Trading Cards,Sci-FFantasy,FP-GTC,14,120.87,10000067,1,dab
+2012-05-16,67703,3,Jewellery & Watches,Jewellery Boxes & Supplies,Jewellery Display,Auction,14,120.87,10000068,1,big
+2012-05-24,65,0,Collectibles,Comics,Platinum Age (1897-1937),FP-non GTC,11,9.26,10000069,1,afj
+2012-01-10,130,0,Collectibles,Transportation,Railroadiana & Trains,FP-non GTC,14,16.26,10000070,1,igh
+2012-05-17,164,0,Computers/Tablets & Networking,Computer Components & Parts,CPUProcessors,FP-GTC,14,157.14,10000071,1,aff
+2012-02-03,216,0,Sports MeCards & Fan Shop,Cards,Hockey,FP-GTC,11,1.88,10000072,1,aif
+2012-05-05,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,12.04,10000073,1,gcd
+2012-11-26,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,14,189.23,10000074,1,hec
+2012-08-30,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,73.26,10000075,1,cdh
+2012-05-21,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,4.13,10000076,1,dei
+2012-06-06,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,290.72,10000077,1,gib
+2012-06-16,223,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-non GTC,5,265.56,10000078,1,hbb
+2012-05-15,279,15,BookMagazines,Children's Books,Children's Books,FP-GTC,5,5.91,10000079,1,jdh
+2012-12-05,314,0,ClothinShoes & Accessories,Women's Clothing,Other,Auction,5,319.79,10000080,1,fag
+2012-03-25,314,211,ClothinShoes & Accessories,Womens' Clothing,Other,Auction,5,246,10000081,1,egj
+2012-03-09,314,211,ClothinShoes & Accessories,Womens' Clothing,Other,Auction,5,20.35,10000082,1,ihi
+2012-05-06,314,0,ClothinShoes & Accessories,Women's Clothing,Other,Auction,5,36.7,10000083,1,efi
+2012-06-15,533,0,Coins & Paper Money,Coins: World,Africa,Auction,13,101.79,10000084,1,hed
+2012-03-14,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,5,47.71,10000085,1,ajb
+2012-05-20,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,5,3.49,10000086,1,gdf
+2012-05-17,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,13,46.44,10000087,1,bch
+2012-03-11,1349,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,13,4.54,10000088,1,hdb
+2012-01-10,1357,0,Collectibles,Decorative Collectibles,Decorative Collectible Brands,ABIN,14,3.18,10000089,1,dda
+2012-04-13,1504,0,Business & Industrial,Electrical & Test Equipment,Test Equipment,FP-GTC,14,86.58,10000090,1,gdd
+2012-07-22,4943,0,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,FP-GTC,13,12.85,10000091,1,ahe
+2012-06-15,6762,0,Unknown,Unknown,Unknown,ABIN,13,16.26,10000092,1,cdj
+2012-06-15,9426,3,Mobile Phones & Communication,Home Phones & Accessories,Phone Accessories,Auction,13,21.14,10000093,1,jff
+2012-06-01,10866,0,Collectibles,Animals,Farm & Countryside,FP-non GTC,14,20.6,10000094,1,jfg
+2012-05-17,11554,0,ClothinShoes & Accessories,Women's Clothing,Jeans,Auction,13,246,10000095,1,cjc
+2012-04-26,11848,0,Health & Beauty,Fragrances,Women,FP-GTC,14,109,10000096,1,baf
+2012-07-28,13836,0,Collectibles,Decorative Collectibles,Spoons,Auction,13,39.41,10000097,1,aha
+2012-03-12,13836,0,Collectibles,Decorative Collectibles,Spoons,Auction,14,16.26,10000098,1,gjc
+2012-04-20,13987,0,Collectibles,Paper,Booklets,FP-GTC,13,112.56,10000099,1,ige
+2012-03-18,15687,0,ClothinShoes & Accessories,Men's Clothing,T-Shirts,Auction,14,184.21,10000100,1,dcg
+2012-05-17,15687,0,ClothinShoes & Accessories,Men's Clothing,T-Shirts,Auction,11,27.48,10000001,1,ifb
+2012-06-23,16145,3,Computers/Tablets & Networking,Computer Components & Parts,Other Components & Parts,FP-non GTC,12,26.45,10000002,1,dei
+2012-05-22,16145,0,Computers/Tablets & Networking,Computer Components & Parts,Other,FP-non GTC,13,415.73,10000003,1,gih
+2012-01-25,16509,0,Toys & Hobbies,Model Railroads & Trains,S Scale,ABIN,5,56.36,10000004,1,iha
+2012-06-12,16509,0,Toys & Hobbies,Model Railroads & Trains,S Scale,ABIN,5,2.44,10000005,1,ige
+2012-05-17,20485,0,Home & Garden,Furniture,Other,FP-GTC,14,269.76,10000006,1,iab
+2012-08-03,20485,101,CasArredamento e Bricolage,Cucina,Altro per cucina,FP-GTC,12,109,10000007,1,hhb
+2012-05-17,20485,101,CasArredamento e Bricolage,Cucina,Altro per cucina,FP-GTC,12,101.79,10000008,1,ehc
+2012-08-21,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,246,10000009,1,dcc
+2012-08-21,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,189.23,10000010,1,hia
+2012-04-18,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,15.65,10000011,1,dja
+2012-06-16,23446,23,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,Auction,14,28.23,10000012,1,baa
+2012-01-09,24541,0,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,College-NCAA,FP-GTC,5,16.26,10000013,1,aib
+2012-05-03,26262,0,Collectibles,Advertising,Food & Beverage,FP-GTC,5,122.78,10000014,1,dia
+2012-05-20,30059,3,Cameras & Photography,Lenses & Filters,Lens AdapterMounts & Tubes,FP-GTC,14,172.03,10000015,1,ebd
+2012-03-12,31387,3,Jewellery & Watches,Watches,Wristwatches,Auction,14,42.99,10000016,1,gbe
+2012-05-19,31387,3,Jewellery & Watches,Watches,Wristwatches,Auction,14,207.5,10000017,1,fhg
+2012-04-21,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,5.91,10000018,1,bhg
+2012-05-18,31519,3,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,39.41,10000019,1,gaf
+2012-06-16,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,16.26,10000020,1,bad
+2012-06-11,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,16.26,10000021,1,fic
+2012-03-31,31519,0,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,78.48,10000022,1,haj
+2012-11-12,31519,3,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,FP-GTC,14,190.22,10000023,1,jcf
+2012-04-06,35570,100,eBay Motors,Parts & Accessories,Motorcycle Parts,FP-GTC,12,2.44,10000024,1,bjc
+2012-11-01,36250,0,Sporting Goods,Hunting,Decoys,Auction,5,7.12,10000025,1,def
+2012-02-06,38238,0,Home & Garden,Home Decor,Other,FP-non GTC,14,36.7,10000026,1,gfj
+2012-06-16,40059,3,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,FP-GTC,14,35.72,10000027,1,gbf
+2012-09-12,40059,3,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,FP-GTC,14,3.49,10000028,1,dfi
+2012-04-16,41940,0,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,FP-GTC,13,223.63,10000029,1,fhf
+2012-11-01,41940,0,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,FP-GTC,13,265.56,10000030,1,ffc
+2012-06-14,43479,0,Cameras & Photo,Film Photography,Other,FP-non GTC,13,62.02,10000031,1,iid
+2012-11-12,44079,0,Sporting Goods,Exercise & Fitness,GyWorkout & Yoga,FP-GTC,12,46.44,10000032,1,dhg
+2012-03-22,45238,101,Abbigliamento e accessori,Donna: Accessori,SciarpFoulard e Scialli,Auction,14,132.33,10000033,1,dab
+2012-05-22,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,Auction,13,448.8,10000034,1,hci
+2012-03-10,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,FP-non GTC,14,207.5,10000035,1,hgh
+2012-05-17,45333,0,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,FP-non GTC,14,190.22,10000036,1,ehc
+2012-09-01,46575,0,Business & Industrial,Light Equipment & Tools,Air Tools,FP-GTC,14,16.71,10000037,1,djg
+2012-04-18,50508,0,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,FP-non GTC,13,4.13,10000038,1,gcg
+2012-06-05,50508,0,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,FP-non GTC,13,1.88,10000039,1,bfa
+2012-10-08,50677,0,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,FP-GTC,13,491.32,10000040,1,dah
+2012-09-17,50677,0,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,FP-GTC,5,2.44,10000041,1,eeh
+2012-02-02,51582,0,ClothinShoes & Accessories,Kids' ClothinShoes & Accs,Girls' Clothing (Sizes 4 & Up),Auction,14,56.36,10000042,1,jaf
+2012-08-23,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-GTC,13,15.85,10000043,1,hfc
+2012-08-17,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-non GTC,14,2.44,10000044,1,ebg
+2012-05-05,57013,0,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,FP-GTC,14,7.12,10000045,1,aff
+2012-03-08,57784,0,ClothinShoes & Accessories,Baby & Toddler Clothing,Boys' Clothing (Newborn-5T),Auction,14,35.72,10000046,1,hdc
+2012-03-16,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,11,9.26,10000047,1,hai
+2012-05-09,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,3.18,10000048,1,fdi
+2012-03-25,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,638.72,10000049,1,gii
+2012-06-05,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,141.7,10000050,1,cgg
+2012-01-10,57990,0,ClothinShoes & Accessories,Men's Clothing,Casual Shirts,ABIN,13,12.19,10000051,1,hed
+2012-06-11,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,14,132.33,10000052,1,jhf
+2012-08-23,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,ABIN,5,5.48,10000053,1,cbb
+2012-08-20,57990,3,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,Auction,11,1.88,10000054,1,bed
+2012-08-09,60340,0,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,FP-GTC,14,12.85,10000055,1,dei
+2012-06-30,60340,0,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,FP-GTC,14,62.02,10000056,1,dbb
+2012-06-03,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,15.85,10000057,1,fbf
+2012-08-14,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,9.26,10000058,1,ijd
+2012-08-14,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,16.71,10000059,1,hci
+2012-05-17,60606,3,Collectables,Badges/ Patches,Golly Badges,FP-GTC,12,20.6,10000060,1,hjh
+2012-07-23,63861,3,ClotheShoes & Accessories,Women's Clothing,Dresses,Auction,5,1.88,10000061,1,cic
+2012-07-27,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,5,141.7,10000062,1,cig
+2012-07-29,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,5,1.88,10000063,1,gae
+2012-11-17,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Others,11,112.56,10000064,1,bjd
+2012-10-29,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Others,11,94.45,10000065,1,cih
+2012-03-28,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,Auction,14,78.48,10000066,1,hbe
+2012-01-30,63861,0,ClothinShoes & Accessories,Women's Clothing,Dresses,ABIN,13,5.48,10000067,1,eec
+2012-01-26,63864,3,ClotheShoes & Accessories,Women's Clothing,Skirts,Auction,14,28.23,10000068,1,deb
+2012-11-21,63889,0,ClothinShoes & Accessories,Women's Shoes,Mixed Items & Lots,Others,13,3.49,10000069,1,bgd
+2012-07-12,67698,2,Business & Industrial,Retail & Services,Jewellery Packaging & Display,FP-GTC,11,15.65,10000070,1,cgg
+2012-06-07,67698,0,Business & Industrial,Retail & Services,Jewelry Packaging & Display,FP-GTC,11,5.48,10000071,1,heb
+2012-12-01,67698,0,Business & Industrial,Retail & Services,Jewelry Packaging & Display,FP-GTC,11,246,10000072,1,jcf
+2012-12-02,73506,0,Collectibles,Decorative Collectibles,Tea PotSets,FP-non GTC,13,122.78,10000073,1,djh
+2012-02-01,75665,0,Home & Garden,YarGarden & Outdoor Living,Gardening Supplies,FP-GTC,14,223.63,10000074,1,ejc
+2012-01-14,75708,3,Toys & Games,Action Figures,TMovies & Video Games,ABIN,5,141.7,10000075,1,gjh
+2012-09-16,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,21.14,10000076,1,cid
+2012-08-09,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,55.89,10000077,1,ccd
+2012-12-26,80053,0,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,FP-non GTC,11,51.23,10000078,1,bfh
+2012-12-31,80135,0,Computers/Tablets & Networking,DriveStorage & Blank Media,Blank Media & Accessories,Auction,14,21.72,10000079,1,gcg
+2012-03-22,95672,3,ClotheShoes & Accessories,Women's Shoes,Trainers,Auction,14,204.28,10000080,1,bji
+2012-12-25,95672,0,ClothinShoes & Accessories,Women's Shoes,Athletic,Others,11,21.14,10000081,1,dig
+2012-01-15,100847,0,Half Books,Half Books,Half Books,Others,5,204.28,10000082,1,eei
+2012-05-27,100847,0,Half Books,Half Books,Half Books,Others,5,122.78,10000083,1,icj
+2012-01-11,139973,3,Video Games & Consoles,Games,Games,ABIN,14,94.45,10000084,1,jfe
+2012-02-04,139973,0,Video Games & Consoles,Video Games,Video Games,ABIN,11,86.58,10000085,1,efj
+2012-12-15,150047,3,Crafts,Jewellery Making,Findings,Auction,14,56.36,10000086,1,icd
+2012-06-30,150047,3,Crafts,Jewellery Making,Findings,Auction,14,290.72,10000087,1,dci
+2012-12-16,155226,0,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,FP-GTC,13,60.37,10000088,1,ghg
+2012-11-11,155226,0,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,FP-GTC,13,112.56,10000089,1,ebc
+2012-10-08,156356,0,Collectibles,Postcards,BuildingArchitecture,FP-GTC,13,265.56,10000090,1,gdh
+2012-04-26,158798,0,Toys & Hobbies,Vintage & Antique Toys,Spinning Tops,FP-GTC,11,35.72,10000091,1,cbc
+2012-10-06,165888,0,Jewelry & Watches,Vintage & Antique Jewelry,Costume,FP-non GTC,13,92.98,10000092,1,gij
+2012-12-28,170083,3,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),Auction,11,28.23,10000093,1,ife
+2012-11-06,170083,3,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),Auction,11,27.48,10000094,1,hgb
+2012-11-06,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,9.26,10000095,1,dfa
+2012-12-27,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,3.18,10000096,1,jja
+2012-01-01,175750,0,Home & Garden,Bedding,Blankets & Throws,FP-GTC,14,12.04,10000097,1,gjf
+2012-08-23,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,13,20.6,10000098,1,bif
+2012-07-10,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,13,12.04,10000099,1,iad
+2012-08-10,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,4.13,10000100,1,bfg
+2012-07-19,175750,3,HomFurniture & DIY,Bedding,Blankets,Auction,14,73.26,10000201,1,dfg
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index 6e444e5..16ce8ae 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -106,7 +106,8 @@
"metrics": [
"PRICE",
"ITEM_COUNT",
- "SELLER_ID"
+ "SELLER_ID",
+ "USER_ID"
],
"last_modified": 1422435345352,
"fact_table": "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
index 621eccc..d7b7227 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
@@ -105,7 +105,8 @@
"metrics": [
"PRICE",
"ITEM_COUNT",
- "SELLER_ID"
+ "SELLER_ID",
+ "USER_ID"
],
"last_modified": 1422435345352,
"fact_table": "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index dab99f9..92b336f 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -106,7 +106,8 @@
"metrics": [
"PRICE",
"ITEM_COUNT",
- "SELLER_ID"
+ "SELLER_ID",
+ "USER_ID"
],
"last_modified": 1422435345352,
"fact_table": "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
index 819b8b0..2ff55d3 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
@@ -105,7 +105,8 @@
"metrics": [
"PRICE",
"ITEM_COUNT",
- "SELLER_ID"
+ "SELLER_ID",
+ "USER_ID"
],
"last_modified": 1422435345352,
"fact_table": "DEFAULT.TEST_KYLIN_FACT",
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index 3e65edd..fdfb322 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -40,13 +40,17 @@
"datatype" : "bigint"
}, {
"id" : "10",
+ "name" : "USER_ID",
+ "datatype" : "varchar(32)"
+ }, {
+ "id" : "11",
"name" : "BUYER_COUNTRY",
"datatype" : "string"
}, {
- "id" : "11",
+ "id" : "12",
"name" : "SELLER_COUNTRY",
"datatype" : "string"
} ],
"database" : "DEFAULT",
"last_modified" : 0
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a0ee798/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
index d5aa5b6..25598ad 100644
--- a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
@@ -38,7 +38,7 @@ public class ITHiveTableReaderTest extends HBaseMetadataTestCase {
int rowNumber = 0;
while (reader.next()) {
String[] row = reader.getRow();
- Assert.assertEquals(11, row.length);
+ Assert.assertEquals(12, row.length);
//System.out.println(ArrayUtils.toString(row));
rowNumber++;
}