You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/08 05:49:49 UTC
[01/18] kylin git commit: KYLIN-1875 Normalize table and column names
in JSON [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/master-hbase1.x 469d9cc5c -> eaf0cc586 (forced update)
KYLIN-1875 Normalize table and column names in JSON
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/967ef180
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/967ef180
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/967ef180
Branch: refs/heads/master-hbase1.x
Commit: 967ef18062048e199cd0dd351ba82618a10b08e5
Parents: 081ed0d
Author: Li Yang <li...@apache.org>
Authored: Tue Nov 29 20:32:04 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Dec 7 17:53:35 2016 +0800
----------------------------------------------------------------------
.../kylin/job/dataGen/FactTableGenerator.java | 22 +++++++++---
.../org/apache/kylin/cube/model/CubeDesc.java | 1 +
.../apache/kylin/cube/model/DimensionDesc.java | 19 +++++-----
.../model/validation/rule/FunctionRule.java | 37 ++++----------------
.../kylin/metadata/model/DataModelDesc.java | 13 ++++++-
.../kylin/metadata/model/FunctionDesc.java | 5 +--
.../metadata/model/ModelDimensionDesc.java | 19 +++++++---
.../kylin/metadata/model/PartitionDesc.java | 4 +--
8 files changed, 64 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 677b713..011035b 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -81,8 +81,7 @@ public class FactTableGenerator {
// table(appear as fk in fact table)
TreeMap<String, LinkedList<String>> lookupTableKeys = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- // possible values of lookupTableKeys, extracted from existing lookup
- // tables.
+ // possible values of lookupTableKeys, extracted from existing lookup tables.
// The key is in the format of tablename/columnname
TreeMap<String, ArrayList<String>> feasibleValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
@@ -244,7 +243,7 @@ public class FactTableGenerator {
JoinDesc join = dim.getJoin();
if (join != null) {
String lookupTable = dim.getTableRef().getTableIdentity();
- for (String column : join.getPrimaryKey()) {
+ for (String column : dropAlias(join.getPrimaryKey())) {
if (!lookupTableKeys.containsKey(lookupTable)) {
lookupTableKeys.put(lookupTable, new LinkedList<String>());
}
@@ -297,8 +296,8 @@ public class FactTableGenerator {
for (DimensionDesc dim : dimensions) {
JoinDesc jDesc = dim.getJoin();
if (jDesc != null) {
- String[] fks = jDesc.getForeignKey();
- String[] pks = jDesc.getPrimaryKey();
+ String[] fks = dropAlias(jDesc.getForeignKey());
+ String[] pks = dropAlias(jDesc.getPrimaryKey());
int num = fks.length;
for (int i = 0; i < num; ++i) {
String value = dim.getTableRef().getTableIdentity() + "/" + pks[i];
@@ -343,6 +342,19 @@ public class FactTableGenerator {
return createTable(this.rowCount, factTableCol2LookupCol, lookupCol2factTableCol, usedCols);
}
+ private String[] dropAlias(String[] aliasDotCol) {
+ String[] result = new String[aliasDotCol.length];
+ for (int i = 0; i < aliasDotCol.length; i++) {
+ String str = aliasDotCol[i];
+ int cut = str.lastIndexOf('.');
+ if (cut >= 0) {
+ str = str.substring(cut + 1);
+ }
+ result[i] = str;
+ }
+ return result;
+ }
+
private String normToTwoDigits(int v) {
if (v < 10)
return "0" + v;
http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 853571c..f95cceb 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -543,6 +543,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
initMeasureColumns();
rowkey.init(this);
+
validateAggregationGroups(); // check if aggregation group is valid
for (AggregationGroup agg : this.aggregationGroups) {
agg.init(this, rowkey);
http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
index 1106103..cd75228 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/DimensionDesc.java
@@ -21,7 +21,6 @@ package org.apache.kylin.cube.model;
import java.util.Arrays;
import org.apache.commons.lang.NotImplementedException;
-import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
@@ -54,16 +53,13 @@ public class DimensionDesc {
private TblColRef[] columnRefs;
public void init(CubeDesc cubeDesc) {
+ DataModelDesc model = cubeDesc.getModel();
+
if (name != null)
name = name.toUpperCase();
- if (table != null)
- table = table.toUpperCase();
-
- DataModelDesc model = cubeDesc.getModel();
- tableRef = model.findTable(this.getTable());
- if (tableRef == null)
- throw new IllegalStateException("Can't find table " + table + " for dimension " + name);
+ tableRef = model.findTable(table);
+ table = tableRef.getAlias();
join = null;
for (JoinTableDesc joinTable : model.getJoinTables()) {
@@ -73,11 +69,16 @@ public class DimensionDesc {
}
}
+ if (column != null && !"{FK}".equals(column)) {
+ column = model.findColumn(table, column).getName();
+ }
if (derived != null && derived.length == 0) {
derived = null;
}
if (derived != null) {
- StringUtil.toUpperCaseArray(derived, derived);
+ for (int i = 0; i < derived.length; i++) {
+ derived[i] = model.findColumn(table, derived[i]).getName();
+ }
}
if (derived != null && join == null) {
throw new IllegalStateException("Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this);
http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
index 1ed7325..36631ce 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
@@ -31,12 +31,10 @@ import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.measure.topn.TopNMeasureType;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
import com.google.common.collect.Lists;
@@ -153,35 +151,12 @@ public class FunctionRule implements IValidatorRule<CubeDesc> {
* @param value
*/
private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
- String factTable = cube.getModel().getRootFactTable().getTableIdentity();
- if (StringUtils.isEmpty(factTable)) {
- context.addResult(ResultLevel.ERROR, "Fact table can not be null.");
- return;
- }
- TableDesc table = MetadataManager.getInstance(cube.getConfig()).getTableDesc(factTable);
- if (table == null) {
- context.addResult(ResultLevel.ERROR, "Fact table can not be found: " + cube);
- return;
- }
- // Prepare column set
- Set<String> set = new HashSet<String>();
- ColumnDesc[] cdesc = table.getColumns();
- for (int i = 0; i < cdesc.length; i++) {
- ColumnDesc columnDesc = cdesc[i];
- set.add(columnDesc.getName());
- }
-
- String[] items = value.split(",");
- for (int i = 0; i < items.length; i++) {
- String item = items[i].trim();
- if (StringUtils.isEmpty(item)) {
- continue;
- }
- if (!set.contains(item)) {
- context.addResult(ResultLevel.ERROR, "Column [" + item + "] does not exist in factable table" + factTable);
- }
+ DataModelDesc model = cube.getModel();
+ try {
+ model.findColumn(value);
+ } catch (IllegalArgumentException e) {
+ context.addResult(ResultLevel.ERROR, e.getMessage());
}
-
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index d917571..898ff74 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -282,7 +282,7 @@ public class DataModelDesc extends RootPersistentEntity {
initTableAlias(tables);
initJoinColumns();
initJoinsTree();
- ModelDimensionDesc.capicalizeStrings(dimensions);
+ initDimensionsAndMetrics();
initPartitionDesc();
}
@@ -354,6 +354,15 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
+ private void initDimensionsAndMetrics() {
+ for (ModelDimensionDesc dim : dimensions) {
+ dim.init(this);
+ }
+ for (int i = 0; i < metrics.length; i++) {
+ metrics[i] = findColumn(metrics[i]).getIdentity();
+ }
+ }
+
private void initPartitionDesc() {
if (this.partitionDesc != null)
this.partitionDesc.init(this);
@@ -381,6 +390,7 @@ public class DataModelDesc extends RootPersistentEntity {
if (col == null || col.getTableRef().equals(dimTable) == false) {
throw new IllegalStateException("Can't find column " + pks[i] + " in table " + dimTable.getTableIdentity());
}
+ pks[i] = col.getIdentity();
pkCols[i] = col;
}
join.setPrimaryKeyColumns(pkCols);
@@ -393,6 +403,7 @@ public class DataModelDesc extends RootPersistentEntity {
if (col == null) {
throw new IllegalStateException("Can't find column " + fks[i] + " in table " + this.getRootFactTable());
}
+ fks[i] = col.getIdentity();
fkCols[i] = col;
}
join.setForeignKeyColumns(fkCols);
http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 7b41552..dfa6f3b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -77,14 +77,11 @@ public class FunctionDesc {
expression = expression.toUpperCase();
returnDataType = DataType.getType(returnType);
- for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) {
- p.setValue(p.getValue().toUpperCase());
- }
-
ArrayList<TblColRef> colRefs = Lists.newArrayList();
for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) {
if (p.isColumnType()) {
TblColRef colRef = model.findColumn(p.getValue());
+ p.setValue(colRef.getIdentity());
colRefs.add(colRef);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
index d196155..6460f71 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ModelDimensionDesc.java
@@ -50,13 +50,24 @@ public class ModelDimensionDesc {
this.columns = columns;
}
+ void init(DataModelDesc model) {
+ table = table.toUpperCase();
+ if (columns != null) {
+ StringUtil.toUpperCaseArray(columns, columns);
+ }
+
+ if (model != null) {
+ table = model.findTable(table).getAlias();
+ for (int i = 0; i < columns.length; i++) {
+ columns[i] = model.findColumn(table, columns[i]).getName();
+ }
+ }
+ }
+
public static void capicalizeStrings(List<ModelDimensionDesc> dimensions) {
if (dimensions != null) {
for (ModelDimensionDesc modelDimensionDesc : dimensions) {
- modelDimensionDesc.setTable(modelDimensionDesc.getTable().toUpperCase());
- if (modelDimensionDesc.getColumns() != null) {
- StringUtil.toUpperCaseArray(modelDimensionDesc.getColumns(), modelDimensionDesc.getColumns());
- }
+ modelDimensionDesc.init(null);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/967ef180/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 0261f41..9925990 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -66,11 +66,11 @@ public class PartitionDesc {
if (StringUtils.isEmpty(partitionDateColumn))
return;
- partitionDateColumn = partitionDateColumn.toUpperCase();
partitionDateColumnRef = model.findColumn(partitionDateColumn);
+ partitionDateColumn = partitionDateColumnRef.getIdentity();
if (StringUtils.isBlank(partitionTimeColumn) == false) {
- partitionTimeColumn = partitionTimeColumn.toUpperCase();
partitionTimeColumnRef = model.findColumn(partitionTimeColumn);
+ partitionTimeColumn = partitionTimeColumnRef.getIdentity();
}
partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
}
[03/18] kylin git commit: KYLIN-1875 fix new instance of
ParameterDesc & FunctionDesc
Posted by li...@apache.org.
KYLIN-1875 fix new instance of ParameterDesc & FunctionDesc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64d9b8b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64d9b8b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64d9b8b8
Branch: refs/heads/master-hbase1.x
Commit: 64d9b8b89f84cc31f9931e5a6cdf89bdfa13925a
Parents: 227bbf4
Author: Li Yang <li...@apache.org>
Authored: Thu Dec 1 14:10:44 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Dec 7 17:53:37 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/cube/RawQueryLastHacker.java | 8 +---
.../metadata/measure/MeasureCodecTest.java | 3 +-
.../kylin/measure/raw/RawMeasureType.java | 12 ++---
.../kylin/metadata/model/FunctionDesc.java | 26 ++++-------
.../kylin/metadata/model/ParameterDesc.java | 47 +++++++++++++++-----
.../apache/kylin/storage/StorageMockUtils.java | 32 +++----------
.../kylin/query/relnode/OLAPAggregateRel.java | 12 ++---
7 files changed, 59 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
index b0a4823..682e48c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -57,12 +57,8 @@ public class RawQueryLastHacker {
sqlDigest.groupbyColumns.add(col);
} else {
// For measure columns, take them as metric columns with aggregation function SUM().
- ParameterDesc colParameter = new ParameterDesc();
- colParameter.setType("column");
- colParameter.setValue(col.getName());
- FunctionDesc sumFunc = new FunctionDesc();
- sumFunc.setExpression("SUM");
- sumFunc.setParameter(colParameter);
+ ParameterDesc parameter = ParameterDesc.newInstance(col);
+ FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", parameter, null);
boolean measureHasSum = false;
for (MeasureDesc colMeasureDesc : cubeDesc.getMeasures()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 6bb71e9..18680ec 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -83,8 +83,7 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase {
private MeasureDesc measure(String returnType) {
MeasureDesc desc = new MeasureDesc();
- FunctionDesc func = new FunctionDesc();
- func.setReturnType(returnType);
+ FunctionDesc func = FunctionDesc.newInstance(null, null, returnType);
desc.setFunction(func);
return desc;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 3a49d31..a5bb06b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -197,18 +197,12 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
if (sqlDigest.isRawQuery) {
for (MeasureDesc measureDesc : measureDescs) {
TblColRef col = this.getRawColumn(measureDesc.getFunction());
- ParameterDesc colParameter = new ParameterDesc();
- colParameter.setType("column");
- colParameter.setValue(col.getName());
- FunctionDesc rawFunc = new FunctionDesc();
- rawFunc.setExpression("RAW");
- rawFunc.setParameter(colParameter);
+ ParameterDesc colParameter = ParameterDesc.newInstance(col);
+ FunctionDesc rawFunc = FunctionDesc.newInstance("RAW", colParameter, null);
if (sqlDigest.allColumns.contains(col)) {
if (measureDesc.getFunction().equals(rawFunc)) {
- FunctionDesc sumFunc = new FunctionDesc();
- sumFunc.setExpression("SUM");
- sumFunc.setParameter(colParameter);
+ FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", colParameter, null);
sqlDigest.aggregations.remove(sumFunc);
sqlDigest.aggregations.add(rawFunc);
logger.info("Add RAW measure on column " + col);
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index b9e5543..832cb4a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -40,6 +40,15 @@ import com.google.common.collect.Sets;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class FunctionDesc {
+ public static FunctionDesc newInstance(String expression, ParameterDesc param, String returnType) {
+ FunctionDesc r = new FunctionDesc();
+ r.expression = (expression == null) ? null : expression.toUpperCase();
+ r.parameter = param;
+ r.returnType = returnType;
+ r.returnDataType = DataType.getType(returnType);
+ return r;
+ }
+
public static final String FUNC_SUM = "SUM";
public static final String FUNC_MIN = "MIN";
public static final String FUNC_MAX = "MAX";
@@ -202,18 +211,10 @@ public class FunctionDesc {
return expression;
}
- public void setExpression(String expression) {
- this.expression = expression;
- }
-
public ParameterDesc getParameter() {
return parameter;
}
- public void setParameter(ParameterDesc parameter) {
- this.parameter = parameter;
- }
-
public int getParameterCount() {
int count = 0;
for (ParameterDesc p = parameter; p != null; p = p.getNextParameter()) {
@@ -230,19 +231,10 @@ public class FunctionDesc {
return returnDataType;
}
- public void setReturnType(String returnType) {
- this.returnType = returnType;
- this.returnDataType = DataType.getType(returnType);
- }
-
public Map<String, String> getConfiguration() {
return configuration;
}
- public void setConfiguration(Map<String, String> configurations) {
- this.configuration = configurations;
- }
-
@Override
public int hashCode() {
final int prime = 31;
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 4a95fea..c14d061 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -19,6 +19,7 @@
package org.apache.kylin.metadata.model;
import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -26,12 +27,42 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
/**
*/
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class ParameterDesc {
+ public static ParameterDesc newInstance(Object... objs) {
+ if (objs.length == 0)
+ throw new IllegalArgumentException();
+
+ ParameterDesc r = new ParameterDesc();
+
+ Object obj = objs[0];
+ if (obj instanceof TblColRef) {
+ TblColRef col = (TblColRef) obj;
+ r.type = FunctionDesc.PARAMETER_TYPE_COLUMN;
+ r.value = col.getIdentity();
+ r.colRefs = ImmutableList.of(col);
+ } else {
+ r.type = FunctionDesc.PARAMETER_TYPE_CONSTANT;
+ r.value = (String) obj;
+ }
+
+ if (objs.length >= 2) {
+ r.nextParameter = newInstance(Arrays.copyOfRange(objs, 1, objs.length));
+ if (r.nextParameter.colRefs.size() > 0) {
+ if (r.colRefs.isEmpty())
+ r.colRefs = r.nextParameter.colRefs;
+ else
+ r.colRefs = ImmutableList.copyOf(Iterables.concat(r.colRefs, r.nextParameter.colRefs));
+ }
+ }
+ return r;
+ }
+
@JsonProperty("type")
private String type;
@JsonProperty("value")
@@ -47,10 +78,6 @@ public class ParameterDesc {
return type;
}
- public void setType(String type) {
- this.type = type;
- }
-
public byte[] getBytes() throws UnsupportedEncodingException {
return value.getBytes("UTF-8");
}
@@ -58,16 +85,16 @@ public class ParameterDesc {
public String getValue() {
return value;
}
-
- public void setValue(String value) {
+
+ void setValue(String value) {
this.value = value;
}
public List<TblColRef> getColRefs() {
return colRefs;
}
-
- public void setColRefs(List<TblColRef> colRefs) {
+
+ void setColRefs(List<TblColRef> colRefs) {
this.colRefs = colRefs;
}
@@ -75,10 +102,6 @@ public class ParameterDesc {
return nextParameter;
}
- public void setNextParameter(ParameterDesc nextParameter) {
- this.nextParameter = nextParameter;
- }
-
public boolean isColumnType() {
return FunctionDesc.PARAMETER_TYPE_COLUMN.equals(type);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
index 0786f32..0706713 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/StorageMockUtils.java
@@ -36,8 +36,6 @@ import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.TupleInfo;
-import com.google.common.collect.ImmutableList;
-
/**
*/
public class StorageMockUtils {
@@ -83,14 +81,8 @@ public class StorageMockUtils {
TblColRef priceCol = model.findColumn("DEFAULT.TEST_KYLIN_FACTPRICE");
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression("SUM");
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("column");
- p1.setValue("PRICE");
- p1.setColRefs(ImmutableList.of(priceCol));
- f1.setParameter(p1);
- f1.setReturnType("decimal(19,4)");
+ FunctionDesc f1 = FunctionDesc.newInstance("SUM", //
+ ParameterDesc.newInstance(priceCol), "decimal(19,4)");
functions.add(f1);
return functions;
@@ -102,24 +94,12 @@ public class StorageMockUtils {
TblColRef priceCol = model.findColumn("DEFAULT.TEST_KYLIN_FACT.PRICE");
TblColRef sellerCol = model.findColumn("DEFAULT.TEST_KYLIN_FACT.SELLER_ID");
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression("SUM");
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("column");
- p1.setValue("PRICE");
- p1.setColRefs(ImmutableList.of(priceCol));
- f1.setParameter(p1);
- f1.setReturnType("decimal(19,4)");
+ FunctionDesc f1 = FunctionDesc.newInstance("SUM", //
+ ParameterDesc.newInstance(priceCol), "decimal(19,4)");
functions.add(f1);
- FunctionDesc f2 = new FunctionDesc();
- f2.setExpression("COUNT_DISTINCT");
- ParameterDesc p2 = new ParameterDesc();
- p2.setType("column");
- p2.setValue("SELLER_ID");
- p2.setColRefs(ImmutableList.of(sellerCol));
- f2.setParameter(p2);
- f2.setReturnType("hllc(10)");
+ FunctionDesc f2 = FunctionDesc.newInstance("COUNT_DISTINCT", //
+ ParameterDesc.newInstance(sellerCol), "hllc(10)");
functions.add(f2);
return functions;
http://git-wip-us.apache.org/repos/asf/kylin/blob/64d9b8b8/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 62351d3..24933f5 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -19,7 +19,6 @@
package org.apache.kylin.query.relnode;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -238,16 +237,11 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
int index = aggCall.getArgList().get(0);
TblColRef column = inputColumnRowType.getColumnByIndex(index);
if (!column.isInnerColumn()) {
- parameter = new ParameterDesc();
- parameter.setValue(column.getName());
- parameter.setType(FunctionDesc.PARAMETER_TYPE_COLUMN);
- parameter.setColRefs(Arrays.asList(column));
+ parameter = ParameterDesc.newInstance(column);
}
}
- FunctionDesc aggFunc = new FunctionDesc();
- String funcName = getAggrFuncName(aggCall);
- aggFunc.setExpression(funcName);
- aggFunc.setParameter(parameter);
+ String expression = getAggrFuncName(aggCall);
+ FunctionDesc aggFunc = FunctionDesc.newInstance(expression, parameter, null);
this.aggregations.add(aggFunc);
}
}
[11/18] kylin git commit: KYLIN-2255 drop v1 coprocessor impl
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
deleted file mode 100644
index 74b2112..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v1.filter;
-
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestFuzzyRowFilterV2 {
- @Test
- public void testSatisfiesNoUnsafeForward() {
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, (byte) -128, 1, 0, 1 }, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, (byte) -128, 2, 0, 1 }, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 2, 1, 3, 3 }, 0, 5, new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS,
- FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 1, 3, 0 }, // row to check
- 0, 5, new byte[] { 1, 2, 0, 3 }, // fuzzy row
- new byte[] { 0, 0, 1, 0 })); // mask
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 1, 1, 3, 0 }, 0, 5, new byte[] { 1, (byte) 245, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(false, new byte[] { 1, 2, 1, 0, 1 }, 0, 5, new byte[] { 0, 1, 2 }, new byte[] { 1, 0, 0 }));
- }
-
- @Test
- public void testSatisfiesForward() {
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, (byte) -128, 1, 0, 1 }, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, (byte) -128, 2, 0, 1 }, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 2, 1, 3, 3 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS,
- FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 }, // row to check
- new byte[] { 1, 2, 0, 3 }, // fuzzy row
- new byte[] { -1, -1, 0, -1 })); // mask
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 1, 1, 3, 0 }, new byte[] { 1, (byte) 245, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(false, new byte[] { 1, 2, 1, 0, 1 }, new byte[] { 0, 1, 2 }, new byte[] { 0, -1, -1 }));
- }
-
- @Test
- public void testSatisfiesReverse() {
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, (byte) -128, 1, 0, 1 }, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, (byte) -128, 2, 0, 1 }, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 2, 3, 1, 1, 1 }, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, 2, 1, 3, 3 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, (byte) 245, 1, 3, 0 }, new byte[] { 1, 1, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, 3, 1, 3, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 2, 1, 1, 1, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfies(true, new byte[] { 1, 2, 1, 0, 1 }, new byte[] { 0, 1, 2 }, new byte[] { 0, -1, -1 }));
- }
-
- @Test
- public void testSatisfiesNoUnsafeReverse() {
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, (byte) -128, 1, 0, 1 }, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, (byte) -128, 2, 0, 1 }, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 2, 3, 1, 1, 1 }, 0, 5, new byte[] { 1, 0, 1 }, new byte[] { 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.YES, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, 2, 1, 3, 3 }, 0, 5, new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, (byte) 245, 1, 3, 0 }, 0, 5, new byte[] { 1, 1, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, 3, 1, 3, 0 }, 0, 5, new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 2, 1, 1, 1, 0 }, 0, 5, new byte[] { 1, 2, 0, 3 }, new byte[] { 0, 0, 1, 0 }));
-
- Assert.assertEquals(FuzzyRowFilterV2.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilterV2.satisfiesNoUnsafe(true, new byte[] { 1, 2, 1, 0, 1 }, 0, 5, new byte[] { 0, 1, 2 }, new byte[] { 1, 0, 0 }));
- }
-
- @Test
- public void testGetNextForFuzzyRuleForward() {
- assertNext(false, new byte[] { 0, 1, 2 }, // fuzzy row
- new byte[] { 0, -1, -1 }, // mask
- new byte[] { 1, 2, 1, 0, 1 }, // current
- new byte[] { 2, 1, 2, 0, 0 }); // expected next
-
- assertNext(false, new byte[] { 0, 1, 2 }, // fuzzy row
- new byte[] { 0, -1, -1 }, // mask
- new byte[] { 1, 1, 2, 0, 1 }, // current
- new byte[] { 1, 1, 2, 0, 2 }); // expected next
-
- assertNext(false, new byte[] { 0, 1, 0, 2, 0 }, // fuzzy row
- new byte[] { 0, -1, 0, -1, 0 }, // mask
- new byte[] { 1, 0, 2, 0, 1 }, // current
- new byte[] { 1, 1, 0, 2, 0 }); // expected next
-
- assertNext(false, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }, new byte[] { 1, (byte) 128, 2, 0, 1 }, new byte[] { 1, (byte) 129, 1, 0, 0 });
-
- assertNext(false, new byte[] { 0, 1, 0, 1 }, new byte[] { 0, -1, 0, -1 }, new byte[] { 5, 1, 0, 1 }, new byte[] { 5, 1, 1, 1 });
-
- assertNext(false, new byte[] { 0, 1, 0, 1 }, new byte[] { 0, -1, 0, -1 }, new byte[] { 5, 1, 0, 1, 1 }, new byte[] { 5, 1, 0, 1, 2 });
-
- assertNext(false, new byte[] { 0, 1, 0, 0 }, // fuzzy row
- new byte[] { 0, -1, 0, 0 }, // mask
- new byte[] { 5, 1, (byte) 255, 1 }, // current
- new byte[] { 5, 1, (byte) 255, 2 }); // expected next
-
- assertNext(false, new byte[] { 0, 1, 0, 1 }, // fuzzy row
- new byte[] { 0, -1, 0, -1 }, // mask
- new byte[] { 5, 1, (byte) 255, 1 }, // current
- new byte[] { 6, 1, 0, 1 }); // expected next
-
- assertNext(false, new byte[] { 0, 1, 0, 1 }, // fuzzy row
- new byte[] { 0, -1, 0, -1 }, // mask
- new byte[] { 5, 1, (byte) 255, 0 }, // current
- new byte[] { 5, 1, (byte) 255, 1 }); // expected next
-
- assertNext(false, new byte[] { 5, 1, 1, 0 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 5, 1, (byte) 255, 1 }, new byte[] { 5, 1, (byte) 255, 2 });
-
- assertNext(false, new byte[] { 1, 1, 1, 1 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 1, 1, 2, 2 }, new byte[] { 1, 1, 2, 3 });
-
- assertNext(false, new byte[] { 1, 1, 1, 1 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 1, 1, 3, 2 }, new byte[] { 1, 1, 3, 3 });
-
- assertNext(false, new byte[] { 1, 1, 1, 1 }, new byte[] { 0, 0, 0, 0 }, new byte[] { 1, 1, 2, 3 }, new byte[] { 1, 1, 2, 4 });
-
- assertNext(false, new byte[] { 1, 1, 1, 1 }, new byte[] { 0, 0, 0, 0 }, new byte[] { 1, 1, 3, 2 }, new byte[] { 1, 1, 3, 3 });
-
- assertNext(false, new byte[] { 1, 1, 0, 0 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 0, 1, 3, 2 }, new byte[] { 1, 1, 0, 0 });
-
- // No next for this one
- Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(new byte[] { 2, 3, 1, 1, 1 }, // row to check
- new byte[] { 1, 0, 1 }, // fuzzy row
- new byte[] { -1, 0, -1 })); // mask
- Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(new byte[] { 1, (byte) 245, 1, 3, 0 }, new byte[] { 1, 1, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
- Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(new byte[] { 1, 3, 1, 3, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
- Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(new byte[] { 2, 1, 1, 1, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
- }
-
- @Test
- public void testGetNextForFuzzyRuleReverse() {
- assertNext(true, new byte[] { 0, 1, 2 }, // fuzzy row
- new byte[] { 0, -1, -1 }, // mask
- new byte[] { 1, 2, 1, 0, 1 }, // current
- // TODO: should be {1, 1, 3} ?
- new byte[] { 1, 1, 2, (byte) 0xFF, (byte) 0xFF }); // expected next
-
- assertNext(true, new byte[] { 0, 1, 0, 2, 0 }, // fuzzy row
- new byte[] { 0, -1, 0, -1, 0 }, // mask
- new byte[] { 1, 2, 1, 3, 1 }, // current
- // TODO: should be {1, 1, 1, 3} ?
- new byte[] { 1, 1, 0, 2, 0 }); // expected next
-
- assertNext(true, new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }, new byte[] { 1, (byte) 128, 2, 0, 1 },
- // TODO: should be {1, (byte) 128, 2} ?
- new byte[] { 1, (byte) 128, 1, (byte) 0xFF, (byte) 0xFF });
-
- assertNext(true, new byte[] { 0, 1, 0, 1 }, new byte[] { 0, -1, 0, -1 }, new byte[] { 5, 1, 0, 2, 1 },
- // TODO: should be {5, 1, 0, 2} ?
- new byte[] { 5, 1, 0, 1, (byte) 0xFF });
-
- assertNext(true, new byte[] { 0, 1, 0, 0 }, // fuzzy row
- new byte[] { 0, -1, 0, 0 }, // mask
- new byte[] { 5, 1, (byte) 255, 1 }, // current
- new byte[] { 5, 1, (byte) 255, 0 }); // expected next
-
- assertNext(true, new byte[] { 0, 1, 0, 1 }, // fuzzy row
- new byte[] { 0, -1, 0, -1 }, // mask
- new byte[] { 5, 1, 0, 1 }, // current
- new byte[] { 4, 1, (byte) 255, 1 }); // expected next
-
- assertNext(true, new byte[] { 0, 1, 0, 1 }, // fuzzy row
- new byte[] { 0, -1, 0, -1 }, // mask
- new byte[] { 5, 1, (byte) 255, 0 }, // current
- new byte[] { 5, 1, (byte) 254, 1 }); // expected next
-
- assertNext(true, new byte[] { 1, 1, 0, 0 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 2, 1, 3, 2 },
- // TODO: should be {1, 0} ?
- new byte[] { 1, 1, 0, 0 });
-
- assertNext(true, new byte[] { 1, 0, 1 }, // fuzzy row
- new byte[] { -1, 0, -1 }, // mask
- new byte[] { 2, 3, 1, 1, 1 }, // row to check
- // TODO: should be {1, (byte) 0xFF, 2} ?
- new byte[] { 1, 0, 1, (byte) 0xFF, (byte) 0xFF });
-
- assertNext(true, new byte[] { 1, 1, 0, 3 }, new byte[] { -1, -1, 0, -1 }, new byte[] { 1, (byte) 245, 1, 3, 0 },
- // TODO: should be {1, 1, (byte) 255, 4} ?
- new byte[] { 1, 1, 0, 3, (byte) 0xFF });
-
- assertNext(true, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }, new byte[] { 1, 3, 1, 3, 0 },
- // TODO: should be 1, 2, (byte) 255, 4 ?
- new byte[] { 1, 2, 0, 3, (byte) 0xFF });
-
- assertNext(true, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }, new byte[] { 2, 1, 1, 1, 0 },
- // TODO: should be {1, 2, (byte) 255, 4} ?
- new byte[] { 1, 2, 0, 3, (byte) 0xFF });
-
- assertNext(true,
- // TODO: should be null?
- new byte[] { 1, 0, 1 }, new byte[] { -1, 0, -1 }, new byte[] { 1, (byte) 128, 2 }, new byte[] { 1, (byte) 128, 1 });
-
- assertNext(true,
- // TODO: should be null?
- new byte[] { 0, 1, 0, 1 }, new byte[] { 0, -1, 0, -1 }, new byte[] { 5, 1, 0, 2 }, new byte[] { 5, 1, 0, 1 });
-
- assertNext(true,
- // TODO: should be null?
- new byte[] { 5, 1, 1, 0 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 5, 1, (byte) 0xFF, 1 }, new byte[] { 5, 1, (byte) 0xFF, 0 });
-
- assertNext(true,
- // TODO: should be null?
- new byte[] { 1, 1, 1, 1 }, new byte[] { -1, -1, 0, 0 }, new byte[] { 1, 1, 2, 2 }, new byte[] { 1, 1, 2, 1 });
-
- assertNext(true,
- // TODO: should be null?
- new byte[] { 1, 1, 1, 1 }, new byte[] { 0, 0, 0, 0 }, new byte[] { 1, 1, 2, 3 }, new byte[] { 1, 1, 2, 2 });
-
- Assert.assertNull(FuzzyRowFilterV2.getNextForFuzzyRule(true, new byte[] { 1, 1, 1, 3, 0 }, new byte[] { 1, 2, 0, 3 }, new byte[] { -1, -1, 0, -1 }));
- }
-
- private static void assertNext(boolean reverse, byte[] fuzzyRow, byte[] mask, byte[] current, byte[] expected) {
- KeyValue kv = KeyValue.createFirstOnRow(current);
- byte[] nextForFuzzyRule = FuzzyRowFilterV2.getNextForFuzzyRule(reverse, kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), fuzzyRow, mask);
- Assert.assertEquals(Bytes.toStringBinary(expected), Bytes.toStringBinary(nextForFuzzyRule));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
deleted file mode 100644
index 1d85922..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v1.filter;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-@Ignore
-public class TestFuzzyRowFilterV2EndToEnd {
- private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- private final static byte fuzzyValue = (byte) 63;
- private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterV2EndToEnd.class);
-
- private static int firstPartCardinality = 4;
- private static int secondPartCardinality = 4;
- private static int thirdPartCardinality = 4;
- private static int colQualifiersTotal = 1;
- private static int totalFuzzyKeys = thirdPartCardinality / 2;
-
- private static String table = "TestFuzzyRowFilterEndToEnd";
-
- /**
- * @throws java.lang.Exception
- */
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- Configuration conf = TEST_UTIL.getConfiguration();
- conf.setInt("hbase.client.scanner.caching", 1000);
- conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
- // set no splits
- conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);
-
- TEST_UTIL.startMiniCluster();
- }
-
- /**
- * @throws java.lang.Exception
- */
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniCluster();
- }
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- // Nothing to do.
- }
-
- /**
- * @throws java.lang.Exception
- */
- @After
- public void tearDown() throws Exception {
- // Nothing to do.
- }
-
- @Test
- public void testEndToEnd() throws Exception {
- String cf = "f";
-
- HTable ht = TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
-
- // 10 byte row key - (2 bytes 4 bytes 4 bytes)
- // 4 byte qualifier
- // 4 byte value
-
- for (int i0 = 0; i0 < firstPartCardinality; i0++) {
-
- for (int i1 = 0; i1 < secondPartCardinality; i1++) {
-
- for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
- byte[] rk = new byte[10];
-
- ByteBuffer buf = ByteBuffer.wrap(rk);
- buf.clear();
- buf.putShort((short) i0);
- buf.putInt(i1);
- buf.putInt(i2);
- for (int c = 0; c < colQualifiersTotal; c++) {
- byte[] cq = new byte[4];
- Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
-
- Put p = new Put(rk);
- p.setDurability(Durability.SKIP_WAL);
- p.add(cf.getBytes(), cq, Bytes.toBytes(c));
- ht.put(p);
- }
- }
- }
- }
-
- TEST_UTIL.flush();
-
- // test passes
- runTest1(ht);
- runTest2(ht);
-
- }
-
- private void runTest1(HTable hTable) throws IOException {
- // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
-
- byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
-
- List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
- for (int i = 0; i < totalFuzzyKeys; i++) {
- byte[] fuzzyKey = new byte[10];
- ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
- buf.clear();
- buf.putShort((short) 2);
- for (int j = 0; j < 4; j++) {
- buf.put(fuzzyValue);
- }
- buf.putInt(i);
-
- Pair<byte[], byte[]> pair = Pair.newPair(fuzzyKey, mask);
- list.add(pair);
- }
-
- int expectedSize = secondPartCardinality * totalFuzzyKeys * colQualifiersTotal;
- FuzzyRowFilterV2 fuzzyRowFilter0 = new FuzzyRowFilterV2(list);
- // Filters are not stateless - we can't reuse them
- FuzzyRowFilterV2 fuzzyRowFilter1 = new FuzzyRowFilterV2(list);
-
- // regular test
- runScanner(hTable, expectedSize, fuzzyRowFilter0);
- // optimized from block cache
- runScanner(hTable, expectedSize, fuzzyRowFilter1);
-
- }
-
- private void runTest2(HTable hTable) throws IOException {
- // [0, 0, ?, ?, ?, ?, 0, 0, 0, 0] , [0, 1, ?, ?, ?, ?, 0, 0, 0, 1]...
-
- byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
-
- List<Pair<byte[], byte[]>> list = new ArrayList<Pair<byte[], byte[]>>();
-
- for (int i = 0; i < totalFuzzyKeys; i++) {
- byte[] fuzzyKey = new byte[10];
- ByteBuffer buf = ByteBuffer.wrap(fuzzyKey);
- buf.clear();
- buf.putShort((short) (i * 2));
- for (int j = 0; j < 4; j++) {
- buf.put(fuzzyValue);
- }
- buf.putInt(i * 2);
-
- Pair<byte[], byte[]> pair = Pair.newPair(fuzzyKey, mask);
- list.add(pair);
- }
-
- int expectedSize = totalFuzzyKeys * secondPartCardinality * colQualifiersTotal;
-
- FuzzyRowFilterV2 fuzzyRowFilter0 = new FuzzyRowFilterV2(list);
- // Filters are not stateless - we can't reuse them
- FuzzyRowFilterV2 fuzzyRowFilter1 = new FuzzyRowFilterV2(list);
-
- // regular test
- runScanner(hTable, expectedSize, fuzzyRowFilter0);
- // optimized from block cache
- runScanner(hTable, expectedSize, fuzzyRowFilter1);
-
- }
-
- private void runScanner(HTable hTable, int expectedSize, Filter filter) throws IOException {
-
- String cf = "f";
- Scan scan = new Scan();
- scan.addFamily(cf.getBytes());
- scan.setFilter(filter);
- List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
- HRegion first = regions.get(0);
- first.getScanner(scan);
- RegionScanner scanner = first.getScanner(scan);
- List<Cell> results = new ArrayList<Cell>();
- // Result result;
- long timeBeforeScan = System.currentTimeMillis();
- int found = 0;
- while (scanner.next(results)) {
- found += results.size();
- results.clear();
- }
- found += results.size();
- long scanTime = System.currentTimeMillis() - timeBeforeScan;
- scanner.close();
-
- LOG.info("\nscan time = " + scanTime + "ms");
- LOG.info("found " + found + " results\n");
-
- assertEquals(expectedSize, found);
- }
-
- @SuppressWarnings("deprecation")
- @Test
- public void testFilterList() throws Exception {
- String cf = "f";
- String table = "TestFuzzyRowFiltersInFilterList";
- HTable ht = TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);
-
- // 10 byte row key - (2 bytes 4 bytes 4 bytes)
- // 4 byte qualifier
- // 4 byte value
-
- for (int i1 = 0; i1 < 5; i1++) {
- for (int i2 = 0; i2 < 5; i2++) {
- byte[] rk = new byte[10];
-
- ByteBuffer buf = ByteBuffer.wrap(rk);
- buf.clear();
- buf.putShort((short) 2);
- buf.putInt(i1);
- buf.putInt(i2);
-
- // Each row contains 5 columns
- for (int c = 0; c < 5; c++) {
- byte[] cq = new byte[4];
- Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);
-
- Put p = new Put(rk);
- p.setDurability(Durability.SKIP_WAL);
- p.add(cf.getBytes(), cq, Bytes.toBytes(c));
- ht.put(p);
- LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: " + Bytes.toStringBinary(cq));
- }
- }
- }
-
- TEST_UTIL.flush();
-
- // test passes if we get back 5 KV's (1 row)
- runTest(ht, 5);
-
- }
-
- @SuppressWarnings("unchecked")
- private void runTest(HTable hTable, int expectedSize) throws IOException {
- // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1]
- byte[] fuzzyKey1 = new byte[10];
- ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1);
- buf.clear();
- buf.putShort((short) 2);
- for (int i = 0; i < 4; i++)
- buf.put(fuzzyValue);
- buf.putInt((short) 1);
- byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 };
-
- byte[] fuzzyKey2 = new byte[10];
- buf = ByteBuffer.wrap(fuzzyKey2);
- buf.clear();
- buf.putShort((short) 2);
- buf.putInt((short) 2);
- for (int i = 0; i < 4; i++)
- buf.put(fuzzyValue);
-
- byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 };
-
- Pair<byte[], byte[]> pair1 = Pair.newPair(fuzzyKey1, mask1);
- Pair<byte[], byte[]> pair2 = Pair.newPair(fuzzyKey2, mask2);
-
- FuzzyRowFilterV2 fuzzyRowFilter1 = new FuzzyRowFilterV2(Lists.newArrayList(pair1));
- FuzzyRowFilterV2 fuzzyRowFilter2 = new FuzzyRowFilterV2(Lists.newArrayList(pair2));
- // regular test - we expect 1 row back (5 KVs)
- runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2);
- }
-
- private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2) throws IOException {
- String cf = "f";
- Scan scan = new Scan();
- scan.addFamily(cf.getBytes());
- FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
- scan.setFilter(filterList);
-
- ResultScanner scanner = hTable.getScanner(scan);
- List<Cell> results = new ArrayList<Cell>();
- Result result;
- long timeBeforeScan = System.currentTimeMillis();
- while ((result = scanner.next()) != null) {
- for (Cell kv : result.listCells()) {
- LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: " + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
- results.add(kv);
- }
- }
- long scanTime = System.currentTimeMillis() - timeBeforeScan;
- scanner.close();
-
- LOG.info("scan time = " + scanTime + "ms");
- LOG.info("found " + results.size() + " results");
-
- assertEquals(expectedSize, results.size());
- }
-}
[16/18] kylin git commit: KYLIN-2217 fix NPE in
FactDistinctColumnsReducer
Posted by li...@apache.org.
KYLIN-2217 fix NPE in FactDistinctColumnsReducer
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c39fcb0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c39fcb0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c39fcb0
Branch: refs/heads/master-hbase1.x
Commit: 4c39fcb0eaab053d43554ac6add377343d585065
Parents: c0c56f4
Author: Li Yang <li...@apache.org>
Authored: Thu Dec 8 11:14:31 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 8 11:14:31 2016 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c39fcb0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index a3e61a1..776d750 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -117,7 +117,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
isPartitionCol = true;
col = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
if (col == null) {
- isPartitionCol = false;
logger.info("Do not have partition col. This reducer will keep empty");
}
colValues = Lists.newLinkedList();
@@ -283,9 +282,9 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
cuboidHLLMap, samplingPercentage, mapperNumber, mapperOverlapRatio);
} else if (isPartitionCol) {
// partition col
- if (col == null)
- return;
- outputPartitionInfo(context);
+ if (col != null) {
+ outputPartitionInfo(context);
+ }
} else {
// normal col
if (isReducerLocalBuildDict) {
[08/18] kylin git commit: KYLIN-2254 allow 2 columns in a
CompareTupleFilter
Posted by li...@apache.org.
KYLIN-2254 allow 2 columns in a CompareTupleFilter
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/350fad8f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/350fad8f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/350fad8f
Branch: refs/heads/master-hbase1.x
Commit: 350fad8f248988596c05b080b3c5a68b77b790a1
Parents: b1fd467
Author: Li Yang <li...@apache.org>
Authored: Wed Dec 7 17:51:26 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 8 09:47:27 2016 +0800
----------------------------------------------------------------------
.../metadata/filter/CompareTupleFilter.java | 23 ++++++------
.../kylin/metadata/model/FunctionDesc.java | 2 +-
.../apache/kylin/metadata/model/TblColRef.java | 2 +-
.../resources/query/sql_subquery/query12.sql | 37 ++++++++++++++++++++
.../kylin/query/relnode/OLAPAggregateRel.java | 13 ++++---
.../kylin/query/relnode/OLAPFilterRel.java | 30 +++++-----------
6 files changed, 66 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index f2af735..d567c89 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -37,6 +37,7 @@ public class CompareTupleFilter extends TupleFilter {
// operand 1 is either a column or a function
private TblColRef column;
private FunctionTupleFilter function;
+ private TblColRef secondColumn;
// operand 2 is constants
private Set<Object> conditionValues;
@@ -73,15 +74,16 @@ public class CompareTupleFilter extends TupleFilter {
if (child instanceof ColumnTupleFilter) {
ColumnTupleFilter columnFilter = (ColumnTupleFilter) child;
if (this.column != null) {
- throw new IllegalStateException("Duplicate columns! old is " + column.getName() + " and new is " + columnFilter.getColumn().getName());
- }
- this.column = columnFilter.getColumn();
- // if value is before column, we need to reverse the operator. e.g. "1 >= c1" => "c1 <= 1"
- // children.size() > 1 means already added one conditionValue or dynamicVariable
- if (this.children.size() > 1 && needSwapOperator()) {
- this.operator = SWAP_OP_MAP.get(this.operator);
- TupleFilter last = this.children.remove(this.children.size() - 1);
- this.children.add(0, last);
+ this.secondColumn = columnFilter.getColumn();
+ } else {
+ this.column = columnFilter.getColumn();
+ // if value is before column, we need to reverse the operator. e.g. "1 >= c1" => "c1 <= 1"
+ // children.size() > 1 means already added one conditionValue or dynamicVariable
+ if (this.children.size() > 1 && needSwapOperator()) {
+ this.operator = SWAP_OP_MAP.get(this.operator);
+ TupleFilter last = this.children.remove(this.children.size() - 1);
+ this.children.add(0, last);
+ }
}
} else if (child instanceof ConstantTupleFilter) {
this.conditionValues.addAll(child.getValues());
@@ -212,7 +214,8 @@ public class CompareTupleFilter extends TupleFilter {
@Override
public boolean isEvaluable() {
- return ((function != null && function.isEvaluable()) || column != null) && !conditionValues.isEmpty();
+ return (column != null || (function != null && function.isEvaluable())) //
+ && !conditionValues.isEmpty() && secondColumn == null;
}
@SuppressWarnings({ "unchecked", "rawtypes" })
http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 832cb4a..9252c42 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -140,7 +140,7 @@ public class FunctionDesc {
} else if (isCount()) {
return "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same
} else {
- return getFullExpression().replaceAll("[(), ]", "_");
+ return getFullExpression().replaceAll("[(),. ]", "_");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 5d72c3f..fd0224f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -150,7 +150,7 @@ public class TblColRef implements Serializable {
return column.getType();
}
- public void markInnerColumn(InnerDataTypeEnum dataType) {
+ private void markInnerColumn(InnerDataTypeEnum dataType) {
this.column.setDatatype(dataType.getDataType());
this.column.getTable().setName(INNER_TABLE_NAME);
this.column.getTable().setDatabase("DEFAULT");
http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/kylin-it/src/test/resources/query/sql_subquery/query12.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query12.sql b/kylin-it/src/test/resources/query/sql_subquery/query12.sql
new file mode 100644
index 0000000..6ed45fa
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query12.sql
@@ -0,0 +1,37 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ f.lstg_format_name
+ ,sum(price) as sum_price
+FROM
+ test_kylin_fact f
+ inner join
+ (
+ select
+ lstg_format_name,
+ min(slr_segment_cd) as min_seg
+ from
+ test_kylin_fact
+ group by
+ lstg_format_name
+ ) t on f.lstg_format_name = t.lstg_format_name
+where
+ f.slr_segment_cd = min_seg
+group by
+ f.lstg_format_name
http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 24933f5..f5fa74d 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -192,17 +192,16 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
for (int i = 0; i < this.aggregations.size(); i++) {
FunctionDesc aggFunc = this.aggregations.get(i);
- TblColRef aggCol = null;
+ String aggOutName;
if (aggFunc.needRewriteField()) {
- aggCol = buildRewriteColumn(aggFunc);
+ aggOutName = aggFunc.getRewriteFieldName();
} else {
AggregateCall aggCall = this.rewriteAggCalls.get(i);
- if (!aggCall.getArgList().isEmpty()) {
- int index = aggCall.getArgList().get(0);
- aggCol = inputColumnRowType.getColumnByIndex(index);
- }
+ int index = aggCall.getArgList().get(0);
+ aggOutName = aggFunc.getExpression() + "_" + inputColumnRowType.getColumnByIndex(index).getIdentity() + "_";
}
- columns.add(aggCol);
+ TblColRef aggOutCol = TblColRef.newInnerColumn(aggOutName, TblColRef.InnerDataTypeEnum.LITERAL);
+ columns.add(aggOutCol);
}
return new ColumnRowType(columns);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/350fad8f/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 411142d..8023df4 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -20,6 +20,7 @@ package org.apache.kylin.query.relnode;
import java.util.GregorianCalendar;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -61,14 +62,13 @@ import org.apache.kylin.metadata.filter.DynamicTupleFilter;
import org.apache.kylin.metadata.filter.ExtractTupleFilter;
import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.filter.UnsupportedTupleFilter;
import org.apache.kylin.metadata.filter.function.Functions;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
/**
*/
@@ -78,6 +78,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
private final ColumnRowType inputRowType;
private final OLAPContext context;
+ private final Set<TblColRef> columnsInFilter = new HashSet<>();
public TupleFilterVisitor(ColumnRowType inputRowType, OLAPContext context) {
super(true);
@@ -227,7 +228,10 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
@Override
public TupleFilter visitInputRef(RexInputRef inputRef) {
TblColRef column = inputRowType.getColumnByIndex(inputRef.getIndex());
- context.allColumns.add(column);
+ if (!column.isInnerColumn()) {
+ context.allColumns.add(column);
+ columnsInFilter.add(column);
+ }
ColumnTupleFilter filter = new ColumnTupleFilter(column);
return filter;
}
@@ -320,25 +324,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
TupleFilterVisitor visitor = new TupleFilterVisitor(this.columnRowType, context);
context.filter = this.condition.accept(visitor);
- context.filterColumns = collectColumns(context.filter);
- }
-
- private Set<TblColRef> collectColumns(TupleFilter filter) {
- Set<TblColRef> ret = Sets.newHashSet();
- collectColumnsRecursively(filter, ret);
- return ret;
- }
-
- private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
- if (filter == null)
- return;
-
- if (filter instanceof ColumnTupleFilter) {
- collector.add(((ColumnTupleFilter) filter).getColumn());
- }
- for (TupleFilter child : filter.getChildren()) {
- collectColumnsRecursively(child, collector);
- }
+ context.filterColumns = visitor.columnsInFilter;
}
@Override
[06/18] kylin git commit: KYLIN-1875 minor, test fixes
Posted by li...@apache.org.
KYLIN-1875 minor, test fixes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/365caaeb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/365caaeb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/365caaeb
Branch: refs/heads/master-hbase1.x
Commit: 365caaebd841917f161dab37b0af7c055626194c
Parents: 92e4d46
Author: Yang Li <li...@apache.org>
Authored: Fri Dec 2 06:32:25 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Dec 7 17:53:46 2016 +0800
----------------------------------------------------------------------
.../metadata/filter/ColumnTupleFilter.java | 1 +
.../kylin/metadata/model/DataModelDesc.java | 2 +-
.../apache/kylin/metadata/model/TblColRef.java | 2 +-
.../template/model_desc/kylin_sales_model.json | 4 +++
.../apache/kylin/query/ITKylinQueryTest.java | 2 +-
.../resources/query/sql_subquery/query02.sql | 31 ++++++++++++++++++++
.../query/sql_subquery/query02.sql.disable | 25 ----------------
7 files changed, 39 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index c8a8f07..ecb8e61 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -121,6 +121,7 @@ public class ColumnTupleFilter extends TupleFilter {
}
}
+ @SuppressWarnings("deprecation")
@Override
public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 898ff74..3f868a2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -112,7 +112,7 @@ public class DataModelDesc extends RootPersistentEntity {
return name;
}
- // for test only
+ // for test mainly
@Deprecated
public void setName(String name) {
this.name = name;
http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index bf8d36b..5d72c3f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -84,7 +84,7 @@ public class TblColRef implements Serializable {
col.table = tableRef;
}
- // for test only
+ // for test mainly
public static TblColRef mockup(TableDesc table, int oneBasedColumnIndex, String name, String datatype) {
ColumnDesc desc = new ColumnDesc();
String id = "" + oneBasedColumnIndex;
http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/examples/sample_cube/template/model_desc/kylin_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 2d3cfb6..cce360f 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -18,6 +18,7 @@
}, {
"table" : "DEFAULT.KYLIN_ACCOUNT",
"alias" : "BUYER_ACCOUNT",
+ "kind" : "LOOKUP",
"join" : {
"type" : "inner",
"primary_key" : [ "ACCOUNT_ID" ],
@@ -26,6 +27,7 @@
}, {
"table" : "DEFAULT.KYLIN_ACCOUNT",
"alias" : "SELLER_ACCOUNT",
+ "kind" : "LOOKUP",
"join" : {
"type" : "inner",
"primary_key" : [ "ACCOUNT_ID" ],
@@ -34,6 +36,7 @@
}, {
"table" : "DEFAULT.KYLIN_COUNTRY",
"alias" : "BUYER_COUNTRY",
+ "kind" : "LOOKUP",
"join" : {
"type" : "inner",
"primary_key" : [ "COUNTRY" ],
@@ -42,6 +45,7 @@
}, {
"table" : "DEFAULT.KYLIN_COUNTRY",
"alias" : "SELLER_COUNTRY",
+ "kind" : "LOOKUP",
"join" : {
"type" : "inner",
"primary_key" : [ "COUNTRY" ],
http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 6d91753..90324b5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -48,7 +48,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
-//@Ignore("KylinQueryTest is contained by ITCombinationTest")
+@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
@Rule
http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/kylin-it/src/test/resources/query/sql_subquery/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
new file mode 100644
index 0000000..e6751b7
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT
+ week_beg_dt
+ ,sum(price) as sum_price
+FROM
+(
+ select
+ test_cal_dt.week_beg_dt
+ ,test_kylin_fact.price
+ from test_kylin_fact
+ inner join edw.test_cal_dt as test_cal_dt
+ on test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+) t
+group by week_beg_dt
http://git-wip-us.apache.org/repos/asf/kylin/blob/365caaeb/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disable
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disable b/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disable
deleted file mode 100644
index 968dbae..0000000
--- a/kylin-it/src/test/resources/query/sql_subquery/query02.sql.disable
+++ /dev/null
@@ -1,25 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-SELECT sum(1) AS "COL"
- FROM (
- select test_cal_dt.week_beg_dt, sum(test_kylin_fact.price) as sum_price
- from test_kylin_fact
- inner join test_cal_dt ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- group by test_cal_dt.week_beg_dt
- ) "TableauSQL"
[13/18] kylin git commit: KYLIN-2255 drop v1 coprocessor impl
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
deleted file mode 100644
index a900ea1..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.gridtable.StorageSideBehavior;
-import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-
-/**
- * @author yangli9
- *
- */
-public class AggregationScanner implements RegionScanner {
-
- private RegionScanner outerScanner;
- private StorageSideBehavior behavior;
-
- public AggregationScanner(CoprocessorRowType type, CoprocessorFilter filter, CoprocessorProjector groupBy, ObserverAggregators aggrs, RegionScanner innerScanner, StorageSideBehavior behavior) throws IOException {
-
- AggregateRegionObserver.LOG.info("Kylin Coprocessor start");
-
- this.behavior = behavior;
-
- ObserverAggregationCache aggCache;
- Stats stats = new Stats();
-
- aggCache = buildAggrCache(innerScanner, type, groupBy, aggrs, filter, stats);
- stats.countOutputRow(aggCache.getSize());
- this.outerScanner = aggCache.getScanner(innerScanner);
-
- AggregateRegionObserver.LOG.info("Kylin Coprocessor aggregation done: " + stats);
- }
-
- @SuppressWarnings("rawtypes")
- ObserverAggregationCache buildAggrCache(final RegionScanner innerScanner, CoprocessorRowType type, CoprocessorProjector projector, ObserverAggregators aggregators, CoprocessorFilter filter, Stats stats) throws IOException {
-
- ObserverAggregationCache aggCache = new ObserverAggregationCache(aggregators);
-
- ObserverTuple tuple = new ObserverTuple(type);
- boolean hasMore = true;
- List<Cell> results = new ArrayList<Cell>();
- byte meaninglessByte = 0;
-
- while (hasMore) {
- results.clear();
- hasMore = innerScanner.nextRaw(results);
- if (results.isEmpty())
- continue;
-
- if (stats != null)
- stats.countInputRow(results);
-
- Cell cell = results.get(0);
- tuple.setUnderlying(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
-
- if (behavior == StorageSideBehavior.SCAN) {
- //touch every byte of the cell so that the cost of scanning will be trully reflected
- int endIndex = cell.getRowOffset() + cell.getRowLength();
- for (int i = cell.getRowOffset(); i < endIndex; ++i) {
- meaninglessByte += cell.getRowArray()[i];
- }
- } else {
- if (behavior.filterToggledOn()) {
- if (filter != null && filter.evaluate(tuple) == false)
- continue;
-
- if (behavior.aggrToggledOn()) {
- AggrKey aggKey = projector.getAggrKey(results);
- MeasureAggregator[] bufs = aggCache.getBuffer(aggKey);
- aggregators.aggregate(bufs, results);
-
- if (behavior.ordinal() >= StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.ordinal()) {
- aggCache.checkMemoryUsage();
- }
- }
- }
- }
- }
-
- if (behavior == StorageSideBehavior.SCAN) {
- System.out.println("meaningless byte is now " + meaninglessByte);
- }
-
- return aggCache;
- }
-
- @Override
- public boolean next(List<Cell> results) throws IOException {
- return outerScanner.next(results);
- }
-
- @Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- return outerScanner.next(result, limit);
- }
-
- @Override
- public boolean nextRaw(List<Cell> result) throws IOException {
- return outerScanner.nextRaw(result);
- }
-
- @Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return outerScanner.nextRaw(result, limit);
- }
-
- @Override
- public void close() throws IOException {
- outerScanner.close();
- }
-
- @Override
- public HRegionInfo getRegionInfo() {
- return outerScanner.getRegionInfo();
- }
-
- @Override
- public boolean isFilterDone() throws IOException {
- return outerScanner.isFilterDone();
- }
-
- @Override
- public boolean reseek(byte[] row) throws IOException {
- return outerScanner.reseek(row);
- }
-
- @Override
- public long getMaxResultSize() {
- return outerScanner.getMaxResultSize();
- }
-
- @Override
- public long getMvccReadPoint() {
- return outerScanner.getMvccReadPoint();
- }
-
- private static class Stats {
- long inputRows = 0;
- long inputBytes = 0;
- long outputRows = 0;
-
- // have no outputBytes because that requires actual serialize all the
- // aggregator buffers
-
- public void countInputRow(List<Cell> row) {
- inputRows++;
- inputBytes += row.get(0).getRowLength();
- for (int i = 0, n = row.size(); i < n; i++) {
- inputBytes += row.get(i).getValueLength();
- }
- }
-
- public void countOutputRow(long rowCount) {
- outputRows += rowCount;
- }
-
- public String toString() {
- double percent = (double) outputRows / inputRows * 100;
- return Math.round(percent) + "% = " + outputRows + " (out rows) / " + inputRows + " (in rows); in bytes = " + inputBytes + "; est. out bytes = " + Math.round(inputBytes * percent / 100);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
deleted file mode 100644
index 8404262..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
-import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
-
-/**
- * @author yangli9
- */
-@SuppressWarnings("rawtypes")
-public class ObserverAggregationCache extends AggregationCache {
-
- private final ObserverAggregators aggregators;
-
- public ObserverAggregationCache(ObserverAggregators aggregators) {
- this.aggregators = aggregators;
- }
-
- public RegionScanner getScanner(RegionScanner innerScanner) {
- return new AggregationRegionScanner(innerScanner);
- }
-
- @Override
- public MeasureAggregator[] createBuffer() {
- return aggregators.createBuffer();
- }
-
- private class AggregationRegionScanner implements RegionScanner {
-
- private final RegionScanner innerScanner;
- private final Iterator<Entry<AggrKey, MeasureAggregator[]>> iterator;
-
- public AggregationRegionScanner(RegionScanner innerScanner) {
- this.innerScanner = innerScanner;
- this.iterator = aggBufMap.entrySet().iterator();
- }
-
- @Override
- public boolean next(List<Cell> results) throws IOException {
- try {
- // AggregateRegionObserver.LOG.info("Kylin Scanner next()");
- boolean hasMore = false;
- if (iterator.hasNext()) {
- Entry<AggrKey, MeasureAggregator[]> entry = iterator.next();
- makeCells(entry, results);
- hasMore = iterator.hasNext();
- }
- // AggregateRegionObserver.LOG.info("Kylin Scanner next() done");
-
- return hasMore;
- } catch (Exception e) {
- throw new IOException("Error when calling next", e);
- }
- }
-
- private void makeCells(Entry<AggrKey, MeasureAggregator[]> entry, List<Cell> results) {
- byte[][] families = aggregators.getHColFamilies();
- byte[][] qualifiers = aggregators.getHColQualifiers();
- int nHCols = aggregators.getHColsNum();
-
- AggrKey rowKey = entry.getKey();
- MeasureAggregator[] aggBuf = entry.getValue();
- ByteBuffer[] rowValues = aggregators.getHColValues(aggBuf);
-
- if (nHCols == 0) {
- Cell keyValue = new KeyValue(rowKey.get(), rowKey.offset(), rowKey.length(), //
- null, 0, 0, //
- null, 0, 0, //
- HConstants.LATEST_TIMESTAMP, Type.Put, //
- null, 0, 0);
- results.add(keyValue);
- } else {
- for (int i = 0; i < nHCols; i++) {
- Cell keyValue = new KeyValue(rowKey.get(), rowKey.offset(), rowKey.length(), //
- families[i], 0, families[i].length, //
- qualifiers[i], 0, qualifiers[i].length, //
- HConstants.LATEST_TIMESTAMP, Type.Put, //
- rowValues[i].array(), 0, rowValues[i].position());
- results.add(keyValue);
- }
- }
- }
-
- @Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- return next(result);
- }
-
- @Override
- public boolean nextRaw(List<Cell> result) throws IOException {
- return next(result);
- }
-
- @Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return next(result);
- }
-
- @Override
- public void close() throws IOException {
- // AggregateRegionObserver.LOG.info("Kylin Scanner close()");
- innerScanner.close();
- // AggregateRegionObserver.LOG.info("Kylin Scanner close() done");
- }
-
- @Override
- public HRegionInfo getRegionInfo() {
- // AggregateRegionObserver.LOG.info("Kylin Scanner getRegionInfo()");
- return innerScanner.getRegionInfo();
- }
-
- @Override
- public long getMaxResultSize() {
- // AggregateRegionObserver.LOG.info("Kylin Scanner getMaxResultSize()");
- return Long.MAX_VALUE;
- }
-
- @Override
- public boolean isFilterDone() throws IOException {
- // AggregateRegionObserver.LOG.info("Kylin Scanner isFilterDone()");
- return false;
- }
-
- @Override
- public boolean reseek(byte[] row) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getMvccReadPoint() {
- // AggregateRegionObserver.LOG.info("Kylin Scanner getMvccReadPoint()");
- return Long.MAX_VALUE;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
deleted file mode 100644
index 29a30c1..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregators.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.MeasureAggregator;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.measure.MeasureTypeFactory;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-
-/**
- * @author yangli9
- */
-@SuppressWarnings({ "rawtypes", "unchecked" })
-public class ObserverAggregators {
-
- public static ObserverAggregators fromValueDecoders(Collection<RowValueDecoder> rowValueDecoders) {
-
- // each decoder represents one HBase column
- HCol[] hcols = new HCol[rowValueDecoders.size()];
- int i = 0;
- for (RowValueDecoder rowValueDecoder : rowValueDecoders) {
- hcols[i++] = buildHCol(rowValueDecoder.getHBaseColumn());
- }
-
- ObserverAggregators aggrs = new ObserverAggregators(hcols);
- return aggrs;
-
- }
-
- private static HCol buildHCol(HBaseColumnDesc desc) {
- byte[] family = Bytes.toBytes(desc.getColumnFamilyName());
- byte[] qualifier = Bytes.toBytes(desc.getQualifier());
- MeasureDesc[] measures = desc.getMeasures();
-
- String[] funcNames = new String[measures.length];
- String[] dataTypes = new String[measures.length];
-
- for (int i = 0; i < measures.length; i++) {
- funcNames[i] = measures[i].getFunction().getExpression();
- dataTypes[i] = measures[i].getFunction().getReturnType();
- }
-
- return new HCol(family, qualifier, funcNames, dataTypes);
- }
-
- public static byte[] serialize(ObserverAggregators o) {
- ByteBuffer buf = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE);
- serializer.serialize(o, buf);
- byte[] result = new byte[buf.position()];
- System.arraycopy(buf.array(), 0, result, 0, buf.position());
- return result;
- }
-
- public static ObserverAggregators deserialize(byte[] bytes) {
- return serializer.deserialize(ByteBuffer.wrap(bytes));
- }
-
- private static final BytesSerializer<ObserverAggregators> serializer = new BytesSerializer<ObserverAggregators>() {
-
- @Override
- public void serialize(ObserverAggregators value, ByteBuffer out) {
- BytesUtil.writeVInt(value.nHCols, out);
- for (int i = 0; i < value.nHCols; i++) {
- HCol col = value.hcols[i];
- BytesUtil.writeByteArray(col.family, out);
- BytesUtil.writeByteArray(col.qualifier, out);
- BytesUtil.writeAsciiStringArray(col.funcNames, out);
- BytesUtil.writeAsciiStringArray(col.dataTypes, out);
- }
- }
-
- @Override
- public ObserverAggregators deserialize(ByteBuffer in) {
- int nHCols = BytesUtil.readVInt(in);
- HCol[] hcols = new HCol[nHCols];
- for (int i = 0; i < nHCols; i++) {
- byte[] family = BytesUtil.readByteArray(in);
- byte[] qualifier = BytesUtil.readByteArray(in);
- String[] funcNames = BytesUtil.readAsciiStringArray(in);
- String[] dataTypes = BytesUtil.readAsciiStringArray(in);
- hcols[i] = new HCol(family, qualifier, funcNames, dataTypes);
- }
- return new ObserverAggregators(hcols);
- }
-
- };
-
- // ============================================================================
-
- final HCol[] hcols;
- final int nHCols;
- final ByteBuffer[] hColValues;
- final int nTotalMeasures;
-
- MeasureType[] measureTypes;
-
- public ObserverAggregators(HCol[] _hcols) {
- this.hcols = sort(_hcols);
- this.nHCols = hcols.length;
- this.hColValues = new ByteBuffer[nHCols];
-
- int nTotalMeasures = 0;
- for (HCol col : hcols)
- nTotalMeasures += col.nMeasures;
- this.nTotalMeasures = nTotalMeasures;
- }
-
- private HCol[] sort(HCol[] hcols) {
- HCol[] copy = Arrays.copyOf(hcols, hcols.length);
- Arrays.sort(copy, new Comparator<HCol>() {
- @Override
- public int compare(HCol o1, HCol o2) {
- int comp = Bytes.compareTo(o1.family, o2.family);
- if (comp != 0)
- return comp;
- comp = Bytes.compareTo(o1.qualifier, o2.qualifier);
- return comp;
- }
- });
- return copy;
- }
-
- public MeasureAggregator[] createBuffer() {
- if (measureTypes == null) {
- measureTypes = new MeasureType[nTotalMeasures];
- int i = 0;
- for (HCol col : hcols) {
- for (int j = 0; j < col.nMeasures; j++)
- measureTypes[i++] = MeasureTypeFactory.create(col.funcNames[j], DataType.getType(col.dataTypes[j]));
- }
- }
-
- MeasureAggregator[] aggrs = new MeasureAggregator[nTotalMeasures];
- for (int i = 0; i < nTotalMeasures; i++) {
- aggrs[i] = measureTypes[i].newAggregator();
- }
- return aggrs;
- }
-
- public void aggregate(MeasureAggregator[] measureAggrs, List<Cell> rowCells) {
- int i = 0;
- for (int ci = 0; ci < nHCols; ci++) {
- HCol col = hcols[ci];
- Cell cell = findCell(col, rowCells);
-
- if (cell == null) {
- i += col.nMeasures;
- continue;
- }
-
- ByteBuffer input = ByteBuffer.wrap(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
-
- col.measureCodec.decode(input, col.measureValues);
- for (int j = 0; j < col.nMeasures; j++)
- measureAggrs[i++].aggregate(col.measureValues[j]);
- }
- }
-
- private Cell findCell(HCol col, List<Cell> cells) {
- // cells are ordered by timestamp asc, thus search from back, first hit
- // is the latest version
- for (int i = cells.size() - 1; i >= 0; i--) {
- Cell cell = cells.get(i);
- if (match(col, cell)) {
- return cell;
- }
- }
- return null;
- }
-
- public static boolean match(HCol col, Cell cell) {
- return Bytes.compareTo(col.family, 0, col.family.length, cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()) == 0 && Bytes.compareTo(col.qualifier, 0, col.qualifier.length, cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()) == 0;
- }
-
- public int getHColsNum() {
- return nHCols;
- }
-
- public byte[][] getHColFamilies() {
- byte[][] result = new byte[nHCols][];
- for (int i = 0; i < nHCols; i++)
- result[i] = hcols[i].family;
- return result;
- }
-
- public byte[][] getHColQualifiers() {
- byte[][] result = new byte[nHCols][];
- for (int i = 0; i < nHCols; i++)
- result[i] = hcols[i].qualifier;
- return result;
- }
-
- public ByteBuffer[] getHColValues(MeasureAggregator[] aggrs) {
- int i = 0;
- for (int ci = 0; ci < nHCols; ci++) {
- HCol col = hcols[ci];
- for (int j = 0; j < col.nMeasures; j++)
- col.measureValues[j] = aggrs[i++].getState();
-
- hColValues[ci] = col.measureCodec.encode(col.measureValues);
- }
- return hColValues;
- }
-
- // ============================================================================
-
- public static class HCol {
- final byte[] family;
- final byte[] qualifier;
- final String[] funcNames;
- final String[] dataTypes;
- final int nMeasures;
-
- final BufferedMeasureCodec measureCodec;
- final Object[] measureValues;
-
- public HCol(byte[] bFamily, byte[] bQualifier, String[] funcNames, String[] dataTypes) {
- this.family = bFamily;
- this.qualifier = bQualifier;
- this.funcNames = funcNames;
- this.dataTypes = dataTypes;
- this.nMeasures = funcNames.length;
- assert funcNames.length == dataTypes.length;
-
- this.measureCodec = new BufferedMeasureCodec(dataTypes);
- this.measureValues = new Object[nMeasures];
- }
-
- @Override
- public String toString() {
- return "HCol [bFamily=" + Bytes.toString(family) + ", bQualifier=" + Bytes.toString(qualifier) + ", nMeasures=" + nMeasures + "]";
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
deleted file mode 100644
index 394b3e2..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.debug.BackdoorToggles;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.gridtable.StorageSideBehavior;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-import org.apache.kylin.storage.hbase.common.coprocessor.FilterDecorator;
-import org.apache.kylin.storage.hbase.cube.v1.RegionScannerAdapter;
-import org.apache.kylin.storage.hbase.cube.v1.ResultScannerAdapter;
-import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-/**
- * @author yangli9
- */
-public class ObserverEnabler {
-
- private static final Logger logger = LoggerFactory.getLogger(ObserverEnabler.class);
-
- static final String FORCE_COPROCESSOR = "forceObserver";
- static final Map<String, Boolean> CUBE_OVERRIDES = Maps.newConcurrentMap();
-
- public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, //
- Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, HTableInterface table, Scan scan) throws IOException {
-
- if (context.isCoprocessorEnabled() == false) {
- return table.getScanner(scan);
- }
-
- CoprocessorRowType type = CoprocessorRowType.fromCuboid(segment, cuboid);
- CoprocessorFilter filter = CoprocessorFilter.fromFilter(segment.getDimensionEncodingMap(), tupleFiler, FilterDecorator.FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT);
- CoprocessorProjector projector = CoprocessorProjector.makeForObserver(segment, cuboid, groupBy);
- ObserverAggregators aggrs = ObserverAggregators.fromValueDecoders(rowValueDecoders);
-
- boolean localCoprocessor = KylinConfig.getInstanceFromEnv().getQueryRunLocalCoprocessor() || BackdoorToggles.getRunLocalCoprocessor();
-
- if (localCoprocessor) {
- RegionScanner innerScanner = new RegionScannerAdapter(table.getScanner(scan));
- AggregationScanner aggrScanner = new AggregationScanner(type, filter, projector, aggrs, innerScanner, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM);
- return new ResultScannerAdapter(aggrScanner);
- } else {
-
- // debug/profiling purpose
- String toggle = BackdoorToggles.getCoprocessorBehavior();
- if (toggle == null) {
- toggle = StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString(); //default behavior
- } else {
- logger.info("The execution of this query will use " + toggle + " as observer's behavior");
- }
-
- scan.setAttribute(AggregateRegionObserver.COPROCESSOR_ENABLE, new byte[] { 0x01 });
- scan.setAttribute(AggregateRegionObserver.BEHAVIOR, toggle.getBytes());
- scan.setAttribute(AggregateRegionObserver.TYPE, CoprocessorRowType.serialize(type));
- scan.setAttribute(AggregateRegionObserver.PROJECTOR, CoprocessorProjector.serialize(projector));
- scan.setAttribute(AggregateRegionObserver.AGGREGATORS, ObserverAggregators.serialize(aggrs));
- scan.setAttribute(AggregateRegionObserver.FILTER, CoprocessorFilter.serialize(filter));
- return table.getScanner(scan);
- }
- }
-
- public static void enableCoprocessorIfBeneficial(CubeInstance cube, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {
- if (isCoprocessorBeneficial(cube, groupBy, rowValueDecoders, context)) {
- context.enableCoprocessor();
- }
- }
-
- private static boolean isCoprocessorBeneficial(CubeInstance cube, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {
-
- String forceFlag = System.getProperty(FORCE_COPROCESSOR);
- if (forceFlag != null) {
- boolean r = Boolean.parseBoolean(forceFlag);
- logger.info("Coprocessor is " + (r ? "enabled" : "disabled") + " according to sys prop " + FORCE_COPROCESSOR);
- return r;
- }
-
- Boolean cubeOverride = CUBE_OVERRIDES.get(cube.getName());
- if (cubeOverride != null) {
- boolean r = cubeOverride.booleanValue();
- logger.info("Coprocessor is " + (r ? "enabled" : "disabled") + " according to cube overrides");
- return r;
- }
-
- if (RowValueDecoder.hasMemHungryMeasures(rowValueDecoders)) {
- logger.info("Coprocessor is disabled because there is memory hungry count distinct");
- return false;
- }
-
- if (context.isExactAggregation()) {
- logger.info("Coprocessor is disabled because exactAggregation is true");
- return false;
- }
-
- Cuboid cuboid = context.getCuboid();
- Set<TblColRef> toAggr = Sets.newHashSet(cuboid.getAggregationColumns());
- toAggr.removeAll(groupBy);
- if (toAggr.isEmpty()) {
- logger.info("Coprocessor is disabled because no additional columns to aggregate");
- return false;
- }
-
- logger.info("Coprocessor is enabled to aggregate " + toAggr + ", returning " + groupBy);
- return true;
- }
-
- @SuppressWarnings("unused")
- private static int getBitsToScan(byte[] startKey, byte[] stopKey) {
- // find the first bit difference from the beginning
- int totalBits = startKey.length * 8;
- int bitsToScan = totalBits;
- for (int i = 0; i < totalBits; i++) {
- int byteIdx = i / 8;
- int bitIdx = 7 - i % 8;
- byte bitMask = (byte) (1 << bitIdx);
- if ((startKey[byteIdx] & bitMask) == (stopKey[byteIdx] & bitMask))
- bitsToScan--;
- else
- break;
- }
- return bitsToScan;
- }
-
- public static void forceCoprocessorOn() {
- System.setProperty(FORCE_COPROCESSOR, "true");
- }
-
- public static void forceCoprocessorOff() {
- System.setProperty(FORCE_COPROCESSOR, "false");
- }
-
- public static String getForceCoprocessor() {
- return System.getProperty(FORCE_COPROCESSOR);
- }
-
- public static void forceCoprocessorUnset() {
- System.clearProperty(FORCE_COPROCESSOR);
- }
-
- public static void updateCubeOverride(String cubeName, String force) {
- if ("null".equalsIgnoreCase(force) || "default".equalsIgnoreCase(force)) {
- CUBE_OVERRIDES.remove(cubeName);
- } else if ("true".equalsIgnoreCase(force)) {
- CUBE_OVERRIDES.put(cubeName, Boolean.TRUE);
- } else if ("false".equalsIgnoreCase(force)) {
- CUBE_OVERRIDES.put(cubeName, Boolean.FALSE);
- }
- }
-
- public static Map<String, Boolean> getCubeOverrides() {
- return CUBE_OVERRIDES;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java
deleted file mode 100644
index e2236d3..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverTuple.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-
-/**
- * A special kind of tuple that exposes column value (dictionary ID) directly on
- * top of row key.
- *
- * @author yangli9
- */
-public class ObserverTuple implements IEvaluatableTuple {
-
- final CoprocessorRowType type;
-
- ImmutableBytesWritable rowkey;
- String[] values;
-
- public ObserverTuple(CoprocessorRowType type) {
- this.type = type;
- this.rowkey = new ImmutableBytesWritable();
- this.values = new String[type.getColumnCount()];
- }
-
- public void setUnderlying(byte[] array, int offset, int length) {
- rowkey.set(array, offset, length);
- for (int i = 0; i < values.length; i++) {
- values[i] = null;
- }
- }
-
- private String getValueAt(int i) {
- int n = type.getColumnCount();
- if (i < 0 || i >= n)
- return null;
-
- if (values[i] == null) {
- values[i] = Dictionary.dictIdToString(rowkey.get(), rowkey.getOffset() + type.columnOffsets[i], type.columnSizes[i]);
- }
-
- return values[i];
- }
-
- @Override
- public Object getValue(TblColRef col) {
- int i = type.getColIndexByTblColRef(col);
- return getValueAt(i);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/FuzzyRowFilterV2.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/FuzzyRowFilterV2.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/FuzzyRowFilterV2.java
deleted file mode 100644
index ea33f9a..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/FuzzyRowFilterV2.java
+++ /dev/null
@@ -1,574 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v1.filter;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.PriorityQueue;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.util.ByteStringer;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- * This is optimized version of a standard FuzzyRowFilter Filters data based on fuzzy row key.
- * Performs fast-forwards during scanning. It takes pairs (row key, fuzzy info) to match row keys.
- * Where fuzzy info is a byte array with 0 or 1 as its values:
- * <ul>
- * <li>0 - means that this byte in provided row key is fixed, i.e. row key's byte at same position
- * must match</li>
- * <li>1 - means that this byte in provided row key is NOT fixed, i.e. row key's byte at this
- * position can be different from the one in provided row key</li>
- * </ul>
- * Example: Let's assume row key format is userId_actionId_year_month. Length of userId is fixed and
- * is 4, length of actionId is 2 and year and month are 4 and 2 bytes long respectively. Let's
- * assume that we need to fetch all users that performed certain action (encoded as "99") in Jan of
- * any year. Then the pair (row key, fuzzy info) would be the following: row key = "????_99_????_01"
- * (one can use any value instead of "?") fuzzy info =
- * "\x01\x01\x01\x01\x00\x00\x00\x00\x01\x01\x01\x01\x00\x00\x00" I.e. fuzzy info tells the matching
- * mask is "????_99_????_01", where at ? can be any value.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class FuzzyRowFilterV2 extends FilterBase {
- private List<Pair<byte[], byte[]>> fuzzyKeysData;
- private boolean done = false;
-
- /**
- * The index of a last successfully found matching fuzzy string (in fuzzyKeysData). We will start
- * matching next KV with this one. If they do not match then we will return back to the one-by-one
- * iteration over fuzzyKeysData.
- */
- private int lastFoundIndex = -1;
-
- /**
- * Row tracker (keeps all next rows after SEEK_NEXT_USING_HINT was returned)
- */
- private RowTracker tracker;
-
- public FuzzyRowFilterV2(List<Pair<byte[], byte[]>> fuzzyKeysData) {
- Pair<byte[], byte[]> p;
- for (int i = 0; i < fuzzyKeysData.size(); i++) {
- p = fuzzyKeysData.get(i);
- if (p.getFirst().length != p.getSecond().length) {
- Pair<String, String> readable = Pair.newPair(Bytes.toStringBinary(p.getFirst()), Bytes.toStringBinary(p.getSecond()));
- throw new IllegalArgumentException("Fuzzy pair lengths do not match: " + readable);
- }
- // update mask ( 0 -> -1 (0xff), 1 -> 0)
- p.setSecond(preprocessMask(p.getSecond()));
- preprocessSearchKey(p);
- }
- this.fuzzyKeysData = fuzzyKeysData;
- this.tracker = new RowTracker();
- }
-
- private void preprocessSearchKey(Pair<byte[], byte[]> p) {
- if (UnsafeAccess.isAvailable() == false) {
- // do nothing
- return;
- }
- byte[] key = p.getFirst();
- byte[] mask = p.getSecond();
- for (int i = 0; i < mask.length; i++) {
- // set non-fixed part of a search key to 0.
- if (mask[i] == 0)
- key[i] = 0;
- }
- }
-
- /**
- * We need to preprocess mask array, as since we treat 0's as unfixed positions and -1 (0xff) as
- * fixed positions
- * @param mask
- * @return mask array
- */
- private byte[] preprocessMask(byte[] mask) {
- if (UnsafeAccess.isAvailable() == false) {
- // do nothing
- return mask;
- }
- if (isPreprocessedMask(mask))
- return mask;
- for (int i = 0; i < mask.length; i++) {
- if (mask[i] == 0) {
- mask[i] = -1; // 0 -> -1
- } else if (mask[i] == 1) {
- mask[i] = 0;// 1 -> 0
- }
- }
- return mask;
- }
-
- private boolean isPreprocessedMask(byte[] mask) {
- for (int i = 0; i < mask.length; i++) {
- if (mask[i] != -1 && mask[i] != 0) {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public ReturnCode filterKeyValue(Cell c) {
- final int startIndex = lastFoundIndex >= 0 ? lastFoundIndex : 0;
- final int size = fuzzyKeysData.size();
- for (int i = startIndex; i < size + startIndex; i++) {
- final int index = i % size;
- Pair<byte[], byte[]> fuzzyData = fuzzyKeysData.get(index);
- SatisfiesCode satisfiesCode = satisfies(isReversed(), c.getRowArray(), c.getRowOffset(), c.getRowLength(), fuzzyData.getFirst(), fuzzyData.getSecond());
- if (satisfiesCode == SatisfiesCode.YES) {
- lastFoundIndex = index;
- return ReturnCode.INCLUDE;
- }
- }
- // NOT FOUND -> seek next using hint
- lastFoundIndex = -1;
- return ReturnCode.SEEK_NEXT_USING_HINT;
-
- }
-
- @Override
- public Cell getNextCellHint(Cell currentCell) {
- boolean result = tracker.updateTracker(currentCell);
- if (result == false) {
- done = true;
- return null;
- }
- byte[] nextRowKey = tracker.nextRow();
- return KeyValue.createFirstOnRow(nextRowKey);
- }
-
- /**
- * If we have multiple fuzzy keys, row tracker should improve overall performance. It calculates
- * all next rows (one per every fuzzy key) and put them (the fuzzy key is bundled) into a priority
- * queue so that the smallest row key always appears at queue head, which helps to decide the
- * "Next Cell Hint". As scanning going on, the number of candidate rows in the RowTracker will
- * remain the size of fuzzy keys until some of the fuzzy keys won't possibly have matches any
- * more.
- */
- private class RowTracker {
- private final PriorityQueue<Pair<byte[], Pair<byte[], byte[]>>> nextRows;
- private boolean initialized = false;
-
- RowTracker() {
- nextRows = new PriorityQueue<Pair<byte[], Pair<byte[], byte[]>>>(fuzzyKeysData.size(), new Comparator<Pair<byte[], Pair<byte[], byte[]>>>() {
- @Override
- public int compare(Pair<byte[], Pair<byte[], byte[]>> o1, Pair<byte[], Pair<byte[], byte[]>> o2) {
- int compare = Bytes.compareTo(o1.getFirst(), o2.getFirst());
- if (!isReversed()) {
- return compare;
- } else {
- return -compare;
- }
- }
- });
- }
-
- byte[] nextRow() {
- if (nextRows.isEmpty()) {
- throw new IllegalStateException("NextRows should not be empty, make sure to call nextRow() after updateTracker() return true");
- } else {
- return nextRows.peek().getFirst();
- }
- }
-
- boolean updateTracker(Cell currentCell) {
- if (!initialized) {
- for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
- updateWith(currentCell, fuzzyData);
- }
- initialized = true;
- } else {
- while (!nextRows.isEmpty() && !lessThan(currentCell, nextRows.peek().getFirst())) {
- Pair<byte[], Pair<byte[], byte[]>> head = nextRows.poll();
- Pair<byte[], byte[]> fuzzyData = head.getSecond();
- updateWith(currentCell, fuzzyData);
- }
- }
- return !nextRows.isEmpty();
- }
-
- boolean lessThan(Cell currentCell, byte[] nextRowKey) {
- int compareResult = Bytes.compareTo(currentCell.getRowArray(), currentCell.getRowOffset(), currentCell.getRowLength(), nextRowKey, 0, nextRowKey.length);
- return (!isReversed() && compareResult < 0) || (isReversed() && compareResult > 0);
- }
-
- void updateWith(Cell currentCell, Pair<byte[], byte[]> fuzzyData) {
- byte[] nextRowKeyCandidate = getNextForFuzzyRule(isReversed(), currentCell.getRowArray(), currentCell.getRowOffset(), currentCell.getRowLength(), fuzzyData.getFirst(), fuzzyData.getSecond());
- if (nextRowKeyCandidate != null) {
- nextRows.add(Pair.newPair(nextRowKeyCandidate, fuzzyData));
- }
- }
-
- }
-
- @Override
- public boolean filterAllRemaining() {
- return done;
- }
-
- /**
- * @return The filter serialized using pb
- */
- public byte[] toByteArray() {
- FilterProtosExt.FuzzyRowFilterV2.Builder builder = FilterProtosExt.FuzzyRowFilterV2.newBuilder();
- for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
- FilterProtosExt.BytesBytesPair.Builder bbpBuilder = FilterProtosExt.BytesBytesPair.newBuilder();
- bbpBuilder.setFirst(ByteStringer.wrap(fuzzyData.getFirst()));
- bbpBuilder.setSecond(ByteStringer.wrap(fuzzyData.getSecond()));
- builder.addFuzzyKeysData(bbpBuilder);
- }
- return builder.build().toByteArray();
- }
-
- public static FuzzyRowFilterV2 parseFrom(final byte[] pbBytes) throws DeserializationException {
- FilterProtosExt.FuzzyRowFilterV2 proto;
- try {
- proto = FilterProtosExt.FuzzyRowFilterV2.parseFrom(pbBytes);
- } catch (InvalidProtocolBufferException e) {
- throw new DeserializationException(e);
- }
- int count = proto.getFuzzyKeysDataCount();
- ArrayList<Pair<byte[], byte[]>> fuzzyKeysData = new ArrayList<Pair<byte[], byte[]>>(count);
- for (int i = 0; i < count; ++i) {
- FilterProtosExt.BytesBytesPair current = proto.getFuzzyKeysData(i);
- byte[] keyBytes = current.getFirst().toByteArray();
- byte[] keyMeta = current.getSecond().toByteArray();
- fuzzyKeysData.add(Pair.newPair(keyBytes, keyMeta));
- }
- return new FuzzyRowFilterV2(fuzzyKeysData);
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder();
- sb.append("FuzzyRowFilter");
- sb.append("{fuzzyKeysData=");
- for (Pair<byte[], byte[]> fuzzyData : fuzzyKeysData) {
- sb.append('{').append(Bytes.toStringBinary(fuzzyData.getFirst())).append(":");
- sb.append(Bytes.toStringBinary(fuzzyData.getSecond())).append('}');
- }
- sb.append("}, ");
- return sb.toString();
- }
-
- // Utility methods
-
- static enum SatisfiesCode {
- /** row satisfies fuzzy rule */
- YES,
- /** row doesn't satisfy fuzzy rule, but there's possible greater row that does */
- NEXT_EXISTS,
- /** row doesn't satisfy fuzzy rule and there's no greater row that does */
- NO_NEXT
- }
-
- @VisibleForTesting
- static SatisfiesCode satisfies(boolean reverse, byte[] row, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
- return satisfies(reverse, row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta);
- }
-
- static SatisfiesCode satisfies(boolean reverse, byte[] row, int offset, int length, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
-
- if (UnsafeAccess.isAvailable() == false) {
- return satisfiesNoUnsafe(reverse, row, offset, length, fuzzyKeyBytes, fuzzyKeyMeta);
- }
-
- if (row == null) {
- // do nothing, let scan to proceed
- return SatisfiesCode.YES;
- }
- length = Math.min(length, fuzzyKeyBytes.length);
- int numWords = length / Bytes.SIZEOF_LONG;
-
- int j = numWords << 3; // numWords * SIZEOF_LONG;
-
- for (int i = 0; i < j; i += Bytes.SIZEOF_LONG) {
- long fuzzyBytes = UnsafeAccess.toLong(fuzzyKeyBytes, i);
- long fuzzyMeta = UnsafeAccess.toLong(fuzzyKeyMeta, i);
- long rowValue = UnsafeAccess.toLong(row, offset + i);
- if ((rowValue & fuzzyMeta) != (fuzzyBytes)) {
- // We always return NEXT_EXISTS
- return SatisfiesCode.NEXT_EXISTS;
- }
- }
-
- int off = j;
-
- if (length - off >= Bytes.SIZEOF_INT) {
- int fuzzyBytes = UnsafeAccess.toInt(fuzzyKeyBytes, off);
- int fuzzyMeta = UnsafeAccess.toInt(fuzzyKeyMeta, off);
- int rowValue = UnsafeAccess.toInt(row, offset + off);
- if ((rowValue & fuzzyMeta) != (fuzzyBytes)) {
- // We always return NEXT_EXISTS
- return SatisfiesCode.NEXT_EXISTS;
- }
- off += Bytes.SIZEOF_INT;
- }
-
- if (length - off >= Bytes.SIZEOF_SHORT) {
- short fuzzyBytes = UnsafeAccess.toShort(fuzzyKeyBytes, off);
- short fuzzyMeta = UnsafeAccess.toShort(fuzzyKeyMeta, off);
- short rowValue = UnsafeAccess.toShort(row, offset + off);
- if ((rowValue & fuzzyMeta) != (fuzzyBytes)) {
- // We always return NEXT_EXISTS
- // even if it does not (in this case getNextForFuzzyRule
- // will return null)
- return SatisfiesCode.NEXT_EXISTS;
- }
- off += Bytes.SIZEOF_SHORT;
- }
-
- if (length - off >= Bytes.SIZEOF_BYTE) {
- int fuzzyBytes = fuzzyKeyBytes[off] & 0xff;
- int fuzzyMeta = fuzzyKeyMeta[off] & 0xff;
- int rowValue = row[offset + off] & 0xff;
- if ((rowValue & fuzzyMeta) != (fuzzyBytes)) {
- // We always return NEXT_EXISTS
- return SatisfiesCode.NEXT_EXISTS;
- }
- }
- return SatisfiesCode.YES;
- }
-
- static SatisfiesCode satisfiesNoUnsafe(boolean reverse, byte[] row, int offset, int length, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
- if (row == null) {
- // do nothing, let scan to proceed
- return SatisfiesCode.YES;
- }
-
- Order order = Order.orderFor(reverse);
- boolean nextRowKeyCandidateExists = false;
-
- for (int i = 0; i < fuzzyKeyMeta.length && i < length; i++) {
- // First, checking if this position is fixed and not equals the given one
- boolean byteAtPositionFixed = fuzzyKeyMeta[i] == 0;
- boolean fixedByteIncorrect = byteAtPositionFixed && fuzzyKeyBytes[i] != row[i + offset];
- if (fixedByteIncorrect) {
- // in this case there's another row that satisfies fuzzy rule and bigger than this row
- if (nextRowKeyCandidateExists) {
- return SatisfiesCode.NEXT_EXISTS;
- }
-
- // If this row byte is less than fixed then there's a byte array bigger than
- // this row and which satisfies the fuzzy rule. Otherwise there's no such byte array:
- // this row is simply bigger than any byte array that satisfies the fuzzy rule
- boolean rowByteLessThanFixed = (row[i + offset] & 0xFF) < (fuzzyKeyBytes[i] & 0xFF);
- if (rowByteLessThanFixed && !reverse) {
- return SatisfiesCode.NEXT_EXISTS;
- } else if (!rowByteLessThanFixed && reverse) {
- return SatisfiesCode.NEXT_EXISTS;
- } else {
- return SatisfiesCode.NO_NEXT;
- }
- }
-
- // Second, checking if this position is not fixed and byte value is not the biggest. In this
- // case there's a byte array bigger than this row and which satisfies the fuzzy rule. To get
- // bigger byte array that satisfies the rule we need to just increase this byte
- // (see the code of getNextForFuzzyRule below) by one.
- // Note: if non-fixed byte is already at biggest value, this doesn't allow us to say there's
- // bigger one that satisfies the rule as it can't be increased.
- if (fuzzyKeyMeta[i] == 1 && !order.isMax(fuzzyKeyBytes[i])) {
- nextRowKeyCandidateExists = true;
- }
- }
- return SatisfiesCode.YES;
- }
-
- @VisibleForTesting
- static byte[] getNextForFuzzyRule(byte[] row, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
- return getNextForFuzzyRule(false, row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta);
- }
-
- @VisibleForTesting
- static byte[] getNextForFuzzyRule(boolean reverse, byte[] row, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
- return getNextForFuzzyRule(reverse, row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta);
- }
-
- /** Abstracts directional comparisons based on scan direction. */
- private enum Order {
- ASC {
- public boolean lt(int lhs, int rhs) {
- return lhs < rhs;
- }
-
- public boolean gt(int lhs, int rhs) {
- return lhs > rhs;
- }
-
- public byte inc(byte val) {
- // TODO: what about over/underflow?
- return (byte) (val + 1);
- }
-
- public boolean isMax(byte val) {
- return val == (byte) 0xff;
- }
-
- public byte min() {
- return 0;
- }
- },
- DESC {
- public boolean lt(int lhs, int rhs) {
- return lhs > rhs;
- }
-
- public boolean gt(int lhs, int rhs) {
- return lhs < rhs;
- }
-
- public byte inc(byte val) {
- // TODO: what about over/underflow?
- return (byte) (val - 1);
- }
-
- public boolean isMax(byte val) {
- return val == 0;
- }
-
- public byte min() {
- return (byte) 0xFF;
- }
- };
-
- public static Order orderFor(boolean reverse) {
- return reverse ? DESC : ASC;
- }
-
- /** Returns true when {@code lhs < rhs}. */
- public abstract boolean lt(int lhs, int rhs);
-
- /** Returns true when {@code lhs > rhs}. */
- public abstract boolean gt(int lhs, int rhs);
-
- /** Returns {@code val} incremented by 1. */
- public abstract byte inc(byte val);
-
- /** Return true when {@code val} is the maximum value */
- public abstract boolean isMax(byte val);
-
- /** Return the minimum value according to this ordering scheme. */
- public abstract byte min();
- }
-
- /**
- * @return greater byte array than given (row) which satisfies the fuzzy rule if it exists, null
- * otherwise
- */
- @VisibleForTesting
- static byte[] getNextForFuzzyRule(boolean reverse, byte[] row, int offset, int length, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) {
- // To find out the next "smallest" byte array that satisfies fuzzy rule and "greater" than
- // the given one we do the following:
- // 1. setting values on all "fixed" positions to the values from fuzzyKeyBytes
- // 2. if during the first step given row did not increase, then we increase the value at
- // the first "non-fixed" position (where it is not maximum already)
-
- // It is easier to perform this by using fuzzyKeyBytes copy and setting "non-fixed" position
- // values than otherwise.
- byte[] result = Arrays.copyOf(fuzzyKeyBytes, length > fuzzyKeyBytes.length ? length : fuzzyKeyBytes.length);
- if (reverse && length > fuzzyKeyBytes.length) {
- // we need trailing 0xff's instead of trailing 0x00's
- for (int i = fuzzyKeyBytes.length; i < result.length; i++) {
- result[i] = (byte) 0xFF;
- }
- }
- int toInc = -1;
- final Order order = Order.orderFor(reverse);
-
- boolean increased = false;
- for (int i = 0; i < result.length; i++) {
- if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 0 /* non-fixed */) {
- result[i] = row[offset + i];
- if (!order.isMax(row[offset + i])) {
- // this is "non-fixed" position and is not at max value, hence we can increase it
- toInc = i;
- }
- } else if (i < fuzzyKeyMeta.length && fuzzyKeyMeta[i] == -1 /* fixed */) {
- if (order.lt((row[i + offset] & 0xFF), (fuzzyKeyBytes[i] & 0xFF))) {
- // if setting value for any fixed position increased the original array,
- // we are OK
- increased = true;
- break;
- }
-
- if (order.gt((row[i + offset] & 0xFF), (fuzzyKeyBytes[i] & 0xFF))) {
- // if setting value for any fixed position makes array "smaller", then just stop:
- // in case we found some non-fixed position to increase we will do it, otherwise
- // there's no "next" row key that satisfies fuzzy rule and "greater" than given row
- break;
- }
- }
- }
-
- if (!increased) {
- if (toInc < 0) {
- return null;
- }
- result[toInc] = order.inc(result[toInc]);
-
- // Setting all "non-fixed" positions to zeroes to the right of the one we increased so
- // that found "next" row key is the smallest possible
- for (int i = toInc + 1; i < result.length; i++) {
- if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 0 /* non-fixed */) {
- result[i] = order.min();
- }
- }
- }
-
- return result;
- }
-
- /**
- * @return true if and only if the fields of the filter that are serialized are equal to the
- * corresponding fields in other. Used for testing.
- */
- boolean areSerializedFieldsEqual(Filter o) {
- if (o == this)
- return true;
- if (!(o instanceof FuzzyRowFilterV2))
- return false;
-
- FuzzyRowFilterV2 other = (FuzzyRowFilterV2) o;
- if (this.fuzzyKeysData.size() != other.fuzzyKeysData.size())
- return false;
- for (int i = 0; i < fuzzyKeysData.size(); ++i) {
- Pair<byte[], byte[]> thisData = this.fuzzyKeysData.get(i);
- Pair<byte[], byte[]> otherData = other.fuzzyKeysData.get(i);
- if (!(Bytes.equals(thisData.getFirst(), otherData.getFirst()) && Bytes.equals(thisData.getSecond(), otherData.getSecond()))) {
- return false;
- }
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/UnsafeAccess.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/UnsafeAccess.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/UnsafeAccess.java
deleted file mode 100644
index 34328ef..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/UnsafeAccess.java
+++ /dev/null
@@ -1,433 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v1.filter;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.lang.reflect.Field;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import sun.misc.Unsafe;
-import sun.nio.ch.DirectBuffer;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class UnsafeAccess {
-
- private static final Log LOG = LogFactory.getLog(UnsafeAccess.class);
-
- static final Unsafe theUnsafe;
-
- /** The offset to the first element in a byte array. */
- static final long BYTE_ARRAY_BASE_OFFSET;
-
- static final boolean littleEndian = ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN);
-
- static {
- theUnsafe = (Unsafe) AccessController.doPrivileged(new PrivilegedAction<Object>() {
- @Override
- public Object run() {
- try {
- Field f = Unsafe.class.getDeclaredField("theUnsafe");
- f.setAccessible(true);
- return f.get(null);
- } catch (Throwable e) {
- LOG.warn("sun.misc.Unsafe is not accessible", e);
- }
- return null;
- }
- });
-
- if (theUnsafe != null) {
- BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
- } else {
- BYTE_ARRAY_BASE_OFFSET = -1;
- }
- }
-
- private UnsafeAccess() {
- }
-
- /**
- * @return true when the running JVM is having sun's Unsafe package available in it.
- */
- public static boolean isAvailable() {
- return theUnsafe != null;
- }
-
- // APIs to read primitive data from a byte[] using Unsafe way
- /**
- * Converts a byte array to a short value considering it was written in big-endian format.
- * @param bytes byte array
- * @param offset offset into array
- * @return the short value
- */
- public static short toShort(byte[] bytes, int offset) {
- if (littleEndian) {
- return Short.reverseBytes(theUnsafe.getShort(bytes, offset + BYTE_ARRAY_BASE_OFFSET));
- } else {
- return theUnsafe.getShort(bytes, offset + BYTE_ARRAY_BASE_OFFSET);
- }
- }
-
- /**
- * Converts a byte array to an int value considering it was written in big-endian format.
- * @param bytes byte array
- * @param offset offset into array
- * @return the int value
- */
- public static int toInt(byte[] bytes, int offset) {
- if (littleEndian) {
- return Integer.reverseBytes(theUnsafe.getInt(bytes, offset + BYTE_ARRAY_BASE_OFFSET));
- } else {
- return theUnsafe.getInt(bytes, offset + BYTE_ARRAY_BASE_OFFSET);
- }
- }
-
- /**
- * Converts a byte array to a long value considering it was written in big-endian format.
- * @param bytes byte array
- * @param offset offset into array
- * @return the long value
- */
- public static long toLong(byte[] bytes, int offset) {
- if (littleEndian) {
- return Long.reverseBytes(theUnsafe.getLong(bytes, offset + BYTE_ARRAY_BASE_OFFSET));
- } else {
- return theUnsafe.getLong(bytes, offset + BYTE_ARRAY_BASE_OFFSET);
- }
- }
-
- // APIs to write primitive data to a byte[] using Unsafe way
- /**
- * Put a short value out to the specified byte array position in big-endian format.
- * @param bytes the byte array
- * @param offset position in the array
- * @param val short to write out
- * @return incremented offset
- */
- public static int putShort(byte[] bytes, int offset, short val) {
- if (littleEndian) {
- val = Short.reverseBytes(val);
- }
- theUnsafe.putShort(bytes, offset + BYTE_ARRAY_BASE_OFFSET, val);
- return offset + Bytes.SIZEOF_SHORT;
- }
-
- /**
- * Put an int value out to the specified byte array position in big-endian format.
- * @param bytes the byte array
- * @param offset position in the array
- * @param val int to write out
- * @return incremented offset
- */
- public static int putInt(byte[] bytes, int offset, int val) {
- if (littleEndian) {
- val = Integer.reverseBytes(val);
- }
- theUnsafe.putInt(bytes, offset + BYTE_ARRAY_BASE_OFFSET, val);
- return offset + Bytes.SIZEOF_INT;
- }
-
- /**
- * Put a long value out to the specified byte array position in big-endian format.
- * @param bytes the byte array
- * @param offset position in the array
- * @param val long to write out
- * @return incremented offset
- */
- public static int putLong(byte[] bytes, int offset, long val) {
- if (littleEndian) {
- val = Long.reverseBytes(val);
- }
- theUnsafe.putLong(bytes, offset + BYTE_ARRAY_BASE_OFFSET, val);
- return offset + Bytes.SIZEOF_LONG;
- }
-
- // APIs to read primitive data from a ByteBuffer using Unsafe way
- /**
- * Reads a short value at the given buffer's offset considering it was written in big-endian
- * format.
- *
- * @param buf
- * @param offset
- * @return short value at offset
- */
- public static short toShort(ByteBuffer buf, int offset) {
- if (littleEndian) {
- return Short.reverseBytes(getAsShort(buf, offset));
- }
- return getAsShort(buf, offset);
- }
-
- /**
- * Reads bytes at the given offset as a short value.
- * @param buf
- * @param offset
- * @return short value at offset
- */
- static short getAsShort(ByteBuffer buf, int offset) {
- if (buf.isDirect()) {
- return theUnsafe.getShort(((DirectBuffer) buf).address() + offset);
- }
- return theUnsafe.getShort(buf.array(), BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset);
- }
-
- /**
- * Reads an int value at the given buffer's offset considering it was written in big-endian
- * format.
- *
- * @param buf
- * @param offset
- * @return int value at offset
- */
- public static int toInt(ByteBuffer buf, int offset) {
- if (littleEndian) {
- return Integer.reverseBytes(getAsInt(buf, offset));
- }
- return getAsInt(buf, offset);
- }
-
- /**
- * Reads bytes at the given offset as an int value.
- * @param buf
- * @param offset
- * @return int value at offset
- */
- static int getAsInt(ByteBuffer buf, int offset) {
- if (buf.isDirect()) {
- return theUnsafe.getInt(((DirectBuffer) buf).address() + offset);
- }
- return theUnsafe.getInt(buf.array(), BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset);
- }
-
- /**
- * Reads a long value at the given buffer's offset considering it was written in big-endian
- * format.
- *
- * @param buf
- * @param offset
- * @return long value at offset
- */
- public static long toLong(ByteBuffer buf, int offset) {
- if (littleEndian) {
- return Long.reverseBytes(getAsLong(buf, offset));
- }
- return getAsLong(buf, offset);
- }
-
- /**
- * Reads bytes at the given offset as a long value.
- * @param buf
- * @param offset
- * @return long value at offset
- */
- static long getAsLong(ByteBuffer buf, int offset) {
- if (buf.isDirect()) {
- return theUnsafe.getLong(((DirectBuffer) buf).address() + offset);
- }
- return theUnsafe.getLong(buf.array(), BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset);
- }
-
- /**
- * Put an int value out to the specified ByteBuffer offset in big-endian format.
- * @param buf the ByteBuffer to write to
- * @param offset offset in the ByteBuffer
- * @param val int to write out
- * @return incremented offset
- */
- public static int putInt(ByteBuffer buf, int offset, int val) {
- if (littleEndian) {
- val = Integer.reverseBytes(val);
- }
- if (buf.isDirect()) {
- theUnsafe.putInt(((DirectBuffer) buf).address() + offset, val);
- } else {
- theUnsafe.putInt(buf.array(), offset + buf.arrayOffset() + BYTE_ARRAY_BASE_OFFSET, val);
- }
- return offset + Bytes.SIZEOF_INT;
- }
-
- // APIs to copy data. This will be direct memory location copy and will be much faster
- /**
- * Copies the bytes from given array's offset to length part into the given buffer.
- * @param src
- * @param srcOffset
- * @param dest
- * @param destOffset
- * @param length
- */
- public static void copy(byte[] src, int srcOffset, ByteBuffer dest, int destOffset, int length) {
- long destAddress = destOffset;
- Object destBase = null;
- if (dest.isDirect()) {
- destAddress = destAddress + ((DirectBuffer) dest).address();
- } else {
- destAddress = destAddress + BYTE_ARRAY_BASE_OFFSET + dest.arrayOffset();
- destBase = dest.array();
- }
- long srcAddress = srcOffset + BYTE_ARRAY_BASE_OFFSET;
- theUnsafe.copyMemory(src, srcAddress, destBase, destAddress, length);
- }
-
- /**
- * Copies specified number of bytes from given offset of {@code src} ByteBuffer to the
- * {@code dest} array.
- *
- * @param src
- * @param srcOffset
- * @param dest
- * @param destOffset
- * @param length
- */
- public static void copy(ByteBuffer src, int srcOffset, byte[] dest, int destOffset, int length) {
- long srcAddress = srcOffset;
- Object srcBase = null;
- if (src.isDirect()) {
- srcAddress = srcAddress + ((DirectBuffer) src).address();
- } else {
- srcAddress = srcAddress + BYTE_ARRAY_BASE_OFFSET + src.arrayOffset();
- srcBase = src.array();
- }
- long destAddress = destOffset + BYTE_ARRAY_BASE_OFFSET;
- theUnsafe.copyMemory(srcBase, srcAddress, dest, destAddress, length);
- }
-
- /**
- * Copies specified number of bytes from given offset of {@code src} buffer into the {@code dest}
- * buffer.
- *
- * @param src
- * @param srcOffset
- * @param dest
- * @param destOffset
- * @param length
- */
- public static void copy(ByteBuffer src, int srcOffset, ByteBuffer dest, int destOffset, int length) {
- long srcAddress, destAddress;
- Object srcBase = null, destBase = null;
- if (src.isDirect()) {
- srcAddress = srcOffset + ((DirectBuffer) src).address();
- } else {
- srcAddress = srcOffset + src.arrayOffset() + BYTE_ARRAY_BASE_OFFSET;
- srcBase = src.array();
- }
- if (dest.isDirect()) {
- destAddress = destOffset + ((DirectBuffer) dest).address();
- } else {
- destAddress = destOffset + BYTE_ARRAY_BASE_OFFSET + dest.arrayOffset();
- destBase = dest.array();
- }
- theUnsafe.copyMemory(srcBase, srcAddress, destBase, destAddress, length);
- }
-
- // APIs to add primitives to BBs
- /**
- * Put a short value out to the specified BB position in big-endian format.
- * @param buf the byte buffer
- * @param offset position in the buffer
- * @param val short to write out
- * @return incremented offset
- */
- public static int putShort(ByteBuffer buf, int offset, short val) {
- if (littleEndian) {
- val = Short.reverseBytes(val);
- }
- if (buf.isDirect()) {
- theUnsafe.putShort(((DirectBuffer) buf).address() + offset, val);
- } else {
- theUnsafe.putShort(buf.array(), BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset, val);
- }
- return offset + Bytes.SIZEOF_SHORT;
- }
-
- /**
- * Put a long value out to the specified BB position in big-endian format.
- * @param buf the byte buffer
- * @param offset position in the buffer
- * @param val long to write out
- * @return incremented offset
- */
- public static int putLong(ByteBuffer buf, int offset, long val) {
- if (littleEndian) {
- val = Long.reverseBytes(val);
- }
- if (buf.isDirect()) {
- theUnsafe.putLong(((DirectBuffer) buf).address() + offset, val);
- } else {
- theUnsafe.putLong(buf.array(), BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset, val);
- }
- return offset + Bytes.SIZEOF_LONG;
- }
-
- /**
- * Put a byte value out to the specified BB position in big-endian format.
- * @param buf the byte buffer
- * @param offset position in the buffer
- * @param b byte to write out
- * @return incremented offset
- */
- public static int putByte(ByteBuffer buf, int offset, byte b) {
- if (buf.isDirect()) {
- theUnsafe.putByte(((DirectBuffer) buf).address() + offset, b);
- } else {
- theUnsafe.putByte(buf.array(), BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset, b);
- }
- return offset + 1;
- }
-
- /**
- * Returns the byte at the given offset
- * @param buf the buffer to read
- * @param offset the offset at which the byte has to be read
- * @return the byte at the given offset
- */
- public static byte toByte(ByteBuffer buf, int offset) {
- if (buf.isDirect()) {
- return theUnsafe.getByte(((DirectBuffer) buf).address() + offset);
- } else {
- return theUnsafe.getByte(buf.array(), BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset);
- }
- }
-}
[17/18] kylin git commit: KYLIN-1528 Create a branch for v1.5 with
HBase 1.x API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 8f69c18..8f7430e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -190,7 +192,7 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeObserverClass, hdfsCoprocessorJar, 1002, null);
}
- public static boolean resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -205,7 +207,7 @@ public class DeployCoprocessorCLI {
logger.info("reset coprocessor on " + tableName);
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
while (desc.hasCoprocessor(CubeObserverClass)) {
desc.removeCoprocessor(CubeObserverClass);
@@ -231,16 +233,15 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
return true;
}
-
- private static List<String> resetCoprocessorOnHTables(final HBaseAdmin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static List<String> resetCoprocessorOnHTables(final Admin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processedTables = Collections.synchronizedList(new ArrayList<String>());
ExecutorService coprocessorPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -261,12 +262,12 @@ public class DeployCoprocessorCLI {
private static class ResetCoprocessorWorker implements Runnable {
private final CountDownLatch countDownLatch;
- private final HBaseAdmin hbaseAdmin;
+ private final Admin hbaseAdmin;
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List<String> processedTables;
- public ResetCoprocessorWorker(CountDownLatch countDownLatch, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
+ public ResetCoprocessorWorker(CountDownLatch countDownLatch, Admin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
this.countDownLatch = countDownLatch;
this.hbaseAdmin = hbaseAdmin;
this.hdfsCoprocessorJar = hdfsCoprocessorJar;
@@ -387,7 +388,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index 61c73d5..1cdb2f8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -25,10 +25,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -235,9 +236,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -257,7 +258,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 86ba22f..dd5f8fa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ Connection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void prepareData(Connection conn) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 6749d6c..940d64a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,9 +24,11 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -55,8 +57,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -71,12 +73,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 937b65f..1daca0a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -31,12 +32,15 @@ import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -58,30 +62,31 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ Table table = null;
+ Admin admin = null;
try {
+ table = hbaseConnection.getTable(TableName.valueOf(tableName));
+ admin = hbaseConnection.getAdmin();
+
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+ logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
+ List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
+ for (HRegionLocation hRegionLocation : regionLocationList) {
+ tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -105,7 +110,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- IOUtils.closeQuietly(hBaseAdmin);
+ IOUtils.closeQuietly(admin);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index 266f7e7..a2f60d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,9 +23,10 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -42,8 +43,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index e26c8e8..da13fa4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -58,11 +58,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- HConnection conn = getConnection();
- HBaseAdmin hadmin = new HBaseAdmin(conn);
+ Connection conn = getConnection();
+ Admin hadmin = conn.getAdmin();
try {
- boolean tableExist = hadmin.tableExists(tableName);
+ boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -119,8 +119,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
byte[] key = new byte[8 + 4];//time + id
@@ -135,7 +135,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -170,8 +170,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -206,7 +206,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(HTableInterface table) throws IOException {
+ private static long getFirstKeyTime(Table table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -224,8 +224,8 @@ public class HbaseStreamingInput {
}
- private static HConnection getConnection() throws IOException {
- return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
+ private static Connection getConnection() throws IOException {
+ return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ca1a060..ea05ab2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -50,8 +51,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index 8ff5b0f..df4e912 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,10 +30,14 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,9 +56,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
-
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -73,12 +77,13 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ TableName tableName = TableName.valueOf(htableName);
+ if (hbaseAdmin.tableExists(tableName)) {
+ if (hbaseAdmin.isTableEnabled(tableName)) {
+ hbaseAdmin.disableTable(tableName);
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(tableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index e219c5a..8a93160 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -22,12 +22,13 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -58,12 +59,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- HConnection conn = null;
- HTableInterface table = null;
+ Connection conn = null;
+ Table table = null;
ResultScanner scanner = null;
try {
- conn = HConnectionManager.createConnection(hconf);
- table = conn.getTable(hbaseTable);
+ conn = ConnectionFactory.createConnection(hconf);
+ table = conn.getTable(TableName.valueOf(hbaseTable));
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index 01edb1f..db516bb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,11 +22,12 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -70,8 +71,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
+ Connection conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 9fe5a23..74a4718 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -56,6 +58,7 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,7 +79,8 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -152,22 +156,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- HBaseAdmin hbaseAdmin;
+ Admin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index e36f662..42a54c8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,16 +24,18 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,14 +51,15 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ hbaseAdmin = conn.getAdmin();
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -166,9 +169,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index 40306c9..fa551f8 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -230,6 +230,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -446,11 +447,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -470,7 +471,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -536,13 +536,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -559,7 +558,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index 19e5db0..f52fc3e 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,10 +25,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -231,9 +232,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -253,7 +254,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
[12/18] kylin git commit: KYLIN-2255 drop v1 coprocessor impl
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
deleted file mode 100644
index 1bd92e6..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/generated/FilterProtosExt.java
+++ /dev/null
@@ -1,1298 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto
-
-package org.apache.kylin.storage.hbase.cube.v1.filter.generated;
-
-public final class FilterProtosExt {
- private FilterProtosExt() {
- }
-
- public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
- }
-
- public interface BytesBytesPairOrBuilder extends com.google.protobuf.MessageOrBuilder {
-
- // required bytes first = 1;
- /**
- * <code>required bytes first = 1;</code>
- */
- boolean hasFirst();
-
- /**
- * <code>required bytes first = 1;</code>
- */
- com.google.protobuf.ByteString getFirst();
-
- // required bytes second = 2;
- /**
- * <code>required bytes second = 2;</code>
- */
- boolean hasSecond();
-
- /**
- * <code>required bytes second = 2;</code>
- */
- com.google.protobuf.ByteString getSecond();
- }
-
- /**
- * Protobuf type {@code BytesBytesPair}
- */
- public static final class BytesBytesPair extends com.google.protobuf.GeneratedMessage implements BytesBytesPairOrBuilder {
- // Use BytesBytesPair.newBuilder() to construct.
- private BytesBytesPair(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
-
- private BytesBytesPair(boolean noInit) {
- this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance();
- }
-
- private static final BytesBytesPair defaultInstance;
-
- public static BytesBytesPair getDefaultInstance() {
- return defaultInstance;
- }
-
- public BytesBytesPair getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
-
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet getUnknownFields() {
- return this.unknownFields;
- }
-
- private BytesBytesPair(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- bitField0_ |= 0x00000001;
- first_ = input.readBytes();
- break;
- }
- case 18: {
- bitField0_ |= 0x00000002;
- second_ = input.readBytes();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
-
- public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_BytesBytesPair_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable internalGetFieldAccessorTable() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_BytesBytesPair_fieldAccessorTable.ensureFieldAccessorsInitialized(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.class, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder.class);
- }
-
- public static com.google.protobuf.Parser<BytesBytesPair> PARSER = new com.google.protobuf.AbstractParser<BytesBytesPair>() {
- public BytesBytesPair parsePartialFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
- return new BytesBytesPair(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<BytesBytesPair> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // required bytes first = 1;
- public static final int FIRST_FIELD_NUMBER = 1;
- private com.google.protobuf.ByteString first_;
-
- /**
- * <code>required bytes first = 1;</code>
- */
- public boolean hasFirst() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
-
- /**
- * <code>required bytes first = 1;</code>
- */
- public com.google.protobuf.ByteString getFirst() {
- return first_;
- }
-
- // required bytes second = 2;
- public static final int SECOND_FIELD_NUMBER = 2;
- private com.google.protobuf.ByteString second_;
-
- /**
- * <code>required bytes second = 2;</code>
- */
- public boolean hasSecond() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
-
- /**
- * <code>required bytes second = 2;</code>
- */
- public com.google.protobuf.ByteString getSecond() {
- return second_;
- }
-
- private void initFields() {
- first_ = com.google.protobuf.ByteString.EMPTY;
- second_ = com.google.protobuf.ByteString.EMPTY;
- }
-
- private byte memoizedIsInitialized = -1;
-
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1)
- return isInitialized == 1;
-
- if (!hasFirst()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasSecond()) {
- memoizedIsInitialized = 0;
- return false;
- }
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, first_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeBytes(2, second_);
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
-
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1)
- return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream.computeBytesSize(1, first_);
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream.computeBytesSize(2, second_);
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
-
- @java.lang.Override
- protected java.lang.Object writeReplace() throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- @java.lang.Override
- public boolean equals(final java.lang.Object obj) {
- if (obj == this) {
- return true;
- }
- if (!(obj instanceof org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair)) {
- return super.equals(obj);
- }
- org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair other = (org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair) obj;
-
- boolean result = true;
- result = result && (hasFirst() == other.hasFirst());
- if (hasFirst()) {
- result = result && getFirst().equals(other.getFirst());
- }
- result = result && (hasSecond() == other.hasSecond());
- if (hasSecond()) {
- result = result && getSecond().equals(other.getSecond());
- }
- result = result && getUnknownFields().equals(other.getUnknownFields());
- return result;
- }
-
- private int memoizedHashCode = 0;
-
- @java.lang.Override
- public int hashCode() {
- if (memoizedHashCode != 0) {
- return memoizedHashCode;
- }
- int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
- if (hasFirst()) {
- hash = (37 * hash) + FIRST_FIELD_NUMBER;
- hash = (53 * hash) + getFirst().hashCode();
- }
- if (hasSecond()) {
- hash = (37 * hash) + SECOND_FIELD_NUMBER;
- hash = (53 * hash) + getSecond().hashCode();
- }
- hash = (29 * hash) + getUnknownFields().hashCode();
- memoizedHashCode = hash;
- return hash;
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parseFrom(com.google.protobuf.ByteString data) throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parseFrom(com.google.protobuf.ByteString data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parseFrom(byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parseFrom(java.io.InputStream input) throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parseFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parseDelimitedFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parseFrom(com.google.protobuf.CodedInputStream input) throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parseFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() {
- return Builder.create();
- }
-
- public Builder newBuilderForType() {
- return newBuilder();
- }
-
- public static Builder newBuilder(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair prototype) {
- return newBuilder().mergeFrom(prototype);
- }
-
- public Builder toBuilder() {
- return newBuilder(this);
- }
-
- @java.lang.Override
- protected Builder newBuilderForType(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
-
- /**
- * Protobuf type {@code BytesBytesPair}
- */
- public static final class Builder extends com.google.protobuf.GeneratedMessage.Builder<Builder> implements org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPairOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_BytesBytesPair_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable internalGetFieldAccessorTable() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_BytesBytesPair_fieldAccessorTable.ensureFieldAccessorsInitialized(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.class, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder.class);
- }
-
- // Construct using org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
-
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- }
- }
-
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- first_ = com.google.protobuf.ByteString.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000001);
- second_ = com.google.protobuf.ByteString.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000002);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_BytesBytesPair_descriptor;
- }
-
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair getDefaultInstanceForType() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.getDefaultInstance();
- }
-
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair build() {
- org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair buildPartial() {
- org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair result = new org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.first_ = first_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- result.second_ = second_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair) {
- return mergeFrom((org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair) other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair other) {
- if (other == org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.getDefaultInstance())
- return this;
- if (other.hasFirst()) {
- setFirst(other.getFirst());
- }
- if (other.hasSecond()) {
- setSecond(other.getSecond());
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- if (!hasFirst()) {
-
- return false;
- }
- if (!hasSecond()) {
-
- return false;
- }
- return true;
- }
-
- public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
- org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
-
- private int bitField0_;
-
- // required bytes first = 1;
- private com.google.protobuf.ByteString first_ = com.google.protobuf.ByteString.EMPTY;
-
- /**
- * <code>required bytes first = 1;</code>
- */
- public boolean hasFirst() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
-
- /**
- * <code>required bytes first = 1;</code>
- */
- public com.google.protobuf.ByteString getFirst() {
- return first_;
- }
-
- /**
- * <code>required bytes first = 1;</code>
- */
- public Builder setFirst(com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- first_ = value;
- onChanged();
- return this;
- }
-
- /**
- * <code>required bytes first = 1;</code>
- */
- public Builder clearFirst() {
- bitField0_ = (bitField0_ & ~0x00000001);
- first_ = getDefaultInstance().getFirst();
- onChanged();
- return this;
- }
-
- // required bytes second = 2;
- private com.google.protobuf.ByteString second_ = com.google.protobuf.ByteString.EMPTY;
-
- /**
- * <code>required bytes second = 2;</code>
- */
- public boolean hasSecond() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
-
- /**
- * <code>required bytes second = 2;</code>
- */
- public com.google.protobuf.ByteString getSecond() {
- return second_;
- }
-
- /**
- * <code>required bytes second = 2;</code>
- */
- public Builder setSecond(com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000002;
- second_ = value;
- onChanged();
- return this;
- }
-
- /**
- * <code>required bytes second = 2;</code>
- */
- public Builder clearSecond() {
- bitField0_ = (bitField0_ & ~0x00000002);
- second_ = getDefaultInstance().getSecond();
- onChanged();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:BytesBytesPair)
- }
-
- static {
- defaultInstance = new BytesBytesPair(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:BytesBytesPair)
- }
-
- public interface FuzzyRowFilterV2OrBuilder extends com.google.protobuf.MessageOrBuilder {
-
- // repeated .BytesBytesPair fuzzy_keys_data = 1;
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- java.util.List<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair> getFuzzyKeysDataList();
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair getFuzzyKeysData(int index);
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- int getFuzzyKeysDataCount();
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- java.util.List<? extends org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPairOrBuilder> getFuzzyKeysDataOrBuilderList();
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPairOrBuilder getFuzzyKeysDataOrBuilder(int index);
- }
-
- /**
- * Protobuf type {@code FuzzyRowFilterV2}
- */
- public static final class FuzzyRowFilterV2 extends com.google.protobuf.GeneratedMessage implements FuzzyRowFilterV2OrBuilder {
- // Use FuzzyRowFilterV2.newBuilder() to construct.
- private FuzzyRowFilterV2(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
-
- private FuzzyRowFilterV2(boolean noInit) {
- this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance();
- }
-
- private static final FuzzyRowFilterV2 defaultInstance;
-
- public static FuzzyRowFilterV2 getDefaultInstance() {
- return defaultInstance;
- }
-
- public FuzzyRowFilterV2 getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
-
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet getUnknownFields() {
- return this.unknownFields;
- }
-
- private FuzzyRowFilterV2(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
- fuzzyKeysData_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair>();
- mutable_bitField0_ |= 0x00000001;
- }
- fuzzyKeysData_.add(input.readMessage(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.PARSER, extensionRegistry));
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(e.getMessage()).setUnfinishedMessage(this);
- } finally {
- if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
- fuzzyKeysData_ = java.util.Collections.unmodifiableList(fuzzyKeysData_);
- }
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
-
- public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_FuzzyRowFilterV2_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable internalGetFieldAccessorTable() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_FuzzyRowFilterV2_fieldAccessorTable.ensureFieldAccessorsInitialized(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2.class, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2.Builder.class);
- }
-
- public static com.google.protobuf.Parser<FuzzyRowFilterV2> PARSER = new com.google.protobuf.AbstractParser<FuzzyRowFilterV2>() {
- public FuzzyRowFilterV2 parsePartialFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
- return new FuzzyRowFilterV2(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<FuzzyRowFilterV2> getParserForType() {
- return PARSER;
- }
-
- // repeated .BytesBytesPair fuzzy_keys_data = 1;
- public static final int FUZZY_KEYS_DATA_FIELD_NUMBER = 1;
- private java.util.List<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair> fuzzyKeysData_;
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public java.util.List<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair> getFuzzyKeysDataList() {
- return fuzzyKeysData_;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPairOrBuilder> getFuzzyKeysDataOrBuilderList() {
- return fuzzyKeysData_;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public int getFuzzyKeysDataCount() {
- return fuzzyKeysData_.size();
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair getFuzzyKeysData(int index) {
- return fuzzyKeysData_.get(index);
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPairOrBuilder getFuzzyKeysDataOrBuilder(int index) {
- return fuzzyKeysData_.get(index);
- }
-
- private void initFields() {
- fuzzyKeysData_ = java.util.Collections.emptyList();
- }
-
- private byte memoizedIsInitialized = -1;
-
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1)
- return isInitialized == 1;
-
- for (int i = 0; i < getFuzzyKeysDataCount(); i++) {
- if (!getFuzzyKeysData(i).isInitialized()) {
- memoizedIsInitialized = 0;
- return false;
- }
- }
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
- getSerializedSize();
- for (int i = 0; i < fuzzyKeysData_.size(); i++) {
- output.writeMessage(1, fuzzyKeysData_.get(i));
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
-
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1)
- return size;
-
- size = 0;
- for (int i = 0; i < fuzzyKeysData_.size(); i++) {
- size += com.google.protobuf.CodedOutputStream.computeMessageSize(1, fuzzyKeysData_.get(i));
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
-
- @java.lang.Override
- protected java.lang.Object writeReplace() throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- @java.lang.Override
- public boolean equals(final java.lang.Object obj) {
- if (obj == this) {
- return true;
- }
- if (!(obj instanceof org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2)) {
- return super.equals(obj);
- }
- org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 other = (org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2) obj;
-
- boolean result = true;
- result = result && getFuzzyKeysDataList().equals(other.getFuzzyKeysDataList());
- result = result && getUnknownFields().equals(other.getUnknownFields());
- return result;
- }
-
- private int memoizedHashCode = 0;
-
- @java.lang.Override
- public int hashCode() {
- if (memoizedHashCode != 0) {
- return memoizedHashCode;
- }
- int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
- if (getFuzzyKeysDataCount() > 0) {
- hash = (37 * hash) + FUZZY_KEYS_DATA_FIELD_NUMBER;
- hash = (53 * hash) + getFuzzyKeysDataList().hashCode();
- }
- hash = (29 * hash) + getUnknownFields().hashCode();
- memoizedHashCode = hash;
- return hash;
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parseFrom(com.google.protobuf.ByteString data) throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parseFrom(com.google.protobuf.ByteString data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parseFrom(byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parseFrom(java.io.InputStream input) throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parseFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parseDelimitedFrom(java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parseFrom(com.google.protobuf.CodedInputStream input) throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
-
- public static org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parseFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() {
- return Builder.create();
- }
-
- public Builder newBuilderForType() {
- return newBuilder();
- }
-
- public static Builder newBuilder(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 prototype) {
- return newBuilder().mergeFrom(prototype);
- }
-
- public Builder toBuilder() {
- return newBuilder(this);
- }
-
- @java.lang.Override
- protected Builder newBuilderForType(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
-
- /**
- * Protobuf type {@code FuzzyRowFilterV2}
- */
- public static final class Builder extends com.google.protobuf.GeneratedMessage.Builder<Builder> implements org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2OrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_FuzzyRowFilterV2_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable internalGetFieldAccessorTable() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_FuzzyRowFilterV2_fieldAccessorTable.ensureFieldAccessorsInitialized(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2.class, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2.Builder.class);
- }
-
- // Construct using org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
-
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- getFuzzyKeysDataFieldBuilder();
- }
- }
-
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- if (fuzzyKeysDataBuilder_ == null) {
- fuzzyKeysData_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000001);
- } else {
- fuzzyKeysDataBuilder_.clear();
- }
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.internal_static_FuzzyRowFilterV2_descriptor;
- }
-
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 getDefaultInstanceForType() {
- return org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2.getDefaultInstance();
- }
-
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 build() {
- org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 buildPartial() {
- org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 result = new org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2(this);
- int from_bitField0_ = bitField0_;
- if (fuzzyKeysDataBuilder_ == null) {
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- fuzzyKeysData_ = java.util.Collections.unmodifiableList(fuzzyKeysData_);
- bitField0_ = (bitField0_ & ~0x00000001);
- }
- result.fuzzyKeysData_ = fuzzyKeysData_;
- } else {
- result.fuzzyKeysData_ = fuzzyKeysDataBuilder_.build();
- }
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2) {
- return mergeFrom((org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2) other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 other) {
- if (other == org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2.getDefaultInstance())
- return this;
- if (fuzzyKeysDataBuilder_ == null) {
- if (!other.fuzzyKeysData_.isEmpty()) {
- if (fuzzyKeysData_.isEmpty()) {
- fuzzyKeysData_ = other.fuzzyKeysData_;
- bitField0_ = (bitField0_ & ~0x00000001);
- } else {
- ensureFuzzyKeysDataIsMutable();
- fuzzyKeysData_.addAll(other.fuzzyKeysData_);
- }
- onChanged();
- }
- } else {
- if (!other.fuzzyKeysData_.isEmpty()) {
- if (fuzzyKeysDataBuilder_.isEmpty()) {
- fuzzyKeysDataBuilder_.dispose();
- fuzzyKeysDataBuilder_ = null;
- fuzzyKeysData_ = other.fuzzyKeysData_;
- bitField0_ = (bitField0_ & ~0x00000001);
- fuzzyKeysDataBuilder_ = com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? getFuzzyKeysDataFieldBuilder() : null;
- } else {
- fuzzyKeysDataBuilder_.addAllMessages(other.fuzzyKeysData_);
- }
- }
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- for (int i = 0; i < getFuzzyKeysDataCount(); i++) {
- if (!getFuzzyKeysData(i).isInitialized()) {
-
- return false;
- }
- }
- return true;
- }
-
- public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException {
- org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2 parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.FuzzyRowFilterV2) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
-
- private int bitField0_;
-
- // repeated .BytesBytesPair fuzzy_keys_data = 1;
- private java.util.List<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair> fuzzyKeysData_ = java.util.Collections.emptyList();
-
- private void ensureFuzzyKeysDataIsMutable() {
- if (!((bitField0_ & 0x00000001) == 0x00000001)) {
- fuzzyKeysData_ = new java.util.ArrayList<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair>(fuzzyKeysData_);
- bitField0_ |= 0x00000001;
- }
- }
-
- private com.google.protobuf.RepeatedFieldBuilder<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPairOrBuilder> fuzzyKeysDataBuilder_;
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public java.util.List<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair> getFuzzyKeysDataList() {
- if (fuzzyKeysDataBuilder_ == null) {
- return java.util.Collections.unmodifiableList(fuzzyKeysData_);
- } else {
- return fuzzyKeysDataBuilder_.getMessageList();
- }
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public int getFuzzyKeysDataCount() {
- if (fuzzyKeysDataBuilder_ == null) {
- return fuzzyKeysData_.size();
- } else {
- return fuzzyKeysDataBuilder_.getCount();
- }
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair getFuzzyKeysData(int index) {
- if (fuzzyKeysDataBuilder_ == null) {
- return fuzzyKeysData_.get(index);
- } else {
- return fuzzyKeysDataBuilder_.getMessage(index);
- }
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public Builder setFuzzyKeysData(int index, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair value) {
- if (fuzzyKeysDataBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureFuzzyKeysDataIsMutable();
- fuzzyKeysData_.set(index, value);
- onChanged();
- } else {
- fuzzyKeysDataBuilder_.setMessage(index, value);
- }
- return this;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public Builder setFuzzyKeysData(int index, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder builderForValue) {
- if (fuzzyKeysDataBuilder_ == null) {
- ensureFuzzyKeysDataIsMutable();
- fuzzyKeysData_.set(index, builderForValue.build());
- onChanged();
- } else {
- fuzzyKeysDataBuilder_.setMessage(index, builderForValue.build());
- }
- return this;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public Builder addFuzzyKeysData(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair value) {
- if (fuzzyKeysDataBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureFuzzyKeysDataIsMutable();
- fuzzyKeysData_.add(value);
- onChanged();
- } else {
- fuzzyKeysDataBuilder_.addMessage(value);
- }
- return this;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public Builder addFuzzyKeysData(int index, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair value) {
- if (fuzzyKeysDataBuilder_ == null) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureFuzzyKeysDataIsMutable();
- fuzzyKeysData_.add(index, value);
- onChanged();
- } else {
- fuzzyKeysDataBuilder_.addMessage(index, value);
- }
- return this;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public Builder addFuzzyKeysData(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder builderForValue) {
- if (fuzzyKeysDataBuilder_ == null) {
- ensureFuzzyKeysDataIsMutable();
- fuzzyKeysData_.add(builderForValue.build());
- onChanged();
- } else {
- fuzzyKeysDataBuilder_.addMessage(builderForValue.build());
- }
- return this;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public Builder addFuzzyKeysData(int index, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder builderForValue) {
- if (fuzzyKeysDataBuilder_ == null) {
- ensureFuzzyKeysDataIsMutable();
- fuzzyKeysData_.add(index, builderForValue.build());
- onChanged();
- } else {
- fuzzyKeysDataBuilder_.addMessage(index, builderForValue.build());
- }
- return this;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public Builder addAllFuzzyKeysData(java.lang.Iterable<? extends org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair> values) {
- if (fuzzyKeysDataBuilder_ == null) {
- ensureFuzzyKeysDataIsMutable();
- super.addAll(values, fuzzyKeysData_);
- onChanged();
- } else {
- fuzzyKeysDataBuilder_.addAllMessages(values);
- }
- return this;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public Builder clearFuzzyKeysData() {
- if (fuzzyKeysDataBuilder_ == null) {
- fuzzyKeysData_ = java.util.Collections.emptyList();
- bitField0_ = (bitField0_ & ~0x00000001);
- onChanged();
- } else {
- fuzzyKeysDataBuilder_.clear();
- }
- return this;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public Builder removeFuzzyKeysData(int index) {
- if (fuzzyKeysDataBuilder_ == null) {
- ensureFuzzyKeysDataIsMutable();
- fuzzyKeysData_.remove(index);
- onChanged();
- } else {
- fuzzyKeysDataBuilder_.remove(index);
- }
- return this;
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder getFuzzyKeysDataBuilder(int index) {
- return getFuzzyKeysDataFieldBuilder().getBuilder(index);
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPairOrBuilder getFuzzyKeysDataOrBuilder(int index) {
- if (fuzzyKeysDataBuilder_ == null) {
- return fuzzyKeysData_.get(index);
- } else {
- return fuzzyKeysDataBuilder_.getMessageOrBuilder(index);
- }
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public java.util.List<? extends org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPairOrBuilder> getFuzzyKeysDataOrBuilderList() {
- if (fuzzyKeysDataBuilder_ != null) {
- return fuzzyKeysDataBuilder_.getMessageOrBuilderList();
- } else {
- return java.util.Collections.unmodifiableList(fuzzyKeysData_);
- }
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder addFuzzyKeysDataBuilder() {
- return getFuzzyKeysDataFieldBuilder().addBuilder(org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.getDefaultInstance());
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder addFuzzyKeysDataBuilder(int index) {
- return getFuzzyKeysDataFieldBuilder().addBuilder(index, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.getDefaultInstance());
- }
-
- /**
- * <code>repeated .BytesBytesPair fuzzy_keys_data = 1;</code>
- */
- public java.util.List<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder> getFuzzyKeysDataBuilderList() {
- return getFuzzyKeysDataFieldBuilder().getBuilderList();
- }
-
- private com.google.protobuf.RepeatedFieldBuilder<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPairOrBuilder> getFuzzyKeysDataFieldBuilder() {
- if (fuzzyKeysDataBuilder_ == null) {
- fuzzyKeysDataBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPair.Builder, org.apache.kylin.storage.hbase.cube.v1.filter.generated.FilterProtosExt.BytesBytesPairOrBuilder>(fuzzyKeysData_, ((bitField0_ & 0x00000001) == 0x00000001), getParentForChildren(), isClean());
- fuzzyKeysData_ = null;
- }
- return fuzzyKeysDataBuilder_;
- }
-
- // @@protoc_insertion_point(builder_scope:FuzzyRowFilterV2)
- }
-
- static {
- defaultInstance = new FuzzyRowFilterV2(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:FuzzyRowFilterV2)
- }
-
- private static com.google.protobuf.Descriptors.Descriptor internal_static_BytesBytesPair_descriptor;
- private static com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_BytesBytesPair_fieldAccessorTable;
- private static com.google.protobuf.Descriptors.Descriptor internal_static_FuzzyRowFilterV2_descriptor;
- private static com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_FuzzyRowFilterV2_fieldAccessorTable;
-
- public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
- return descriptor;
- }
-
- private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
- static {
- java.lang.String[] descriptorData = { "\nbstorage-hbase/src/main/java/org/apache" + "/kylin/storage/hbase/cube/v1/filter/prot" + "obuf/FilterExt.proto\"/\n\016BytesBytesPair\022\r" + "\n\005first\030\001 \002(\014\022\016\n\006second\030\002 \002(\014\"<\n\020FuzzyRo" + "wFilterV2\022(\n\017fuzzy_keys_data\030\001 \003(\0132\017.Byt" + "esBytesPairBR\n7org.apache.kylin.storage." + "hbase.cube.v1.filter.generatedB\017FilterPr" + "otosExtH\001\210\001\001\240\001\001" };
- com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
- public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) {
- descriptor = root;
- internal_static_BytesBytesPair_descriptor = getDescriptor().getMessageTypes().get(0);
- internal_static_BytesBytesPair_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable(internal_static_BytesBytesPair_descriptor, new java.lang.String[] { "First", "Second", });
- internal_static_FuzzyRowFilterV2_descriptor = getDescriptor().getMessageTypes().get(1);
- internal_static_FuzzyRowFilterV2_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable(internal_static_FuzzyRowFilterV2_descriptor, new java.lang.String[] { "FuzzyKeysData", });
- return null;
- }
- };
- com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[] {}, assigner);
- }
-
- // @@protoc_insertion_point(outer_class_scope)
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto
deleted file mode 100644
index 29b0069..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto
+++ /dev/null
@@ -1,39 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// usage:
-// protoc --java_out=./storage-hbase/src/main/java ./storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/filter/protobuf/FilterExt.proto
-
-option java_package = "org.apache.kylin.storage.hbase.cube.v1.filter.generated";
-
-option java_outer_classname = "FilterProtosExt";
-
-option java_generic_services = true;
-
-option java_generate_equals_and_hash = true;
-
-option optimize_for = SPEED;
-
-message BytesBytesPair {
- required bytes first = 1;
- required bytes second = 2;
-}
-
-message FuzzyRowFilterV2 {
- repeated BytesBytesPair fuzzy_keys_data = 1;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
index e07ea91..f151876 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
@@ -20,12 +20,9 @@ package org.apache.kylin.storage.hbase.cube;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertTrue;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -42,7 +39,6 @@ import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.IStorageQuery;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -93,23 +89,6 @@ public class MeasureTypeOnlyAggrInBaseTest extends LocalFileMetadataTestCase {
}
@Test
- public void testIdentifyCuboidV1() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, NoSuchFieldException {
- IStorageQuery query = new org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery(cube);
- long baseCuboidId = cubeDesc.getRowkey().getFullMask();
-
- Method method = query.getClass().getDeclaredMethod("identifyCuboid", Set.class, Collection.class);
- method.setAccessible(true);
-
- Object ret = method.invoke(query, Sets.newHashSet(), Lists.newArrayList());
-
- assertTrue(ret instanceof Cuboid);
- assertNotEquals(baseCuboidId, ((Cuboid) ret).getId());
-
- ret = method.invoke(query, dimensions, metrics);
- assertEquals(baseCuboidId, ((Cuboid) ret).getId());
- }
-
- @Test
public void testIdentifyCuboidV2() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, NoSuchFieldException {
CubeDesc cubeDesc = cube.getDescriptor();
Cuboid ret = Cuboid.identifyCuboid(cubeDesc, Sets.<TblColRef> newHashSet(), Lists.<FunctionDesc> newArrayList());
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
deleted file mode 100644
index c25b690..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ /dev/null
@@ -1,339 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.gridtable.StorageSideBehavior;
-import org.apache.kylin.metadata.datatype.LongMutable;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverAggregators.HCol;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-/**
- * @author yangli9
- */
-public class AggregateRegionObserverTest {
-
- byte[] mask = new byte[] { (byte) 0xff, (byte) 0xff, 0, 0 };
- byte[] k1 = new byte[] { 0x01, 0x01, 0, 0x01 };
- byte[] k2 = new byte[] { 0x01, 0x01, 0, 0x02 };
- byte[] k3 = new byte[] { 0x02, 0x02, 0, 0x03 };
- byte[] k4 = new byte[] { 0x02, 0x02, 0, 0x04 };
-
- ArrayList<Cell> cellsInput = Lists.newArrayList();
-
- byte[] family = Bytes.toBytes("f");
- byte[] q1 = Bytes.toBytes("q1");
- byte[] q2 = Bytes.toBytes("q2");
-
- HCol c1 = new HCol(family, q1, new String[] { "SUM", "COUNT" }, new String[] { "decimal", "long" });
- HCol c2 = new HCol(family, q2, new String[] { "SUM" }, new String[] { "decimal" });
-
- @Before
- public void setup() {
- cellsInput.add(newCell(k1, c1, "10.5", 1));
- cellsInput.add(newCell(k2, c1, "11.5", 2));
- cellsInput.add(newCell(k3, c1, "12.5", 3));
- cellsInput.add(newCell(k4, c1, "13.5", 4));
-
- cellsInput.add(newCell(k1, c2, "21.5"));
- cellsInput.add(newCell(k2, c2, "22.5"));
- cellsInput.add(newCell(k3, c2, "23.5"));
- cellsInput.add(newCell(k4, c2, "24.5"));
-
- }
-
- private Cell newCell(byte[] key, HCol col, String decimal) {
- return newCell(key, col, decimal, Integer.MIN_VALUE);
- }
-
- private Cell newCell(byte[] key, HCol col, String decimal, int number) {
- Object[] values = number == Integer.MIN_VALUE ? //
- new Object[] { new BigDecimal(decimal) } //
- : new Object[] { new BigDecimal(decimal), new LongMutable(number) };
- ByteBuffer buf = col.measureCodec.encode(values);
-
- Cell keyValue = new KeyValue(key, 0, key.length, //
- col.family, 0, col.family.length, //
- col.qualifier, 0, col.qualifier.length, //
- HConstants.LATEST_TIMESTAMP, Type.Put, //
- buf.array(), 0, buf.position());
-
- return keyValue;
- }
-
- @Test
- public void test() throws IOException {
-
- CoprocessorRowType rowType = newRowType();
- CoprocessorProjector projector = new CoprocessorProjector(mask, true);
- ObserverAggregators aggregators = new ObserverAggregators(new HCol[] { c1, c2 });
- CoprocessorFilter filter = CoprocessorFilter.deserialize(null); // a default,
- // always-true,
- // filter
- HashSet<String> expectedResult = new HashSet<String>();
-
- expectedResult.add("\\x02\\x02\\x00\\x00, f:q1, [26.0, 7]");
- expectedResult.add("\\x02\\x02\\x00\\x00, f:q2, [48.0]");
- expectedResult.add("\\x01\\x01\\x00\\x00, f:q1, [22.0, 3]");
- expectedResult.add("\\x01\\x01\\x00\\x00, f:q2, [44.0]");
-
- MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
-
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM);
- ArrayList<Cell> result = Lists.newArrayList();
- boolean hasMore = true;
- while (hasMore) {
- result.clear();
- hasMore = aggrScanner.next(result);
- if (result.isEmpty())
- continue;
-
- Cell cell = result.get(0);
- HCol hcol = null;
- if (ObserverAggregators.match(c1, cell)) {
- hcol = c1;
- } else if (ObserverAggregators.match(c2, cell)) {
- hcol = c2;
- } else
- fail();
-
- hcol.measureCodec.decode(ByteBuffer.wrap(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()), hcol.measureValues);
-
- String rowKey = toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), mask);
- String col = Bytes.toString(hcol.family) + ":" + Bytes.toString(hcol.qualifier);
- String values = Arrays.toString(hcol.measureValues);
-
- System.out.println(rowKey);
- System.out.println(col);
- System.out.println(values);
-
- assertTrue(expectedResult.contains(rowKey + ", " + col + ", " + values));
- }
- aggrScanner.close();
- }
-
- @Test
- public void testNoMeasure() throws IOException {
-
- CoprocessorRowType rowType = newRowType();
- CoprocessorProjector projector = new CoprocessorProjector(mask, true);
- ObserverAggregators aggregators = new ObserverAggregators(new HCol[] {});
- CoprocessorFilter filter = CoprocessorFilter.deserialize(null); // a default,
- // always-true,
- // filter
- HashSet<String> expectedResult = new HashSet<String>();
-
- expectedResult.add("\\x02\\x02\\x00\\x00");
- expectedResult.add("\\x01\\x01\\x00\\x00");
-
- MockupRegionScanner innerScanner = new MockupRegionScanner(cellsInput);
-
- RegionScanner aggrScanner = new AggregationScanner(rowType, filter, projector, aggregators, innerScanner, StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM);
- ArrayList<Cell> result = Lists.newArrayList();
- boolean hasMore = true;
- while (hasMore) {
- result.clear();
- hasMore = aggrScanner.next(result);
- if (result.isEmpty())
- continue;
-
- Cell cell = result.get(0);
-
- String rowKey = toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), mask);
-
- assertTrue(expectedResult.contains(rowKey));
- }
- aggrScanner.close();
- }
-
- private String toString(byte[] array, int offset, short length, byte[] mask) {
- StringBuilder result = new StringBuilder();
- for (int i = 0; i < length; i++) {
- int ch = array[offset + i] & 0xFF & mask[i];
- result.append(String.format("\\x%02X", ch));
- }
- return result.toString();
- }
-
- private CoprocessorRowType newRowType() {
- TableDesc t = new TableDesc();
- t.setName("TABLE");
- t.setDatabase("DEFAULT");
- TblColRef[] cols = new TblColRef[] { newCol(1, "A", t), newCol(2, "B", t), newCol(3, "C", t), newCol(4, "D", t) };
- int[] sizes = new int[] { 1, 1, 1, 1 };
- return new CoprocessorRowType(cols, sizes, 0);
- }
-
- private TblColRef newCol(int i, String name, TableDesc t) {
- return TblColRef.mockup(t, i, name, null);
- }
-
- public static class MockupRegionScanner implements RegionScanner {
- List<Cell> input;
- int i = 0;
-
- public MockupRegionScanner(List<Cell> cellInputs) {
- this.input = cellInputs;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.InternalScanner#next(java.util
- * .List)
- */
- @Override
- public boolean next(List<Cell> results) throws IOException {
- return nextRaw(results);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.InternalScanner#next(java.util
- * .List, int)
- */
- @Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- return next(result);
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.hbase.regionserver.InternalScanner#close()
- */
- @Override
- public void close() throws IOException {
-
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#getRegionInfo()
- */
- @Override
- public HRegionInfo getRegionInfo() {
- return null;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#isFilterDone()
- */
- @Override
- public boolean isFilterDone() throws IOException {
- return false;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#reseek(byte[])
- */
- @Override
- public boolean reseek(byte[] row) throws IOException {
- return false;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#getMaxResultSize()
- */
- @Override
- public long getMaxResultSize() {
- return 0;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#getMvccReadPoint()
- */
- @Override
- public long getMvccReadPoint() {
- return 0;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#nextRaw(java.util
- * .List)
- */
- @Override
- public boolean nextRaw(List<Cell> result) throws IOException {
- if (i < input.size()) {
- result.add(input.get(i));
- i++;
- }
- return i < input.size();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#nextRaw(java.util
- * .List, int)
- */
- @Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return nextRaw(result);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowAggregatorsTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowAggregatorsTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowAggregatorsTest.java
deleted file mode 100644
index da34ff6..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowAggregatorsTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.Bytes;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class RowAggregatorsTest {
-
- @Test
- public void testSerialize() {
- ObserverAggregators.HCol[] hcols = new ObserverAggregators.HCol[] { //
- newHCol("f", "c1", new String[] { "SUM", "COUNT" }, new String[] { "decimal", "long" }), //
- newHCol("f", "c2", new String[] { "SUM", "SUM" }, new String[] { "long", "long" }) };
- ObserverAggregators sample = new ObserverAggregators(hcols);
-
- byte[] bytes = ObserverAggregators.serialize(sample);
- ObserverAggregators copy = ObserverAggregators.deserialize(bytes);
-
- assertTrue(sample.nHCols == copy.nHCols);
- assertTrue(sample.nTotalMeasures == copy.nTotalMeasures);
- assertEquals(sample.hcols[0], copy.hcols[0]);
- assertEquals(sample.hcols[1], copy.hcols[1]);
- }
-
- private static ObserverAggregators.HCol newHCol(String family, String qualifier, String[] funcNames, String[] dataTypes) {
- return new ObserverAggregators.HCol(Bytes.toBytes(family), Bytes.toBytes(qualifier), funcNames, dataTypes);
- }
-
- private static void assertEquals(ObserverAggregators.HCol a, ObserverAggregators.HCol b) {
- assertTrue(a.nMeasures == b.nMeasures);
- assertTrue(Arrays.equals(a.family, b.family));
- assertTrue(Arrays.equals(a.qualifier, b.qualifier));
- assertTrue(Arrays.equals(a.funcNames, b.funcNames));
- assertTrue(Arrays.equals(a.dataTypes, b.dataTypes));
- }
-
-}
[02/18] kylin git commit: KYLIN-1875 fix ParameterDesc equals() &
hashcode()
Posted by li...@apache.org.
KYLIN-1875 fix ParameterDesc equals() & hashcode()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/227bbf41
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/227bbf41
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/227bbf41
Branch: refs/heads/master-hbase1.x
Commit: 227bbf4152074c0621bf2b1047102d416d4dd3eb
Parents: 967ef18
Author: Yang Li <li...@apache.org>
Authored: Thu Dec 1 06:58:08 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Dec 7 17:53:36 2016 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/FunctionDesc.java | 1 +
.../kylin/metadata/model/ParameterDesc.java | 36 +++++++++++++++-----
2 files changed, 28 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/227bbf41/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index dfa6f3b..b9e5543 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -248,6 +248,7 @@ public class FunctionDesc {
final int prime = 31;
int result = 1;
result = prime * result + ((expression == null) ? 0 : expression.hashCode());
+ result = prime * result + ((returnType == null) ? 0 : returnType.hashCode());
result = prime * result + ((isCount() || parameter == null) ? 0 : parameter.hashCode());
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/227bbf41/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
index 329799f..4a95fea 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ParameterDesc.java
@@ -22,8 +22,10 @@ import java.io.UnsupportedEncodingException;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableList;
/**
*/
@@ -36,9 +38,10 @@ public class ParameterDesc {
private String value;
@JsonProperty("next_parameter")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
private ParameterDesc nextParameter;
- private List<TblColRef> colRefs;
+ private List<TblColRef> colRefs = ImmutableList.of();
public String getType() {
return type;
@@ -89,21 +92,36 @@ public class ParameterDesc {
ParameterDesc that = (ParameterDesc) o;
- if (nextParameter != null ? !nextParameter.equals(that.nextParameter) : that.nextParameter != null)
- return false;
if (type != null ? !type.equals(that.type) : that.type != null)
return false;
- if (value != null ? !value.equals(that.value) : that.value != null)
- return false;
-
- return true;
+
+ ParameterDesc p = this, q = that;
+ int refi = 0, refj = 0;
+ for (; p != null && q != null; p = p.nextParameter, q = q.nextParameter) {
+ if (p.isColumnType()) {
+ if (q.isColumnType() == false)
+ return false;
+ if (refi >= this.colRefs.size() || refj >= that.colRefs.size())
+ return false;
+ if (this.colRefs.get(refi).equals(that.colRefs.get(refj)) == false)
+ return false;
+ refi++;
+ refj++;
+ } else {
+ if (q.isColumnType() == true)
+ return false;
+ if (p.value.equals(q.value) == false)
+ return false;
+ }
+ }
+
+ return p == null && q == null;
}
@Override
public int hashCode() {
int result = type != null ? type.hashCode() : 0;
- result = 31 * result + (value != null ? value.hashCode() : 0);
- result = 31 * result + (nextParameter != null ? nextParameter.hashCode() : 0);
+ result = 31 * result + (colRefs != null ? colRefs.hashCode() : 0);
return result;
}
[07/18] kylin git commit: minor,
add API to get AggregationGroup combination cuboid num
Posted by li...@apache.org.
minor, add API to get AggregationGroup combination cuboid num
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b1fd4679
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b1fd4679
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b1fd4679
Branch: refs/heads/master-hbase1.x
Commit: b1fd4679fa80e8e971d553a42baf20198afb9eac
Parents: 365caae
Author: Billy Liu <bi...@apache.org>
Authored: Wed Dec 7 18:29:07 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Dec 7 18:29:07 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/CubeController.java | 33 ++++++++++++++++++++
1 file changed, 33 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b1fd4679/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f537231..ab32551 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.AggregationGroup;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.dimension.DimensionEncodingFactory;
@@ -655,6 +656,22 @@ public class CubeController extends BasicController {
return response;
}
+ /**
+ * Calculate Cuboid Combination based on the AggreationGroup definition.
+ *
+ * @param aggregationGroupStr
+ * @return number of cuboid, -1 if failed
+ */
+ @RequestMapping(value = "aggregationgroups/cuboid", method = RequestMethod.POST)
+ @ResponseBody
+ public long calculateCuboidCombination(@RequestBody String aggregationGroupStr) {
+ AggregationGroup aggregationGroup = deserializeAggregationGroup(aggregationGroupStr);
+ if (aggregationGroup != null) {
+ return aggregationGroup.calculateCuboidCombination();
+ } else
+ return -1;
+ }
+
private CubeDesc deserializeCubeDesc(CubeRequest cubeRequest) {
CubeDesc desc = null;
try {
@@ -673,6 +690,22 @@ public class CubeController extends BasicController {
return desc;
}
+ private AggregationGroup deserializeAggregationGroup(String aggregationGroupStr) {
+ AggregationGroup aggreationGroup = null;
+ try {
+ logger.debug("Parsing AggregationGroup " + aggregationGroupStr);
+ aggreationGroup = JsonUtil.readValue(aggregationGroupStr, AggregationGroup.class);
+ } catch (JsonParseException e) {
+ logger.error("The AggregationGroup definition is not valid.", e);
+ } catch (JsonMappingException e) {
+ logger.error("The AggregationGroup definition is not valid.", e);
+ } catch (IOException e) {
+ logger.error("Failed to deal with the request.", e);
+ throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+ }
+ return aggreationGroup;
+ }
+
private void updateRequest(CubeRequest request, boolean success, String message) {
request.setCubeDescData("");
request.setSuccessful(success);
[14/18] kylin git commit: KYLIN-2255 drop v1 coprocessor impl
Posted by li...@apache.org.
KYLIN-2255 drop v1 coprocessor impl
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/545201f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/545201f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/545201f6
Branch: refs/heads/master-hbase1.x
Commit: 545201f6c0c2e4da9b3dcabac0f606e2ef880d19
Parents: 21bcd2f
Author: Li Yang <li...@apache.org>
Authored: Thu Dec 8 10:41:18 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 8 10:54:58 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/query/ITCombinationTest.java | 15 -
.../org/apache/kylin/query/KylinTestBase.java | 2 -
.../kylin/rest/controller/CubeController.java | 14 -
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 344 -----
.../storage/hbase/cube/v1/CubeStorageQuery.java | 796 -----------
.../hbase/cube/v1/CubeTupleConverter.java | 271 ----
.../hbase/cube/v1/RegionScannerAdapter.java | 97 --
.../hbase/cube/v1/ResultScannerAdapter.java | 100 --
.../cube/v1/SerializedHBaseTupleIterator.java | 156 ---
.../observer/AggregateRegionObserver.java | 112 --
.../observer/AggregationScanner.java | 188 ---
.../observer/ObserverAggregationCache.java | 166 ---
.../observer/ObserverAggregators.java | 265 ----
.../coprocessor/observer/ObserverEnabler.java | 191 ---
.../v1/coprocessor/observer/ObserverTuple.java | 71 -
.../hbase/cube/v1/filter/FuzzyRowFilterV2.java | 574 --------
.../hbase/cube/v1/filter/UnsafeAccess.java | 433 ------
.../v1/filter/generated/FilterProtosExt.java | 1298 ------------------
.../cube/v1/filter/protobuf/FilterExt.proto | 39 -
.../cube/MeasureTypeOnlyAggrInBaseTest.java | 21 -
.../observer/AggregateRegionObserverTest.java | 339 -----
.../observer/RowAggregatorsTest.java | 62 -
.../cube/v1/filter/TestFuzzyRowFilterV2.java | 249 ----
.../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 346 -----
24 files changed, 6149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index 496ac4e..84573b5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -27,7 +27,6 @@ import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.routing.Candidate;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.storage.hbase.HBaseStorage;
-import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.runner.RunWith;
@@ -67,7 +66,6 @@ public class ITCombinationTest extends ITKylinQueryTest {
public static Collection<Object[]> configs() {
return Arrays.asList(new Object[][] { //
{ "inner", "on", "v2", true }, //
- { "left", "on", "v1", true }, //
{ "left", "on", "v2", true }, //
//{ "inner", "on", "v2", false }, // exclude view to simply model/cube selection
//{ "left", "on", "v1", false }, // exclude view to simply model/cube selection
@@ -84,23 +82,10 @@ public class ITCombinationTest extends ITKylinQueryTest {
ITKylinQueryTest.joinType = joinType;
ITKylinQueryTest.setupAll();
- if (coprocessorToggle.equals("on")) {
- ObserverEnabler.forceCoprocessorOn();
- } else if (coprocessorToggle.equals("off")) {
- ObserverEnabler.forceCoprocessorOff();
- } else if (coprocessorToggle.equals("unset")) {
- // unset
- }
-
RemoveBlackoutRealizationsRule.blackList.clear();
if (excludeViewCubes) {
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
RemoveBlackoutRealizationsRule.blackList.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
}
-
- if ("v1".equalsIgnoreCase(queryEngine))
- HBaseStorage.overwriteStorageQuery = HBaseStorage.v1CubeStorageQuery;
- else
- HBaseStorage.overwriteStorageQuery = null;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 52461c4..bcf55e5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -52,7 +52,6 @@ import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
-import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.Assertion;
import org.dbunit.database.DatabaseConfig;
import org.dbunit.database.DatabaseConnection;
@@ -628,7 +627,6 @@ public class KylinTestBase {
if (h2Connection != null)
closeConnection(h2Connection);
- ObserverEnabler.forceCoprocessorUnset();
HBaseMetadataTestCase.staticCleanupTestMetadata();
RemoveBlackoutRealizationsRule.blackList.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index ab32551..aef1ffc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -55,7 +55,6 @@ import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.source.kafka.util.KafkaClient;
-import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -198,19 +197,6 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/coprocessor", method = { RequestMethod.PUT })
- @ResponseBody
- public Map<String, Boolean> updateCubeCoprocessor(@PathVariable String cubeName, @RequestParam(value = "force") String force) {
- try {
- ObserverEnabler.updateCubeOverride(cubeName, force);
- return ObserverEnabler.getCubeOverrides();
- } catch (Exception e) {
- String message = "Failed to update cube coprocessor: " + cubeName + " : " + force;
- logger.error(message, e);
- throw new InternalErrorException(message + " Caused by: " + e.getMessage(), e);
- }
- }
-
/**
* Force rebuild a cube's lookup table snapshot
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
deleted file mode 100644
index 8ac3832..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1;
-
-import java.text.MessageFormat;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.StorageException;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.Tuple;
-import org.apache.kylin.metadata.tuple.TupleInfo;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
-import org.apache.kylin.storage.hbase.cube.v1.filter.FuzzyRowFilterV2;
-import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-import org.apache.kylin.storage.translate.HBaseKeyRange;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- * @author xjiang
- *
- */
-public class CubeSegmentTupleIterator implements ITupleIterator {
-
- public static final Logger logger = LoggerFactory.getLogger(CubeSegmentTupleIterator.class);
-
- protected final CubeSegment cubeSeg;
- private final TupleFilter filter;
- private final Collection<TblColRef> groupBy;
- protected final List<RowValueDecoder> rowValueDecoders;
- private final StorageContext context;
- private final String tableName;
- private final HTableInterface table;
-
- protected CubeTupleConverter tupleConverter;
- protected final Iterator<HBaseKeyRange> rangeIterator;
- protected final Tuple oneTuple; // avoid new instance
-
- private Scan scan;
- private ResultScanner scanner;
- protected Iterator<Result> resultIterator;
- protected int scanCount;
- protected int scanCountDelta;
- protected Tuple next;
- protected final Cuboid cuboid;
-
- private List<MeasureType.IAdvMeasureFiller> advMeasureFillers;
- private int advMeasureRowsRemaining;
- private int advMeasureRowIndex;
-
- public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, HConnection conn, //
- Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, //
- List<RowValueDecoder> rowValueDecoders, StorageContext context, TupleInfo returnTupleInfo) {
- this.cubeSeg = cubeSeg;
- this.filter = filter;
- this.groupBy = groupBy;
- this.rowValueDecoders = rowValueDecoders;
- this.context = context;
- this.tableName = cubeSeg.getStorageLocationIdentifier();
-
- cuboid = keyRanges.get(0).getCuboid();
- for (HBaseKeyRange range : keyRanges) {
- assert cuboid.equals(range.getCuboid());
- }
-
- this.tupleConverter = new CubeTupleConverter(cubeSeg, cuboid, rowValueDecoders, returnTupleInfo);
- this.oneTuple = new Tuple(returnTupleInfo);
- this.rangeIterator = keyRanges.iterator();
-
- try {
- this.table = conn.getTable(tableName);
- } catch (Throwable t) {
- throw new StorageException("Error when open connection to table " + tableName, t);
- }
- }
-
- @Override
- public boolean hasNext() {
-
- if (next != null)
- return true;
-
- // consume any left rows from advanced measure filler
- if (advMeasureRowsRemaining > 0) {
- for (MeasureType.IAdvMeasureFiller filler : advMeasureFillers) {
- filler.fillTuple(oneTuple, advMeasureRowIndex);
- }
- advMeasureRowIndex++;
- advMeasureRowsRemaining--;
- next = oneTuple;
- return true;
- }
-
- if (resultIterator == null) {
- if (rangeIterator.hasNext() == false)
- return false;
-
- resultIterator = doScan(rangeIterator.next());
- }
-
- if (resultIterator.hasNext() == false) {
- closeScanner();
- resultIterator = null;
- return hasNext();
- }
-
- Result result = resultIterator.next();
- scanCount++;
- if (++scanCountDelta >= 1000)
- flushScanCountDelta();
-
- // translate into tuple
- advMeasureFillers = tupleConverter.translateResult(result, oneTuple);
-
- // the simple case
- if (advMeasureFillers == null) {
- next = oneTuple;
- return true;
- }
-
- // advanced measure filling, like TopN, will produce multiple tuples out of one record
- advMeasureRowsRemaining = -1;
- for (MeasureType.IAdvMeasureFiller filler : advMeasureFillers) {
- if (advMeasureRowsRemaining < 0)
- advMeasureRowsRemaining = filler.getNumOfRows();
- if (advMeasureRowsRemaining != filler.getNumOfRows())
- throw new IllegalStateException();
- }
- if (advMeasureRowsRemaining < 0)
- throw new IllegalStateException();
-
- advMeasureRowIndex = 0;
- return hasNext();
-
- }
-
- @Override
- public Tuple next() {
- if (next == null) {
- hasNext();
- if (next == null)
- throw new NoSuchElementException();
- }
- Tuple r = next;
- next = null;
- return r;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- protected final Iterator<Result> doScan(HBaseKeyRange keyRange) {
- Iterator<Result> iter = null;
- try {
- scan = buildScan(keyRange);
- applyFuzzyFilter(scan, keyRange);
- logScan(keyRange);
-
- scanner = ObserverEnabler.scanWithCoprocessorIfBeneficial(cubeSeg, keyRange.getCuboid(), filter, groupBy, rowValueDecoders, context, table, scan);
-
- iter = scanner.iterator();
- } catch (Throwable t) {
- String msg = MessageFormat.format("Error when scan from lower key {1} to upper key {2} on table {0}.", tableName, Bytes.toString(keyRange.getStartKey()), Bytes.toString(keyRange.getStopKey()));
- throw new StorageException(msg, t);
- }
- return iter;
- }
-
- private void logScan(HBaseKeyRange keyRange) {
- StringBuilder info = new StringBuilder();
- info.append("Scan hbase table ").append(tableName).append(": ");
- if (keyRange.getCuboid().requirePostAggregation()) {
- info.append(" cuboid require post aggregation, from ");
- } else {
- info.append(" cuboid exact match, from ");
- }
- info.append(keyRange.getCuboid().getInputID());
- info.append(" to ");
- info.append(keyRange.getCuboid().getId());
- info.append(" Start: ");
- info.append(keyRange.getStartKeyAsString());
- info.append(" - ");
- info.append(Bytes.toStringBinary(keyRange.getStartKey()));
- info.append(" Stop: ");
- info.append(keyRange.getStopKeyAsString());
- info.append(" - ");
- info.append(Bytes.toStringBinary(keyRange.getStopKey()));
- if (this.scan.getFilter() != null) {
- info.append(" Fuzzy key counts: " + keyRange.getFuzzyKeys().size());
- info.append(" Fuzzy: ");
- info.append(keyRange.getFuzzyKeyAsString());
- }
- logger.info(info.toString());
- }
-
- private Scan buildScan(HBaseKeyRange keyRange) {
- Scan scan = new Scan();
- tuneScanParameters(scan);
- scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
- for (RowValueDecoder valueDecoder : this.rowValueDecoders) {
- HBaseColumnDesc hbaseColumn = valueDecoder.getHBaseColumn();
- byte[] byteFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
- byte[] byteQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
- scan.addColumn(byteFamily, byteQualifier);
- }
- scan.setStartRow(keyRange.getStartKey());
- scan.setStopRow(keyRange.getStopKey());
- return scan;
- }
-
- private void tuneScanParameters(Scan scan) {
- KylinConfig config = cubeSeg.getCubeDesc().getConfig();
-
- scan.setCaching(config.getHBaseScanCacheRows());
- scan.setMaxResultSize(config.getHBaseScanMaxResultSize());
- scan.setCacheBlocks(true);
-
- // cache less when there are memory hungry measures
- // if (RowValueDecoder.hasMemHungryMeasures(rowValueDecoders)) {
- // scan.setCaching(scan.getCaching() / 10);
- // }
- }
-
- private void applyFuzzyFilter(Scan scan, HBaseKeyRange keyRange) {
- List<org.apache.kylin.common.util.Pair<byte[], byte[]>> fuzzyKeys = keyRange.getFuzzyKeys();
- if (fuzzyKeys != null && fuzzyKeys.size() > 0) {
-
- //FuzzyRowFilterV2 is a back ported from https://issues.apache.org/jira/browse/HBASE-13761
- //However we found a bug of it and fixed it in https://issues.apache.org/jira/browse/HBASE-14269
- //After fix the performance is not much faster than the original one. So by default use defalt one.
- boolean useFuzzyRowFilterV2 = false;
- Filter fuzzyFilter = null;
- if (useFuzzyRowFilterV2) {
- fuzzyFilter = new FuzzyRowFilterV2(convertToHBasePair(fuzzyKeys));
- } else {
- fuzzyFilter = new FuzzyRowFilter(convertToHBasePair(fuzzyKeys));
- }
-
- Filter filter = scan.getFilter();
- if (filter != null) {
- throw new RuntimeException("Scan filter not empty : " + filter);
- } else {
- scan.setFilter(fuzzyFilter);
- }
- }
- }
-
- private List<org.apache.hadoop.hbase.util.Pair<byte[], byte[]>> convertToHBasePair(List<org.apache.kylin.common.util.Pair<byte[], byte[]>> pairList) {
- List<org.apache.hadoop.hbase.util.Pair<byte[], byte[]>> result = Lists.newArrayList();
- for (org.apache.kylin.common.util.Pair<byte[], byte[]> pair : pairList) {
- org.apache.hadoop.hbase.util.Pair<byte[], byte[]> element = new org.apache.hadoop.hbase.util.Pair<byte[], byte[]>(pair.getFirst(), pair.getSecond());
- result.add(element);
- }
-
- return result;
- }
-
- protected void closeScanner() {
- flushScanCountDelta();
-
- if (logger.isDebugEnabled() && scan != null) {
- byte[] metricsBytes = scan.getAttribute(Scan.SCAN_ATTRIBUTES_METRICS_DATA);
- if (metricsBytes != null) {
- ScanMetrics scanMetrics = ProtobufUtil.toScanMetrics(metricsBytes);
- logger.debug("HBase Metrics when scanning " + this.tableName + " count={}, ms={}, bytes={}, remote_bytes={}, regions={}, not_serving_region={}, rpc={}, rpc_retries={}, remote_rpc={}, remote_rpc_retries={}", //
- new Object[] { scanCount, scanMetrics.sumOfMillisSecBetweenNexts, scanMetrics.countOfBytesInResults, scanMetrics.countOfBytesInRemoteResults, scanMetrics.countOfRegions, scanMetrics.countOfNSRE, scanMetrics.countOfRPCcalls, scanMetrics.countOfRPCRetries, scanMetrics.countOfRemoteRPCcalls, scanMetrics.countOfRemoteRPCRetries });
- }
- scan = null;
- }
- try {
- if (scanner != null) {
- scanner.close();
- scanner = null;
- }
- } catch (Throwable t) {
- throw new StorageException("Error when close scanner for table " + tableName, t);
- }
- }
-
- private void closeTable() {
- try {
- if (table != null) {
- table.close();
- }
- } catch (Throwable t) {
- throw new StorageException("Error when close table " + tableName, t);
- }
- }
-
- @Override
- public void close() {
- logger.info("Closing CubeSegmentTupleIterator");
- closeScanner();
- closeTable();
- }
-
- protected void flushScanCountDelta() {
- context.increaseTotalScanCount(scanCountDelta);
- scanCountDelta = 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
deleted file mode 100644
index 75c3fd7..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ /dev/null
@@ -1,796 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.ShardingHash;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.RawQueryLastHacker;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
-import org.apache.kylin.cube.model.HBaseColumnDesc;
-import org.apache.kylin.cube.model.HBaseMappingDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.TupleInfo;
-import org.apache.kylin.storage.IStorageQuery;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
-import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-import org.apache.kylin.storage.translate.ColumnValueRange;
-import org.apache.kylin.storage.translate.DerivedFilterTranslator;
-import org.apache.kylin.storage.translate.HBaseKeyRange;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-@SuppressWarnings("unused")
-public class CubeStorageQuery implements IStorageQuery {
-
- private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class);
-
- private static final int MERGE_KEYRANGE_THRESHOLD = 100;
-
- private final CubeInstance cubeInstance;
- private final CubeDesc cubeDesc;
- private final String uuid;
-
- public CubeStorageQuery(CubeInstance cube) {
- this.cubeInstance = cube;
- this.cubeDesc = cube.getDescriptor();
- this.uuid = cube.getUuid();
- }
-
- @Override
- public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
-
- //deal with participant columns in subquery join
- sqlDigest.includeSubqueryJoinParticipants();
-
- //cope with queries with no aggregations
- RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc, returnTupleInfo);
-
- // Customized measure taking effect: e.g. allow custom measures to help raw queries
- notifyBeforeStorageQuery(sqlDigest);
-
- Collection<TblColRef> groups = sqlDigest.groupbyColumns;
- TupleFilter filter = sqlDigest.filter;
-
- // build dimension & metrics
- Collection<TblColRef> dimensions = new HashSet<TblColRef>();
- Collection<FunctionDesc> metrics = new HashSet<FunctionDesc>();
- buildDimensionsAndMetrics(dimensions, metrics, sqlDigest);
-
- // all dimensions = groups + others
- Set<TblColRef> others = Sets.newHashSet(dimensions);
- others.removeAll(groups);
-
- // expand derived
- Set<TblColRef> derivedPostAggregation = Sets.newHashSet();
- Set<TblColRef> groupsD = expandDerived(groups, derivedPostAggregation);
- Set<TblColRef> othersD = expandDerived(others, derivedPostAggregation);
- othersD.removeAll(groupsD);
-
- // identify cuboid
- Set<TblColRef> dimensionsD = Sets.newHashSet();
- dimensionsD.addAll(groupsD);
- dimensionsD.addAll(othersD);
- Cuboid cuboid = identifyCuboid(dimensionsD, metrics);
- context.setCuboid(cuboid);
-
- // isExactAggregation? meaning: tuples returned from storage requires no further aggregation in query engine
- Set<TblColRef> singleValuesD = findSingleValueColumns(filter);
- boolean isExactAggregation = isExactAggregation(cuboid, groupsD, othersD, singleValuesD, derivedPostAggregation);
- context.setExactAggregation(isExactAggregation);
-
- // translate filter for scan range and compose returning groups for coprocessor, note:
- // - columns on non-evaluatable filter have to return
- // - columns on loosened filter (due to derived translation) have to return
- Set<TblColRef> groupsCopD = Sets.newHashSet(groupsD);
- collectNonEvaluable(filter, groupsCopD);
- TupleFilter filterD = translateDerived(filter, groupsCopD);
-
- // translate filter into segment scan ranges
- List<HBaseKeyRange> scans = buildScanRanges(flattenToOrAndFilter(filterD), dimensionsD);
-
- // check involved measures, build value decoder for each each family:column
- List<RowValueDecoder> valueDecoders = translateAggregation(cubeDesc.getHbaseMapping(), metrics, context);
-
- // memory hungry distinct count are pushed down to coprocessor, no need to set threshold any more
- setThreshold(dimensionsD, valueDecoders, context); // set cautious threshold to prevent out of memory
- setCoprocessor(groupsCopD, valueDecoders, context); // enable coprocessor if beneficial
- setLimit(filter, context);
-
- HConnection conn = HBaseConnection.get(context.getConnUrl());
-
- // notice we're passing filterD down to storage instead of flatFilter
- return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context, returnTupleInfo);
- }
-
- private void buildDimensionsAndMetrics(Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics, SQLDigest sqlDigest) {
-
- for (FunctionDesc func : sqlDigest.aggregations) {
- if (!func.isDimensionAsMetric()) {
- metrics.add(func);
- }
- }
-
- for (TblColRef column : sqlDigest.allColumns) {
- // skip measure columns
- if (sqlDigest.metricColumns.contains(column)) {
- continue;
- }
-
- dimensions.add(column);
- }
- }
-
- private Cuboid identifyCuboid(Set<TblColRef> dimensions, Collection<FunctionDesc> metrics) {
- for (FunctionDesc metric : metrics) {
- if (metric.getMeasureType().onlyAggrInBaseCuboid())
- return Cuboid.getBaseCuboid(cubeDesc);
- }
-
- long cuboidID = 0;
- for (TblColRef column : dimensions) {
- int index = cubeDesc.getRowkey().getColumnBitIndex(column);
- cuboidID |= 1L << index;
- }
- return Cuboid.findById(cubeDesc, cuboidID);
- }
-
- private boolean isExactAggregation(Cuboid cuboid, Collection<TblColRef> groups, Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation) {
- boolean exact = true;
-
- if (cuboid.requirePostAggregation()) {
- exact = false;
- logger.info("exactAggregation is false because cuboid " + cuboid.getInputID() + "=> " + cuboid.getId());
- }
-
- // derived aggregation is bad, unless expanded columns are already in group by
- if (groups.containsAll(derivedPostAggregation) == false) {
- exact = false;
- logger.info("exactAggregation is false because derived column require post aggregation: " + derivedPostAggregation);
- }
-
- // other columns (from filter) is bad, unless they are ensured to have single value
- if (singleValuesD.containsAll(othersD) == false) {
- exact = false;
- logger.info("exactAggregation is false because some column not on group by: " + othersD //
- + " (single value column: " + singleValuesD + ")");
- }
-
- if (exact) {
- logger.info("exactAggregation is true");
- }
- return exact;
- }
-
- private Set<TblColRef> expandDerived(Collection<TblColRef> cols, Set<TblColRef> derivedPostAggregation) {
- Set<TblColRef> expanded = Sets.newHashSet();
- for (TblColRef col : cols) {
- if (cubeDesc.hasHostColumn(col)) {
- DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
- for (TblColRef hostCol : hostInfo.columns) {
- expanded.add(hostCol);
- if (hostInfo.isOneToOne == false)
- derivedPostAggregation.add(hostCol);
- }
- } else {
- expanded.add(col);
- }
- }
- return expanded;
- }
-
- @SuppressWarnings("unchecked")
- private Set<TblColRef> findSingleValueColumns(TupleFilter filter) {
- Collection<? extends TupleFilter> toCheck;
- if (filter instanceof CompareTupleFilter) {
- toCheck = Collections.singleton(filter);
- } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
- toCheck = filter.getChildren();
- } else {
- return (Set<TblColRef>) Collections.EMPTY_SET;
- }
-
- Set<TblColRef> result = Sets.newHashSet();
- for (TupleFilter f : toCheck) {
- if (f instanceof CompareTupleFilter) {
- CompareTupleFilter compFilter = (CompareTupleFilter) f;
- // is COL=const ?
- if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1 && compFilter.getColumn() != null) {
- result.add(compFilter.getColumn());
- }
- }
- }
-
- // expand derived
- Set<TblColRef> resultD = Sets.newHashSet();
- for (TblColRef col : result) {
- if (cubeDesc.isExtendedColumn(col)) {
- throw new CubeDesc.CannotFilterExtendedColumnException(col);
- }
-
- if (cubeDesc.isDerived(col)) {
- DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
- if (hostInfo.isOneToOne) {
- for (TblColRef hostCol : hostInfo.columns) {
- resultD.add(hostCol);
- }
- }
- //if not one2one, it will be pruned
- } else {
- resultD.add(col);
- }
- }
- return resultD;
- }
-
- private void collectNonEvaluable(TupleFilter filter, Set<TblColRef> collector) {
- if (filter == null)
- return;
-
- if (filter.isEvaluable()) {
- for (TupleFilter child : filter.getChildren())
- collectNonEvaluable(child, collector);
- } else {
- collectColumnsRecursively(filter, collector);
- }
- }
-
- private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
- if (filter == null)
- return;
-
- if (filter instanceof ColumnTupleFilter) {
- collectColumns(((ColumnTupleFilter) filter).getColumn(), collector);
- }
- for (TupleFilter child : filter.getChildren()) {
- collectColumnsRecursively(child, collector);
- }
- }
-
- private void collectColumns(TblColRef col, Set<TblColRef> collector) {
- if (cubeDesc.isExtendedColumn(col)) {
- throw new CubeDesc.CannotFilterExtendedColumnException(col);
- }
-
- if (cubeDesc.isDerived(col)) {
- DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
- for (TblColRef h : hostInfo.columns)
- collector.add(h);
- } else {
- collector.add(col);
- }
- }
-
- @SuppressWarnings("unchecked")
- private TupleFilter translateDerived(TupleFilter filter, Set<TblColRef> collector) {
- if (filter == null)
- return filter;
-
- if (filter instanceof CompareTupleFilter) {
- return translateDerivedInCompare((CompareTupleFilter) filter, collector);
- }
-
- List<TupleFilter> children = (List<TupleFilter>) filter.getChildren();
- List<TupleFilter> newChildren = Lists.newArrayListWithCapacity(children.size());
- boolean modified = false;
- for (TupleFilter child : children) {
- TupleFilter translated = translateDerived(child, collector);
- newChildren.add(translated);
- if (child != translated)
- modified = true;
- }
- if (modified) {
- filter = replaceChildren(filter, newChildren);
- }
- return filter;
- }
-
- private TupleFilter replaceChildren(TupleFilter filter, List<TupleFilter> newChildren) {
- if (filter instanceof LogicalTupleFilter) {
- LogicalTupleFilter r = new LogicalTupleFilter(filter.getOperator());
- r.addChildren(newChildren);
- return r;
- } else
- throw new IllegalStateException("Cannot replaceChildren on " + filter);
- }
-
- private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
- if (compf.getColumn() == null || compf.getValues().isEmpty())
- return compf;
-
- TblColRef derived = compf.getColumn();
- if (cubeDesc.isExtendedColumn(derived)) {
- throw new CubeDesc.CannotFilterExtendedColumnException(derived);
- }
- if (cubeDesc.isDerived(derived) == false) {
- return compf;
- }
-
- DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
- CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
- CubeSegment seg = cubeInstance.getLatestReadySegment();
- LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.join);
- Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
- TupleFilter translatedFilter = translated.getFirst();
- boolean loosened = translated.getSecond();
- if (loosened) {
- collectColumnsRecursively(translatedFilter, collector);
- }
- return translatedFilter;
- }
-
- private List<RowValueDecoder> translateAggregation(HBaseMappingDesc hbaseMapping, Collection<FunctionDesc> metrics, //
- StorageContext context) {
- Map<HBaseColumnDesc, RowValueDecoder> codecMap = Maps.newHashMap();
- for (FunctionDesc aggrFunc : metrics) {
- Collection<HBaseColumnDesc> hbCols = hbaseMapping.findHBaseColumnByFunction(aggrFunc);
- if (hbCols.isEmpty()) {
- throw new IllegalStateException("can't find HBaseColumnDesc for function " + aggrFunc.getFullExpression());
- }
- HBaseColumnDesc bestHBCol = null;
- int bestIndex = -1;
- for (HBaseColumnDesc hbCol : hbCols) {
- bestHBCol = hbCol;
- bestIndex = hbCol.findMeasure(aggrFunc);
- // we used to prefer specific measure over another (holistic distinct count), now it's gone
- break;
- }
-
- RowValueDecoder codec = codecMap.get(bestHBCol);
- if (codec == null) {
- codec = new RowValueDecoder(bestHBCol);
- codecMap.put(bestHBCol, codec);
- }
- codec.setProjectIndex(bestIndex);
- }
- return new ArrayList<RowValueDecoder>(codecMap.values());
- }
-
- //check TupleFilter.flatFilter's comment
- private TupleFilter flattenToOrAndFilter(TupleFilter filter) {
- if (filter == null)
- return null;
-
- // core
- TupleFilter flatFilter = filter.flatFilter();
-
- // normalize to OR-AND filter
- if (flatFilter.getOperator() == FilterOperatorEnum.AND) {
- LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR);
- f.addChild(flatFilter);
- flatFilter = f;
- }
-
- if (flatFilter.getOperator() != FilterOperatorEnum.OR)
- throw new IllegalStateException();
-
- return flatFilter;
- }
-
- private List<HBaseKeyRange> buildScanRanges(TupleFilter flatFilter, Collection<TblColRef> dimensionColumns) {
-
- List<HBaseKeyRange> result = Lists.newArrayList();
-
- logger.info("Current cubeInstance is " + cubeInstance + " with " + cubeInstance.getSegments().size() + " segs in all");
- List<CubeSegment> segs = cubeInstance.getSegments(SegmentStatusEnum.READY);
- logger.info("READY segs count: " + segs.size());
-
- // build row key range for each cube segment
- StringBuilder sb = new StringBuilder("hbasekeyrange trace: ");
- for (CubeSegment cubeSeg : segs) {
- CubeDesc cubeDesc = cubeSeg.getCubeDesc();
- if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
- logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
- continue;
- }
- // consider derived (lookup snapshot), filter on dimension may
- // differ per segment
- List<Collection<ColumnValueRange>> orAndDimRanges = translateToOrAndDimRanges(flatFilter, cubeSeg);
- if (orAndDimRanges == null) { // has conflict
- continue;
- }
-
- List<HBaseKeyRange> scanRanges = Lists.newArrayListWithCapacity(orAndDimRanges.size());
- for (Collection<ColumnValueRange> andDimRanges : orAndDimRanges) {
- HBaseKeyRange rowKeyRange = new HBaseKeyRange(dimensionColumns, andDimRanges, cubeSeg, cubeDesc);
- scanRanges.add(rowKeyRange);
- }
-
- //log
- sb.append(scanRanges.size() + "=(mergeoverlap)>");
-
- List<HBaseKeyRange> mergedRanges = mergeOverlapRanges(scanRanges);
-
- //log
- sb.append(mergedRanges.size() + "=(mergetoomany)>");
-
- mergedRanges = mergeTooManyRanges(mergedRanges);
-
- //log
- sb.append(mergedRanges.size() + ",");
-
- result.addAll(mergedRanges);
- }
- logger.info(sb.toString());
-
- logger.info("hbasekeyrange count: " + result.size());
-
- dropUnhitSegments(result);
- logger.info("hbasekeyrange count after dropping unhit :" + result.size());
-
- //TODO: should use LazyRowKeyEncoder.getRowKeysDifferentShards like CubeHBaseRPC, not do so because v1 query engine is retiring. not worth changing it
- if (cubeDesc.isEnableSharding()) {
- result = duplicateRangeByShard(result);
- }
- logger.info("hbasekeyrange count after dropping duplicatebyshard :" + result.size());
-
- return result;
- }
-
- private List<Collection<ColumnValueRange>> translateToOrAndDimRanges(TupleFilter flatFilter, CubeSegment cubeSegment) {
- List<Collection<ColumnValueRange>> result = Lists.newArrayList();
-
- if (flatFilter == null) {
- result.add(Collections.<ColumnValueRange> emptyList());
- return result;
- }
-
- for (TupleFilter andFilter : flatFilter.getChildren()) {
- if (andFilter.getOperator() != FilterOperatorEnum.AND) {
- throw new IllegalStateException("Filter should be AND instead of " + andFilter);
- }
-
- Collection<ColumnValueRange> andRanges = translateToAndDimRanges(andFilter.getChildren(), cubeSegment);
-
- if (andRanges != null) {
- result.add(andRanges);
- }
- }
-
- return preprocessConstantConditions(result);
- }
-
- private List<Collection<ColumnValueRange>> preprocessConstantConditions(List<Collection<ColumnValueRange>> orAndRanges) {
- boolean globalAlwaysTrue = false;
- Iterator<Collection<ColumnValueRange>> iterator = orAndRanges.iterator();
- while (iterator.hasNext()) {
- Collection<ColumnValueRange> andRanges = iterator.next();
- Iterator<ColumnValueRange> iterator2 = andRanges.iterator();
- boolean hasAlwaysFalse = false;
- while (iterator2.hasNext()) {
- ColumnValueRange range = iterator2.next();
- if (range.satisfyAll())
- iterator2.remove();
- else if (range.satisfyNone())
- hasAlwaysFalse = true;
- }
- if (hasAlwaysFalse) {
- iterator.remove();
- } else if (andRanges.isEmpty()) {
- globalAlwaysTrue = true;
- break;
- }
- }
- if (globalAlwaysTrue) {
- orAndRanges.clear();
- orAndRanges.add(Collections.<ColumnValueRange> emptyList());
- }
- return orAndRanges;
- }
-
- // return empty collection to mean true; return null to mean false
- @SuppressWarnings("unchecked")
- private Collection<ColumnValueRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters, CubeSegment cubeSegment) {
- Map<TblColRef, ColumnValueRange> rangeMap = new HashMap<TblColRef, ColumnValueRange>();
- for (TupleFilter filter : andFilters) {
- if ((filter instanceof CompareTupleFilter) == false) {
- continue;
- }
-
- CompareTupleFilter comp = (CompareTupleFilter) filter;
- if (comp.getColumn() == null) {
- continue;
- }
-
- ColumnValueRange range = new ColumnValueRange(comp.getColumn(), (Collection<String>) comp.getValues(), comp.getOperator());
- andMerge(range, rangeMap);
- }
-
- // a little pre-evaluation to remove invalid EQ/IN values and round start/end according to dictionary
- RowKeyDesc rowkey = cubeSegment.getCubeDesc().getRowkey();
- Iterator<ColumnValueRange> it = rangeMap.values().iterator();
- while (it.hasNext()) {
- ColumnValueRange range = it.next();
- if (rowkey.isUseDictionary(range.getColumn())) {
- range.preEvaluateWithDict((Dictionary<String>) cubeSegment.getDictionary(range.getColumn()));
- }
- if (range.satisfyAll())
- it.remove();
- else if (range.satisfyNone())
- return null;
- }
-
- return rangeMap.values();
- }
-
- private void andMerge(ColumnValueRange range, Map<TblColRef, ColumnValueRange> rangeMap) {
- ColumnValueRange columnRange = rangeMap.get(range.getColumn());
- if (columnRange == null) {
- rangeMap.put(range.getColumn(), range);
- } else {
- columnRange.andMerge(range);
- }
- }
-
- private List<HBaseKeyRange> mergeOverlapRanges(List<HBaseKeyRange> keyRanges) {
- if (keyRanges.size() <= 1) {
- return keyRanges;
- }
-
- if (logger.isDebugEnabled()) {
- logger.debug("Merging key range from " + keyRanges.size());
- }
-
- // sort ranges by start key
- Collections.sort(keyRanges);
-
- // merge the overlap range
- List<HBaseKeyRange> mergedRanges = new LinkedList<HBaseKeyRange>();
- int beginIndex = 0;
- byte[] maxStopKey = keyRanges.get(0).getStopKey();
- for (int index = 0; index < keyRanges.size(); index++) {
- HBaseKeyRange keyRange = keyRanges.get(index);
- if (Bytes.compareTo(maxStopKey, keyRange.getStartKey()) < 0) {
- // merge the current key ranges
- HBaseKeyRange mergedRange = mergeKeyRange(keyRanges, beginIndex, index - 1);
- mergedRanges.add(mergedRange);
- // start new merge
- beginIndex = index;
- }
- if (Bytes.compareTo(maxStopKey, keyRange.getStopKey()) < 0) {
- // update the stop key
- maxStopKey = keyRange.getStopKey();
- }
- }
- // merge last range
- HBaseKeyRange mergedRange = mergeKeyRange(keyRanges, beginIndex, keyRanges.size() - 1);
- mergedRanges.add(mergedRange);
- if (logger.isDebugEnabled()) {
- logger.debug("Merging key range to " + mergedRanges.size());
- }
- return mergedRanges;
- }
-
- private HBaseKeyRange mergeKeyRange(List<HBaseKeyRange> keyRanges, int from, int to) {
- HBaseKeyRange keyRange = keyRanges.get(from);
- int mergeSize = to - from + 1;
- if (mergeSize > 1) {
- // merge range from mergeHeader to i - 1
- CubeSegment cubeSegment = keyRange.getCubeSegment();
- Cuboid cuboid = keyRange.getCuboid();
- byte[] startKey = keyRange.getStartKey();
- byte[] stopKey = keyRange.getStopKey();
- long partitionColumnStartDate = Long.MAX_VALUE;
- long partitionColumnEndDate = 0;
-
- TreeSet<Pair<byte[], byte[]>> newFuzzyKeys = new TreeSet<>(new Comparator<Pair<byte[], byte[]>>() {
- @Override
- public int compare(Pair<byte[], byte[]> o1, Pair<byte[], byte[]> o2) {
- int partialResult = Bytes.compareTo(o1.getFirst(), o2.getFirst());
- if (partialResult != 0) {
- return partialResult;
- } else {
- return Bytes.compareTo(o1.getSecond(), o2.getSecond());
- }
- }
- });
- List<Collection<ColumnValueRange>> newFlatOrAndFilter = Lists.newLinkedList();
-
- boolean hasNonFuzzyRange = false;
- for (int k = from; k <= to; k++) {
- HBaseKeyRange nextRange = keyRanges.get(k);
- hasNonFuzzyRange = hasNonFuzzyRange || nextRange.getFuzzyKeys().isEmpty();
- newFuzzyKeys.addAll(nextRange.getFuzzyKeys());
- newFlatOrAndFilter.addAll(nextRange.getFlatOrAndFilter());
- if (Bytes.compareTo(stopKey, nextRange.getStopKey()) < 0) {
- stopKey = nextRange.getStopKey();
- }
- if (nextRange.getPartitionColumnStartDate() > 0 && nextRange.getPartitionColumnStartDate() < partitionColumnStartDate) {
- partitionColumnStartDate = nextRange.getPartitionColumnStartDate();
- }
- if (nextRange.getPartitionColumnEndDate() < Long.MAX_VALUE && nextRange.getPartitionColumnEndDate() > partitionColumnEndDate) {
- partitionColumnEndDate = nextRange.getPartitionColumnEndDate();
- }
- }
-
- // if any range is non-fuzzy, then all fuzzy keys must be cleared
- if (hasNonFuzzyRange) {
- newFuzzyKeys.clear();
- }
-
- partitionColumnStartDate = (partitionColumnStartDate == Long.MAX_VALUE) ? 0 : partitionColumnStartDate;
- partitionColumnEndDate = (partitionColumnEndDate == 0) ? Long.MAX_VALUE : partitionColumnEndDate;
- keyRange = new HBaseKeyRange(cubeSegment, cuboid, startKey, stopKey, Lists.newArrayList(newFuzzyKeys), newFlatOrAndFilter, partitionColumnStartDate, partitionColumnEndDate);
- }
- return keyRange;
- }
-
- private List<HBaseKeyRange> mergeTooManyRanges(List<HBaseKeyRange> keyRanges) {
- if (keyRanges.size() < MERGE_KEYRANGE_THRESHOLD) {
- return keyRanges;
- }
- // TODO: check the distance between range. and merge the large distance range
- List<HBaseKeyRange> mergedRanges = new LinkedList<HBaseKeyRange>();
- HBaseKeyRange mergedRange = mergeKeyRange(keyRanges, 0, keyRanges.size() - 1);
- mergedRanges.add(mergedRange);
- return mergedRanges;
- }
-
- private void dropUnhitSegments(List<HBaseKeyRange> scans) {
- if (cubeDesc.getModel().getPartitionDesc().isPartitioned()) {
- Iterator<HBaseKeyRange> iterator = scans.iterator();
- while (iterator.hasNext()) {
- HBaseKeyRange scan = iterator.next();
- if (scan.hitSegment() == false) {
- iterator.remove();
- }
- }
- }
- }
-
- private List<HBaseKeyRange> duplicateRangeByShard(List<HBaseKeyRange> scans) {
- List<HBaseKeyRange> ret = Lists.newArrayList();
-
- for (HBaseKeyRange scan : scans) {
- CubeSegment segment = scan.getCubeSegment();
-
- byte[] startKey = scan.getStartKey();
- byte[] stopKey = scan.getStopKey();
-
- short cuboidShardNum = segment.getCuboidShardNum(scan.getCuboid().getId());
- short cuboidShardBase = segment.getCuboidBaseShard(scan.getCuboid().getId());
- for (short i = 0; i < cuboidShardNum; ++i) {
- short newShard = ShardingHash.normalize(cuboidShardBase, i, segment.getTotalShards(scan.getCuboid().getId()));
- byte[] newStartKey = duplicateKeyAndChangeShard(newShard, startKey);
- byte[] newStopKey = duplicateKeyAndChangeShard(newShard, stopKey);
- HBaseKeyRange newRange = new HBaseKeyRange(segment, scan.getCuboid(), newStartKey, newStopKey, //
- scan.getFuzzyKeys(), scan.getFlatOrAndFilter(), scan.getPartitionColumnStartDate(), scan.getPartitionColumnEndDate());
- ret.add(newRange);
- }
- }
-
- Collections.sort(ret, new Comparator<HBaseKeyRange>() {
- @Override
- public int compare(HBaseKeyRange o1, HBaseKeyRange o2) {
- return Bytes.compareTo(o1.getStartKey(), o2.getStartKey());
- }
- });
-
- return ret;
- }
-
- private byte[] duplicateKeyAndChangeShard(short newShard, byte[] bytes) {
- byte[] ret = Arrays.copyOf(bytes, bytes.length);
- BytesUtil.writeShort(newShard, ret, 0, RowConstants.ROWKEY_SHARDID_LEN);
- return ret;
- }
-
- private void setThreshold(Collection<TblColRef> dimensions, List<RowValueDecoder> valueDecoders, StorageContext context) {
- if (RowValueDecoder.hasMemHungryMeasures(valueDecoders) == false) {
- return;
- }
-
- int rowSizeEst = dimensions.size() * 3;
- for (RowValueDecoder decoder : valueDecoders) {
- MeasureDesc[] measures = decoder.getMeasures();
- BitSet projectionIndex = decoder.getProjectionIndex();
- for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) {
- FunctionDesc func = measures[i].getFunction();
- // FIXME getStorageBytesEstimate() is not appropriate as here we want size in memory (not in storage)
- rowSizeEst += func.getReturnDataType().getStorageBytesEstimate();
- }
- }
-
- long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst;
- if (rowEst > 0) {
- logger.info("Memory budget is set to: " + rowEst);
- context.setThreshold((int) rowEst);
- } else {
- logger.info("Memory budget is not set.");
- }
- }
-
- private void setLimit(TupleFilter filter, StorageContext context) {
- boolean goodAggr = context.isExactAggregation();
- boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter) && context.isCoprocessorEnabled());
- boolean goodSort = !context.hasSort();
- if (goodAggr && goodFilter && goodSort) {
- logger.info("Enable limit " + context.getLimit());
- context.enableLimit();
- }
- }
-
- private void setCoprocessor(Set<TblColRef> groupsCopD, List<RowValueDecoder> valueDecoders, StorageContext context) {
- ObserverEnabler.enableCoprocessorIfBeneficial(cubeInstance, groupsCopD, valueDecoders, context);
- }
-
- private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
-
- Map<String, List<MeasureDesc>> map = Maps.newHashMap();
- for (MeasureDesc measure : cubeDesc.getMeasures()) {
- MeasureType<?> measureType = measure.getFunction().getMeasureType();
-
- String key = measureType.getClass().getCanonicalName();
- List<MeasureDesc> temp = null;
- if ((temp = map.get(key)) != null) {
- temp.add(measure);
- } else {
- map.put(key, Lists.<MeasureDesc> newArrayList(measure));
- }
- }
-
- for (List<MeasureDesc> sublist : map.values()) {
- sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
deleted file mode 100644
index 64feff0..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.storage.hbase.cube.v1;
-
-import java.io.IOException;
-import java.util.BitSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowKeyDecoder;
-import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
-import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.Tuple;
-import org.apache.kylin.metadata.tuple.TupleInfo;
-import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class CubeTupleConverter {
-
- final CubeSegment cubeSeg;
- final Cuboid cuboid;
- final TupleInfo tupleInfo;
- final RowKeyDecoder rowKeyDecoder;
- final List<RowValueDecoder> rowValueDecoders;
- final List<IDerivedColumnFiller> derivedColFillers;
- final int[] dimensionTupleIdx;
- final int[][] metricsMeasureIdx;
- final int[][] metricsTupleIdx;
-
- final List<MeasureType<?>> measureTypes;
-
- final List<MeasureType.IAdvMeasureFiller> advMeasureFillers;
- final List<Pair<Integer, Integer>> advMeasureIndexInRV;//first=> which rowValueDecoders,second => metric index
-
- public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, List<RowValueDecoder> rowValueDecoders, TupleInfo tupleInfo) {
- this.cubeSeg = cubeSeg;
- this.cuboid = cuboid;
- this.tupleInfo = tupleInfo;
- this.rowKeyDecoder = new RowKeyDecoder(this.cubeSeg);
- this.rowValueDecoders = rowValueDecoders;
- this.derivedColFillers = Lists.newArrayList();
-
- List<TblColRef> dimCols = cuboid.getColumns();
-
- measureTypes = Lists.newArrayList();
- advMeasureFillers = Lists.newArrayListWithCapacity(1);
- advMeasureIndexInRV = Lists.newArrayListWithCapacity(1);
-
- // pre-calculate dimension index mapping to tuple
- dimensionTupleIdx = new int[dimCols.size()];
- for (int i = 0; i < dimCols.size(); i++) {
- TblColRef col = dimCols.get(i);
- dimensionTupleIdx[i] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
- }
-
- // pre-calculate metrics index mapping to tuple
- metricsMeasureIdx = new int[rowValueDecoders.size()][];
- metricsTupleIdx = new int[rowValueDecoders.size()][];
- for (int i = 0; i < rowValueDecoders.size(); i++) {
- RowValueDecoder decoder = rowValueDecoders.get(i);
- MeasureDesc[] measures = decoder.getMeasures();
- BitSet selectedMeasures = decoder.getProjectionIndex();
- metricsMeasureIdx[i] = new int[selectedMeasures.cardinality()];
- metricsTupleIdx[i] = new int[selectedMeasures.cardinality()];
- for (int j = 0, mi = selectedMeasures.nextSetBit(0); j < metricsMeasureIdx[i].length; j++, mi = selectedMeasures.nextSetBit(mi + 1)) {
- FunctionDesc aggrFunc = measures[mi].getFunction();
-
- int tupleIdx;
- if (aggrFunc.needRewrite()) {
- // a rewrite metrics is identified by its rewrite field name
- String rewriteFieldName = aggrFunc.getRewriteFieldName();
- tupleIdx = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
- } else {
- // a non-rewrite metrics (like sum, or dimension playing as metrics) is like a dimension column
- TblColRef col = aggrFunc.getParameter().getColRefs().get(0);
- tupleIdx = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
- }
- metricsMeasureIdx[i][j] = mi;
- metricsTupleIdx[i][j] = tupleIdx;
-
- MeasureType<?> measureType = aggrFunc.getMeasureType();
- if (measureType.needAdvancedTupleFilling()) {
- Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap(measureType.getColumnsNeedDictionary(aggrFunc));
- advMeasureFillers.add(measureType.getAdvancedTupleFiller(aggrFunc, tupleInfo, dictionaryMap));
- advMeasureIndexInRV.add(Pair.newPair(i, mi));
- measureTypes.add(null);
- } else {
- measureTypes.add(measureType);
- }
- }
- }
-
- // prepare derived columns and filler
- Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(dimCols, null);
- for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
- TblColRef[] hostCols = entry.getKey().data;
- for (DeriveInfo deriveInfo : entry.getValue()) {
- IDerivedColumnFiller filler = newDerivedColumnFiller(hostCols, deriveInfo);
- if (filler != null) {
- derivedColFillers.add(filler);
- }
- }
- }
- }
-
- // load only needed dictionaries
- private Map<TblColRef, Dictionary<String>> buildDictionaryMap(List<TblColRef> columnsNeedDictionary) {
- Map<TblColRef, Dictionary<String>> result = Maps.newHashMap();
- for (TblColRef col : columnsNeedDictionary) {
- result.put(col, cubeSeg.getDictionary(col));
- }
- return result;
- }
-
- public List<MeasureType.IAdvMeasureFiller> translateResult(Result hbaseRow, Tuple tuple) {
- try {
- byte[] rowkey = hbaseRow.getRow();
- rowKeyDecoder.decode(rowkey);
- } catch (IOException ex) {
- throw new RuntimeException("Cannot translate hbase result " + hbaseRow);
- }
-
- // dimensions
- List<String> dimensionValues = rowKeyDecoder.getValues();
- for (int i = 0; i < dimensionValues.size(); i++) {
- int tupleIdx = dimensionTupleIdx[i];
- if (tupleIdx >= 0) {
- tuple.setDimensionValue(tupleIdx, dimensionValues.get(i));
- }
- }
-
- // derived
- for (IDerivedColumnFiller filler : derivedColFillers) {
- filler.fillDerivedColumns(dimensionValues, tuple);
- }
-
- // measures
- int index = 0;
- for (int i = 0; i < rowValueDecoders.size(); i++) {
- RowValueDecoder rowValueDecoder = rowValueDecoders.get(i);
- rowValueDecoder.decodeAndConvertJavaObj(hbaseRow);
- Object[] measureValues = rowValueDecoder.getValues();
-
- int[] measureIdx = metricsMeasureIdx[i];
- int[] tupleIdx = metricsTupleIdx[i];
- for (int j = 0; j < measureIdx.length; j++) {
- if (measureTypes.get(index++) != null) {
- tuple.setMeasureValue(tupleIdx[j], measureValues[measureIdx[j]]);
- }
- }
- }
-
- // advanced measure filling, due to possible row split, will complete at caller side
- if (advMeasureFillers.isEmpty()) {
- return null;
- } else {
- for (int i = 0; i < advMeasureFillers.size(); i++) {
- Pair<Integer, Integer> metricLocation = advMeasureIndexInRV.get(i);
- Object measureValue = rowValueDecoders.get(metricLocation.getFirst()).getValues()[metricLocation.getSecond()];
- advMeasureFillers.get(i).reload(measureValue);
- }
- return advMeasureFillers;
- }
- }
-
- private interface IDerivedColumnFiller {
- public void fillDerivedColumns(List<String> rowValues, Tuple tuple);
- }
-
- private IDerivedColumnFiller newDerivedColumnFiller(TblColRef[] hostCols, final DeriveInfo deriveInfo) {
- List<TblColRef> rowColumns = cuboid.getColumns();
-
- final int[] hostColIdx = new int[hostCols.length];
- for (int i = 0; i < hostCols.length; i++) {
- hostColIdx[i] = rowColumns.indexOf(hostCols[i]);
- }
-
- boolean needCopyDerived = false;
- final int[] derivedTupleIdx = new int[deriveInfo.columns.length];
- for (int i = 0; i < deriveInfo.columns.length; i++) {
- TblColRef col = deriveInfo.columns[i];
- derivedTupleIdx[i] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
- needCopyDerived = needCopyDerived || derivedTupleIdx[i] >= 0;
- }
-
- if (needCopyDerived == false)
- return null;
-
- switch (deriveInfo.type) {
- case LOOKUP:
- return new IDerivedColumnFiller() {
- CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
- LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.join);
- int[] derivedColIdx = initDerivedColIdx();
- Array<String> lookupKey = new Array<String>(new String[hostColIdx.length]);
-
- private int[] initDerivedColIdx() {
- int[] idx = new int[deriveInfo.columns.length];
- for (int i = 0; i < idx.length; i++) {
- idx[i] = deriveInfo.columns[i].getColumnDesc().getZeroBasedIndex();
- }
- return idx;
- }
-
- @Override
- public void fillDerivedColumns(List<String> rowValues, Tuple tuple) {
- for (int i = 0; i < hostColIdx.length; i++) {
- lookupKey.data[i] = rowValues.get(hostColIdx[i]);
- }
-
- String[] lookupRow = lookupTable.getRow(lookupKey);
-
- if (lookupRow != null) {
- for (int i = 0; i < derivedTupleIdx.length; i++) {
- if (derivedTupleIdx[i] >= 0) {
- String value = lookupRow[derivedColIdx[i]];
- tuple.setDimensionValue(derivedTupleIdx[i], value);
- }
- }
- } else {
- for (int i = 0; i < derivedTupleIdx.length; i++) {
- if (derivedTupleIdx[i] >= 0) {
- tuple.setDimensionValue(derivedTupleIdx[i], null);
- }
- }
- }
- }
- };
- case PK_FK:
- return new IDerivedColumnFiller() {
- @Override
- public void fillDerivedColumns(List<String> rowValues, Tuple tuple) {
- // composite keys are split, so only copy [0] is enough, see CubeDesc.initDimensionColumns()
- tuple.setDimensionValue(derivedTupleIdx[0], rowValues.get(hostColIdx[0]));
- }
- };
- default:
- throw new IllegalArgumentException();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
deleted file mode 100644
index 8a20c65..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-
-/**
- * @author yangli9
- *
- */
-public class RegionScannerAdapter implements RegionScanner {
-
- private ResultScanner scanner;
-
- public RegionScannerAdapter(ResultScanner scanner) {
- this.scanner = scanner;
- }
-
- @Override
- public boolean next(List<Cell> results) throws IOException {
- Result result = scanner.next();
- if (result == null) // EOF
- return false;
-
- results.addAll(result.listCells());
- return true;
- }
-
- @Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- return next(result);
- }
-
- @Override
- public boolean nextRaw(List<Cell> result) throws IOException {
- return next(result);
- }
-
- @Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return next(result);
- }
-
- @Override
- public void close() throws IOException {
- scanner.close();
- }
-
- @Override
- public HRegionInfo getRegionInfo() {
- return null;
- }
-
- @Override
- public long getMaxResultSize() {
- return Long.MAX_VALUE;
- }
-
- @Override
- public boolean isFilterDone() throws IOException {
- return false;
- }
-
- @Override
- public boolean reseek(byte[] row) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getMvccReadPoint() {
- return Long.MAX_VALUE;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/ResultScannerAdapter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/ResultScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/ResultScannerAdapter.java
deleted file mode 100644
index 99058e7..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/ResultScannerAdapter.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-
-import com.google.common.collect.Lists;
-
-/**
- * @author yangli9
- *
- */
-public class ResultScannerAdapter implements ResultScanner {
-
- private RegionScanner scanner;
-
- public ResultScannerAdapter(RegionScanner scanner) {
- this.scanner = scanner;
- }
-
- @Override
- public Iterator<Result> iterator() {
- return new Iterator<Result>() {
-
- Result next = null;
-
- @Override
- public boolean hasNext() {
- if (next == null) {
- try {
- next = ResultScannerAdapter.this.next();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- return next != null;
- }
-
- @Override
- public Result next() {
- Result r = next;
- next = null;
- return r;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-
- @Override
- public Result next() throws IOException {
- List<Cell> cells = Lists.newArrayList();
- scanner.next(cells);
- if (cells.isEmpty())
- return null;
- else
- return Result.create(cells);
- }
-
- @Override
- public Result[] next(int nbRows) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- try {
- scanner.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
deleted file mode 100644
index e8dd5b9..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.TupleInfo;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.exception.ScanOutOfLimitException;
-import org.apache.kylin.storage.hbase.steps.RowValueDecoder;
-import org.apache.kylin.storage.translate.HBaseKeyRange;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * @author xjiang
- */
-public class SerializedHBaseTupleIterator implements ITupleIterator {
-
- private static final int PARTIAL_DEFAULT_LIMIT = 10000;
-
- private final StorageContext context;
- private final int partialResultLimit;
- private final List<CubeSegmentTupleIterator> segmentIteratorList;
- private final Iterator<CubeSegmentTupleIterator> segmentIteratorIterator;
-
- private ITupleIterator segmentIterator;
- private int scanCount;
- private ITuple next;
-
- public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
- Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, //
- StorageContext context, TupleInfo returnTupleInfo) {
-
- this.context = context;
- int limit = context.getLimit();
- this.partialResultLimit = Math.max(limit, PARTIAL_DEFAULT_LIMIT);
-
- this.segmentIteratorList = new ArrayList<CubeSegmentTupleIterator>(segmentKeyRanges.size());
- Map<CubeSegment, List<HBaseKeyRange>> rangesMap = makeRangesMap(segmentKeyRanges);
-
- for (Map.Entry<CubeSegment, List<HBaseKeyRange>> entry : rangesMap.entrySet()) {
- CubeSegmentTupleIterator it = new CubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context, returnTupleInfo);
- this.segmentIteratorList.add(it);
- }
-
- this.segmentIteratorIterator = this.segmentIteratorList.iterator();
- if (this.segmentIteratorIterator.hasNext()) {
- this.segmentIterator = this.segmentIteratorIterator.next();
- } else {
- this.segmentIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR;
- }
- }
-
- private Map<CubeSegment, List<HBaseKeyRange>> makeRangesMap(List<HBaseKeyRange> segmentKeyRanges) {
- Map<CubeSegment, List<HBaseKeyRange>> map = Maps.newHashMap();
- for (HBaseKeyRange range : segmentKeyRanges) {
- List<HBaseKeyRange> list = map.get(range.getCubeSegment());
- if (list == null) {
- list = Lists.newArrayList();
- map.put(range.getCubeSegment(), list);
- }
- list.add(range);
- }
- return map;
- }
-
- @Override
- public boolean hasNext() {
- if (next != null)
- return true;
-
- // 1. check limit
- if (context.isLimitEnabled() && scanCount >= context.getLimit() + context.getOffset()) {
- return false;
- }
- // 2. check partial result
- if (context.isAcceptPartialResult() && scanCount > partialResultLimit) {
- context.setPartialResultReturned(true);
- return false;
- }
- // 3. check threshold
- if (scanCount >= context.getThreshold()) {
- throw new ScanOutOfLimitException("Scan row count exceeded threshold: " + context.getThreshold() + ", please add filter condition to narrow down backend scan range, like where clause.");
- }
- // 4. check cube segments
- if (segmentIterator.hasNext()) {
- next = segmentIterator.next();
- scanCount++;
- return true;
- } else if (segmentIteratorIterator.hasNext()) {
- segmentIterator.close();
- segmentIterator = segmentIteratorIterator.next();
- return hasNext();
- }
- return false;
- }
-
- @Override
- public ITuple next() {
- if (next == null) {
- hasNext();
- if (next == null)
- throw new NoSuchElementException();
- }
- ITuple r = next;
- next = null;
- return r;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- // hasNext() loop may exit because of limit, threshold, etc.
- // close all the remaining segmentIterator
- segmentIterator.close();
- while (segmentIteratorIterator.hasNext()) {
- segmentIterator = segmentIteratorIterator.next();
- segmentIterator.close();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/545201f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
deleted file mode 100644
index 7139ca7..0000000
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.kylin.gridtable.StorageSideBehavior;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
-
-/**
- * @author yangli9
- */
-public class AggregateRegionObserver extends BaseRegionObserver {
-
- // HBase uses common logging (vs. Kylin uses slf4j)
- static final Log LOG = LogFactory.getLog(AggregateRegionObserver.class);
-
- static final String COPROCESSOR_ENABLE = "_Coprocessor_Enable";
- static final String TYPE = "_Type";
- static final String PROJECTOR = "_Projector";
- static final String AGGREGATORS = "_Aggregators";
- static final String FILTER = "_Filter";
- static final String BEHAVIOR = "_Behavior";
-
- @Override
- public final RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> ctxt, final Scan scan, final RegionScanner innerScanner) throws IOException {
-
- boolean copAbortOnError = ctxt.getEnvironment().getConfiguration().getBoolean(RegionCoprocessorHost.ABORT_ON_ERROR_KEY, RegionCoprocessorHost.DEFAULT_ABORT_ON_ERROR);
-
- // never throw out exception that could abort region server
- if (copAbortOnError) {
- try {
- return doPostScannerObserver(ctxt, scan, innerScanner);
- } catch (Throwable e) {
- LOG.error("Kylin Coprocessor Error", e);
- return innerScanner;
- }
- } else {
- return doPostScannerObserver(ctxt, scan, innerScanner);
- }
- }
-
- private RegionScanner doPostScannerObserver(final ObserverContext<RegionCoprocessorEnvironment> ctxt, final Scan scan, final RegionScanner innerScanner) throws IOException {
- byte[] coprocessorEnableBytes = scan.getAttribute(COPROCESSOR_ENABLE);
- if (coprocessorEnableBytes == null || coprocessorEnableBytes.length == 0 || coprocessorEnableBytes[0] == 0) {
- return innerScanner;
- }
-
- byte[] typeBytes = scan.getAttribute(TYPE);
- CoprocessorRowType type = CoprocessorRowType.deserialize(typeBytes);
-
- byte[] projectorBytes = scan.getAttribute(PROJECTOR);
- CoprocessorProjector projector = CoprocessorProjector.deserialize(projectorBytes);
-
- byte[] aggregatorBytes = scan.getAttribute(AGGREGATORS);
- ObserverAggregators aggregators = ObserverAggregators.deserialize(aggregatorBytes);
-
- byte[] filterBytes = scan.getAttribute(FILTER);
- CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);
-
- StorageSideBehavior storageSideBehavior = StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM;
- try {
- byte[] behavior = scan.getAttribute(BEHAVIOR);
- if (behavior != null && behavior.length != 0) {
- storageSideBehavior = StorageSideBehavior.valueOf(new String(behavior));
- }
- } catch (Exception e) {
- LOG.error("failed to parse behavior,using default behavior SCAN_FILTER_AGGR_CHECKMEM", e);
- storageSideBehavior = StorageSideBehavior.SCAN_FILTER_AGGR_CHECKMEM;
- }
-
- // start/end region operation & sync on scanner is suggested by the
- // javadoc of RegionScanner.nextRaw()
- // FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
- HRegion region = ctxt.getEnvironment().getRegion();
- region.startRegionOperation();
- try {
- synchronized (innerScanner) {
- return new AggregationScanner(type, filter, projector, aggregators, innerScanner, storageSideBehavior);
- }
- } finally {
- region.closeRegionOperation();
- }
- }
-}
[04/18] kylin git commit: KYLIN-1875 Support measure on
non-root-fact-table
Posted by li...@apache.org.
KYLIN-1875 Support measure on non-root-fact-table
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/79e48011
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/79e48011
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/79e48011
Branch: refs/heads/master-hbase1.x
Commit: 79e48011bd3f4696a916f0bfee244ec28355f3b3
Parents: 64d9b8b
Author: Li Yang <li...@apache.org>
Authored: Thu Dec 1 18:59:08 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Dec 7 17:53:38 2016 +0800
----------------------------------------------------------------------
.../kylin/metadata/project/ProjectL2Cache.java | 26 +++++++++++++-------
.../template/cube_desc/kylin_sales_cube.json | 18 +++++++-------
.../kylin/query/relnode/OLAPTableScan.java | 4 +++
3 files changed, 30 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/79e48011/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 82c0de3..14d7843 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -25,6 +25,7 @@ import java.util.Set;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.ExternalFilterDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -126,17 +127,17 @@ class ProjectL2Cache {
return Collections.unmodifiableSet(tableCache.realizations);
}
- public List<MeasureDesc> listEffectiveRewriteMeasures(String project, String factTable, boolean onlyRewriteMeasure) {
- Set<IRealization> realizations = getRealizationsByTable(project, factTable);
+ public List<MeasureDesc> listEffectiveRewriteMeasures(String project, String table, boolean onlyRewriteMeasure) {
+ Set<IRealization> realizations = getRealizationsByTable(project, table);
List<MeasureDesc> result = Lists.newArrayList();
for (IRealization r : realizations) {
- if (r.getModel().isFactTable(factTable) && r.isReady()) {
- for (MeasureDesc m : r.getMeasures()) {
- FunctionDesc func = m.getFunction();
- if (onlyRewriteMeasure) {
- if (func.needRewrite())
- result.add(m);
- } else {
+ if (!r.isReady())
+ continue;
+
+ for (MeasureDesc m : r.getMeasures()) {
+ FunctionDesc func = m.getFunction();
+ if (belongToTable(func, table, r.getModel())) {
+ if (!onlyRewriteMeasure || func.needRewrite()) {
result.add(m);
}
}
@@ -145,6 +146,13 @@ class ProjectL2Cache {
return result;
}
+ private boolean belongToTable(FunctionDesc func, String table, DataModelDesc model) {
+ // measure belong to the first column parameter's table
+ List<TblColRef> cols = func.getParameter().getColRefs();
+ String belongTo = cols.isEmpty() ? model.getRootFactTable().getTableIdentity() : cols.get(0).getTable();
+ return belongTo.equals(table);
+ }
+
// ============================================================================
// build the cache
// ----------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/79e48011/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
index 48eef46..2a27305 100644
--- a/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube.json
@@ -103,26 +103,26 @@
"returntype" : "decimal(19,4)"
}
}, {
- "name" : "GMV_MIN",
+ "name" : "BUYER_LEVEL_SUM",
"function" : {
- "expression" : "MIN",
+ "expression" : "SUM",
"parameter" : {
"type" : "column",
- "value" : "PRICE",
+ "value" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
"next_parameter" : null
},
- "returntype" : "decimal(19,4)"
+ "returntype" : "bigint"
}
}, {
- "name" : "GMV_MAX",
+ "name" : "SELLER_LEVEL_SUM",
"function" : {
- "expression" : "MAX",
+ "expression" : "SUM",
"parameter" : {
"type" : "column",
- "value" : "PRICE",
+ "value" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
"next_parameter" : null
},
- "returntype" : "decimal(19,4)"
+ "returntype" : "bigint"
}
}, {
"name" : "TRANS_CNT",
@@ -224,7 +224,7 @@
"name" : "F1",
"columns" : [ {
"qualifier" : "M",
- "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
+ "measure_refs" : [ "GMV_SUM", "BUYER_LEVEL_SUM", "SELLER_LEVEL_SUM", "TRANS_CNT" ]
} ]
}, {
"name" : "F2",
http://git-wip-us.apache.org/repos/asf/kylin/blob/79e48011/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index 14758c9..b157f34 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -236,6 +236,10 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
TblColRef colRef = TblColRef.columnForUnknownModel(tableRef, sourceColumn);
columns.add(colRef);
}
+
+ if (columns.size() != rowType.getFieldCount()) {
+ throw new IllegalStateException("RowType=" + rowType.getFieldCount() + ", ColumnRowType=" + columns.size());
+ }
return new ColumnRowType(columns);
}
[05/18] kylin git commit: KYLIN-1875 fix PK-FK derive from each other
Posted by li...@apache.org.
KYLIN-1875 fix PK-FK derive from each other
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92e4d464
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92e4d464
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92e4d464
Branch: refs/heads/master-hbase1.x
Commit: 92e4d464069745f0c58cc0559e054efa470960c1
Parents: 79e4801
Author: Yang Li <li...@apache.org>
Authored: Thu Dec 1 21:42:52 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Dec 7 17:53:39 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 8 +--
.../org/apache/kylin/cube/model/CubeDesc.java | 52 +++++++++++---------
.../storage/gtrecord/CubeTupleConverter.java | 2 +-
.../gtrecord/GTCubeStorageQueryBase.java | 2 +-
.../translate/DerivedFilterTranslator.java | 2 +-
.../apache/kylin/query/ITKylinQueryTest.java | 2 +-
.../query/enumerator/LookupTableEnumerator.java | 2 +-
.../storage/hbase/cube/v1/CubeStorageQuery.java | 2 +-
.../hbase/cube/v1/CubeTupleConverter.java | 2 +-
9 files changed, 41 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index bda1423..fe2030a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -44,7 +44,6 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DictionaryDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.lookup.LookupStringTable;
@@ -54,6 +53,7 @@ import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TableDesc;
@@ -629,10 +629,10 @@ public class CubeManager implements IRealizationProvider {
cubeMap.removeLocal(cubeName);
}
- public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {
+ public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) {
- String tableName = dim.getTableRef().getTableIdentity();
- String[] pkCols = dim.getJoin().getPrimaryKey();
+ String tableName = join.getPKSide().getTableIdentity();
+ String[] pkCols = join.getPrimaryKey();
String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
if (snapshotResPath == null)
throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);
http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f95cceb..f8c316c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -62,6 +62,7 @@ import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -101,20 +102,20 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
public static class DeriveInfo {
public DeriveType type;
- public DimensionDesc dimension;
+ public JoinDesc join;
public TblColRef[] columns;
public boolean isOneToOne; // only used when ref from derived to host
- DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean isOneToOne) {
+ DeriveInfo(DeriveType type, JoinDesc join, TblColRef[] columns, boolean isOneToOne) {
this.type = type;
- this.dimension = dimension;
+ this.join = join;
this.columns = columns;
this.isOneToOne = isOneToOne;
}
@Override
public String toString() {
- return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]";
+ return "DeriveInfo [type=" + type + ", join=" + join + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]";
}
}
@@ -742,27 +743,34 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
for (int i = 0; i < derivedNames.length; i++) {
derivedCols[i] = initDimensionColRef(dim, derivedNames[i]);
}
- initDerivedMap(dimColArray, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
+ initDerivedMap(dimColArray, DeriveType.LOOKUP, join, derivedCols, derivedExtra);
}
-
- // PK-FK derive the other side
+
if (join != null) {
- TblColRef[] fk = join.getForeignKeyColumns();
- TblColRef[] pk = join.getPrimaryKeyColumns();
-
- allColumns.addAll(Arrays.asList(fk));
- allColumns.addAll(Arrays.asList(pk));
- for (int i = 0; i < fk.length; i++) {
- int find = ArrayUtils.indexOf(dimColArray, fk[i]);
- if (find >= 0) {
- TblColRef derivedCol = initDimensionColRef(pk[i]);
- initDerivedMap(new TblColRef[] { dimColArray[find] }, DeriveType.PK_FK, dim, new TblColRef[] { derivedCol }, null);
- }
+ allColumns.addAll(Arrays.asList(join.getForeignKeyColumns()));
+ allColumns.addAll(Arrays.asList(join.getPrimaryKeyColumns()));
+ }
+ }
+
+ // PK-FK derive the other side
+ Set<TblColRef> realDimensions = new HashSet<>(listDimensionColumnsExcludingDerived(true));
+ for (JoinTableDesc joinTable : model.getJoinTables()) {
+ JoinDesc join = joinTable.getJoin();
+ int n = join.getForeignKeyColumns().length;
+ for (int i = 0; i < n; i++) {
+ TblColRef pk = join.getPrimaryKeyColumns()[i];
+ TblColRef fk = join.getForeignKeyColumns()[i];
+ if (realDimensions.contains(pk) && !realDimensions.contains(fk)) {
+ initDimensionColRef(fk);
+ initDerivedMap(new TblColRef[] { pk }, DeriveType.PK_FK, join, new TblColRef[] { fk }, null);
+ } else if (realDimensions.contains(fk) && !realDimensions.contains(pk)) {
+ initDimensionColRef(pk);
+ initDerivedMap(new TblColRef[] { fk }, DeriveType.PK_FK, join, new TblColRef[] { pk }, null);
}
}
}
}
-
+
private String[][] splitDerivedColumnAndExtra(String[] derived) {
String[] cols = new String[derived.length];
String[] extra = new String[derived.length];
@@ -780,7 +788,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return new String[][] { cols, extra };
}
- private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) {
+ private void initDerivedMap(TblColRef[] hostCols, DeriveType type, JoinDesc join, TblColRef[] derivedCols, String[] extra) {
if (hostCols.length == 0 || derivedCols.length == 0)
throw new IllegalStateException("host/derived columns must not be empty");
@@ -803,12 +811,12 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
if (infoList == null) {
hostToMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
}
- infoList.add(new DeriveInfo(type, dimension, derivedCols, false));
+ infoList.add(new DeriveInfo(type, join, derivedCols, false));
for (int i = 0; i < derivedCols.length; i++) {
TblColRef derivedCol = derivedCols[i];
boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1"));
- toHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne));
+ toHostMap.put(derivedCol, new DeriveInfo(type, join, hostCols, isOneToOne));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 7ec24b2..3159318 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@ -266,7 +266,7 @@ public class CubeTupleConverter {
case LOOKUP:
return new IDerivedColumnFiller() {
CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
- LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.dimension);
+ LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.join);
int[] derivedColIdx = initDerivedColIdx();
Array<String> lookupKey = new Array<String>(new String[hostTmpIdx.length]);
http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 9c74cca..31573d0 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -294,7 +294,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
CubeSegment seg = cubeInstance.getLatestReadySegment();
- LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
+ LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.join);
Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
TupleFilter translatedFilter = translated.getFirst();
boolean loosened = translated.getSecond();
http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
index 160338f..13c655c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/DerivedFilterTranslator.java
@@ -54,7 +54,7 @@ public class DerivedFilterTranslator {
TblColRef derivedCol = compf.getColumn();
TblColRef[] hostCols = hostInfo.columns;
- TblColRef[] pkCols = hostInfo.dimension.getJoin().getPrimaryKeyColumns();
+ TblColRef[] pkCols = hostInfo.join.getPrimaryKeyColumns();
if (hostInfo.type == DeriveType.PK_FK) {
assert hostCols.length == 1;
http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 90324b5..6d91753 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -48,7 +48,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
-@Ignore("KylinQueryTest is contained by ITCombinationTest")
+//@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
@Rule
http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
index 1c2bc2d..28ee623 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/LookupTableEnumerator.java
@@ -67,7 +67,7 @@ public class LookupTableEnumerator implements Enumerator<Object[]> {
throw new IllegalStateException("No dimension with derived columns found for lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor());
CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig());
- LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim);
+ LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim.getJoin());
this.allRows = table.getAllRows();
OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable();
http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 02aa64a..75c3fd7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -368,7 +368,7 @@ public class CubeStorageQuery implements IStorageQuery {
DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
CubeSegment seg = cubeInstance.getLatestReadySegment();
- LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
+ LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.join);
Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
TupleFilter translatedFilter = translated.getFirst();
boolean loosened = translated.getSecond();
http://git-wip-us.apache.org/repos/asf/kylin/blob/92e4d464/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
index d2378b9..64feff0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeTupleConverter.java
@@ -220,7 +220,7 @@ public class CubeTupleConverter {
case LOOKUP:
return new IDerivedColumnFiller() {
CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
- LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.dimension);
+ LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSeg, deriveInfo.join);
int[] derivedColIdx = initDerivedColIdx();
Array<String> lookupKey = new Array<String>(new String[hostColIdx.length]);
[10/18] kylin git commit: minor, fix date format in sample data
Posted by li...@apache.org.
minor, fix date format in sample data
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21bcd2f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21bcd2f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21bcd2f6
Branch: refs/heads/master-hbase1.x
Commit: 21bcd2f6b7d56e639aba18f3f376875922c9cd68
Parents: 350fad8
Author: lidongsjtu <li...@apache.org>
Authored: Thu Dec 8 10:40:45 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Dec 8 10:40:55 2016 +0800
----------------------------------------------------------------------
.../sample_cube/data/DEFAULT.KYLIN_SALES.csv | 20000 ++++++++---------
1 file changed, 10000 insertions(+), 10000 deletions(-)
----------------------------------------------------------------------
[15/18] kylin git commit: KYLIN-2217 fix null partition col bug
Posted by li...@apache.org.
KYLIN-2217 fix null partition col bug
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c0c56f4b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c0c56f4b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c0c56f4b
Branch: refs/heads/master-hbase1.x
Commit: c0c56f4b6b05bb108a4952a56af992b237c1bbe6
Parents: 545201f
Author: xiefan46 <95...@qq.com>
Authored: Thu Dec 8 10:33:48 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 8 11:09:31 2016 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c0c56f4b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 8933ee2..a3e61a1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -116,13 +116,17 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
isStatistics = false;
isPartitionCol = true;
col = cubeDesc.getModel().getPartitionDesc().getPartitionDateColumnRef();
+ if (col == null) {
+ isPartitionCol = false;
+ logger.info("Do not have partition col. This reducer will keep empty");
+ }
colValues = Lists.newLinkedList();
} else {
// normal col
isStatistics = false;
col = columnList.get(reducerIdToColumnIndex.get(taskId));
colValues = Lists.newLinkedList();
-
+
// local build dict
isReducerLocalBuildDict = config.isReducerLocalBuildDict();
if (col != null && isReducerLocalBuildDict) {
@@ -279,6 +283,8 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
cuboidHLLMap, samplingPercentage, mapperNumber, mapperOverlapRatio);
} else if (isPartitionCol) {
// partition col
+ if (col == null)
+ return;
outputPartitionInfo(context);
} else {
// normal col
[09/18] kylin git commit: minor, fix date format in sample data
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/21bcd2f6/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
----------------------------------------------------------------------
diff --git a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
index e13b7ad..2555515 100644
--- a/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
+++ b/examples/sample_cube/data/DEFAULT.KYLIN_SALES.csv
@@ -1,10000 +1,10000 @@
-0,14/12/2012,Others,88750,0,11,36.2828,4,10000349,10002313,ANALYST,Beijing
-1,28/08/2012,Others,175750,0,13,23.8563,20,10000927,10004376,ANALYST,Beijing
-2,16/02/2012,ABIN,148324,15,13,88.3418,18,10000005,10006710,ADMIN,Shanghai
-3,19/10/2013,FP-non GTC,37831,0,13,47.3015,3,10000209,10003717,ANALYST,Beijing
-4,22/10/2012,Others,140746,100,11,83.454,16,10000154,10006076,ADMIN,Shanghai
-5,25/01/2013,FP-GTC,16509,0,-99,22.9896,11,10000372,10007436,ADMIN,Shanghai
-6,04/04/2013,Others,963,0,13,88.5907,11,10000648,10009869,MODELER,Hongkong
-7,11/04/2012,Others,15687,0,15,88.194,19,10000866,10000400,ADMIN,Shanghai
-8,19/01/2013,ABIN,60606,3,13,77.9727,13,10000936,10000687,MODELER,Hongkong
-9,30/04/2012,FP-non GTC,106246,0,14,52.295,17,10000047,10009223,MODELER,Hongkong
-10,03/02/2013,Auction,45333,0,16,56.3584,10,10000470,10006759,MODELER,Hongkong
-11,02/09/2012,FP-GTC,25147,0,-99,15.3553,8,10000203,10000062,ANALYST,Beijing
-12,30/07/2013,Others,24760,0,16,25.077,5,10000382,10003539,ADMIN,Shanghai
-13,17/04/2012,Auction,31519,0,16,72.7384,11,10000069,10001245,ADMIN,Shanghai
-14,30/04/2013,FP-GTC,51582,0,15,75.82,6,10000085,10008148,ANALYST,Beijing
-15,23/03/2013,FP-GTC,16509,0,15,8.6653,15,10000559,10008079,MODELER,Hongkong
-16,01/04/2013,ABIN,175750,3,16,64.2802,16,10000507,10007797,MODELER,Hongkong
-17,12/07/2013,ABIN,50677,0,-99,24.5987,17,10000938,10006224,ADMIN,Shanghai
-18,23/04/2012,FP-GTC,45333,0,5,72.6553,15,10000863,10003824,MODELER,Hongkong
-19,19/12/2012,ABIN,3838,0,12,87.4142,4,10000612,10007157,ADMIN,Shanghai
-20,15/08/2012,FP-GTC,62179,0,16,67.4238,15,10000839,10003452,MODELER,Hongkong
-21,22/08/2013,Auction,1357,0,5,18.36,9,10000303,10000114,ANALYST,Beijing
-22,23/08/2013,ABIN,31519,0,5,67.6344,2,10000877,10007417,ANALYST,Beijing
-23,12/09/2013,FP-GTC,223,0,14,71.7898,15,10000848,10000627,MODELER,Hongkong
-24,14/03/2012,FP-GTC,132939,0,15,26.1496,20,10000815,10005292,MODELER,Hongkong
-25,13/07/2013,Others,67703,3,16,83.9661,2,10000858,10008123,MODELER,Hongkong
-26,15/12/2013,FP-non GTC,100847,0,5,34.3451,16,10000387,10007806,ADMIN,Shanghai
-27,30/08/2012,Auction,62179,0,11,21.0496,9,10000393,10007360,ANALYST,Beijing
-28,10/01/2013,FP-GTC,48027,0,13,23.4142,20,10000757,10002135,MODELER,Hongkong
-29,25/01/2013,ABIN,145970,0,5,26.7842,6,10000402,10009682,ANALYST,Beijing
-30,05/07/2012,Auction,48027,0,-99,46.732,5,10000972,10009522,MODELER,Hongkong
-31,07/11/2012,FP-GTC,20213,0,-99,19.727,8,10000873,10009944,ANALYST,Beijing
-32,20/06/2013,ABIN,100847,0,5,76.2783,1,10000409,10005395,ADMIN,Shanghai
-33,08/10/2012,ABIN,26262,0,5,48.7447,4,10000029,10001836,ADMIN,Shanghai
-34,30/04/2013,ABIN,87118,0,15,63.2187,11,10000529,10000928,ANALYST,Beijing
-35,24/04/2013,FP-non GTC,20213,0,16,39.5057,19,10000329,10000768,ANALYST,Beijing
-36,13/05/2012,Others,164261,0,13,84.1246,15,10000688,10000727,ADMIN,Shanghai
-37,02/08/2013,ABIN,31519,0,5,37.1504,14,10000905,10000719,MODELER,Hongkong
-38,03/04/2012,Auction,164261,0,11,92.0974,13,10000341,10001337,ADMIN,Shanghai
-39,02/09/2012,FP-GTC,31519,0,14,87.7829,20,10000460,10008484,ANALYST,Beijing
-40,05/10/2013,ABIN,13987,0,16,74.0719,4,10000381,10002975,MODELER,Hongkong
-41,13/01/2012,FP-GTC,48027,0,15,71.2637,14,10000915,10005778,ADMIN,Shanghai
-42,01/03/2013,Others,13836,0,14,16.7288,14,10000953,10009385,ADMIN,Shanghai
-43,12/09/2012,FP-non GTC,16509,0,5,12.2933,20,10000277,10002286,MODELER,Hongkong
-44,07/02/2012,Auction,45333,0,5,64.977,16,10000040,10005583,MODELER,Hongkong
-45,14/12/2013,FP-non GTC,158798,0,16,72.4413,18,10000500,10007635,MODELER,Hongkong
-46,13/10/2013,Auction,31519,0,5,79.3053,17,10000816,10006423,MODELER,Hongkong
-47,11/06/2012,ABIN,4943,0,5,11.6942,15,10000696,10001237,ANALYST,Beijing
-48,18/10/2012,ABIN,80053,0,-99,54.0933,4,10000604,10006851,MODELER,Hongkong
-49,03/11/2012,Auction,20886,0,5,9.8258,8,10000764,10001220,MODELER,Hongkong
-50,15/01/2012,Auction,44079,0,14,13.0371,19,10000343,10001259,MODELER,Hongkong
-51,03/07/2012,FP-GTC,159184,0,-99,92.5314,4,10000724,10009008,ADMIN,Shanghai
-52,16/02/2012,FP-GTC,314,0,13,49.1825,20,10000334,10004005,ADMIN,Shanghai
-53,17/06/2012,FP-non GTC,31387,3,12,0.6677,5,10000603,10002595,ANALYST,Beijing
-54,15/07/2012,FP-non GTC,32876,0,-99,50.9634,11,10000248,10006890,ANALYST,Beijing
-55,24/07/2012,FP-GTC,150265,15,16,57.5645,6,10000653,10007269,ADMIN,Shanghai
-56,06/10/2012,Auction,145970,0,5,44.3091,16,10000082,10001640,ADMIN,Shanghai
-57,14/06/2013,ABIN,80053,0,5,20.2388,19,10000632,10002068,MODELER,Hongkong
-58,10/10/2013,FP-non GTC,15868,0,11,28.629,15,10000523,10004108,MODELER,Hongkong
-59,30/09/2012,ABIN,31673,0,12,75.8658,20,10000719,10009011,ADMIN,Shanghai
-60,02/04/2012,FP-GTC,63861,0,5,25.8748,3,10000311,10000289,ANALYST,Beijing
-61,16/03/2012,Others,314,0,-99,9.2733,19,10000409,10001533,MODELER,Hongkong
-62,30/09/2012,FP-non GTC,60340,0,14,74.661,8,10000790,10006831,MODELER,Hongkong
-63,20/03/2012,Others,87118,0,5,79.3781,1,10000384,10000632,MODELER,Hongkong
-64,12/09/2012,FP-GTC,1120,3,5,61.4408,11,10000645,10001462,ANALYST,Beijing
-65,10/12/2013,FP-non GTC,20886,0,11,14.814,18,10000943,10008334,ANALYST,Beijing
-66,05/12/2012,FP-GTC,106246,0,12,24.938,17,10000858,10007796,ANALYST,Beijing
-67,21/08/2012,Auction,43479,0,12,22.2008,8,10000310,10001440,MODELER,Hongkong
-68,30/11/2012,FP-GTC,175750,0,15,35.1161,8,10000440,10007199,ADMIN,Shanghai
-69,10/10/2013,Others,95672,0,11,82.3718,16,10000128,10003194,ADMIN,Shanghai
-70,17/10/2012,Auction,80053,0,15,9.8166,7,10000105,10009975,ADMIN,Shanghai
-71,16/03/2012,FP-non GTC,57784,0,5,66.2369,9,10000439,10008763,MODELER,Hongkong
-72,14/11/2012,ABIN,65,0,11,17.5513,3,10000855,10007000,MODELER,Hongkong
-73,24/06/2013,FP-GTC,16509,0,15,53.9195,7,10000130,10005095,MODELER,Hongkong
-74,13/08/2012,FP-non GTC,65,0,5,69.8447,8,10000548,10005613,MODELER,Hongkong
-75,03/01/2012,Others,38238,0,11,90.3722,3,10000739,10004281,ADMIN,Shanghai
-76,24/11/2013,FP-non GTC,175750,3,13,47.3441,5,10000291,10003352,ANALYST,Beijing
-77,08/11/2013,Auction,106246,0,5,38.6729,4,10000310,10007150,ADMIN,Shanghai
-78,07/04/2013,Others,73506,0,-99,69.8999,17,10000235,10003270,ADMIN,Shanghai
-79,15/10/2012,FP-GTC,314,0,13,44.1598,14,10000265,10001620,ADMIN,Shanghai
-80,20/01/2012,FP-GTC,80287,0,-99,85.7263,12,10000704,10002716,ANALYST,Beijing
-81,17/08/2012,ABIN,32876,0,16,21.2934,14,10000163,10007391,ANALYST,Beijing
-82,09/07/2013,Auction,37831,0,15,38.2213,13,10000557,10006017,ADMIN,Shanghai
-83,21/09/2013,Others,45333,0,13,25.5384,6,10000184,10003754,MODELER,Hongkong
-84,28/10/2013,FP-non GTC,11554,0,16,89.9654,8,10000940,10005821,MODELER,Hongkong
-85,05/04/2013,Auction,11554,0,14,41.8676,15,10000384,10007941,ANALYST,Beijing
-86,25/03/2013,FP-GTC,43398,0,14,75.011,12,10000697,10006079,ANALYST,Beijing
-87,03/05/2012,ABIN,33038,15,12,64.6735,20,10000147,10001734,ADMIN,Shanghai
-88,13/10/2012,ABIN,95672,3,14,3.2128,13,10000547,10003232,MODELER,Hongkong
-89,11/09/2012,FP-GTC,37831,0,16,60.3228,15,10000899,10002001,MODELER,Hongkong
-90,31/08/2012,Auction,314,0,5,13.248,13,10000350,10005727,ANALYST,Beijing
-91,25/12/2012,Others,165888,0,16,60.3492,9,10000367,10000234,ANALYST,Beijing
-92,26/12/2012,FP-non GTC,50677,0,5,58.6078,19,10000523,10003148,ADMIN,Shanghai
-93,24/05/2012,FP-GTC,145970,0,5,33.1342,20,10000926,10009430,MODELER,Hongkong
-94,27/06/2012,Auction,174106,3,13,55.3039,16,10000430,10006553,ANALYST,Beijing
-95,02/01/2012,ABIN,87118,0,14,48.2433,20,10000731,10006539,ADMIN,Shanghai
-96,14/12/2013,FP-GTC,50508,0,5,51.5822,18,10000977,10000022,ANALYST,Beijing
-97,30/12/2012,Auction,63889,0,12,3.8393,17,10000957,10008304,MODELER,Hongkong
-98,26/10/2013,FP-non GTC,24541,0,5,23.6812,19,10000350,10005174,ANALYST,Beijing
-99,29/12/2013,ABIN,38238,0,15,99.4389,4,10000605,10002057,ANALYST,Beijing
-100,26/05/2012,FP-GTC,100847,0,15,11.0672,14,10000291,10002184,ADMIN,Shanghai
-101,18/12/2012,FP-non GTC,164261,0,11,54.9352,1,10000509,10000385,ANALYST,Beijing
-102,12/03/2013,ABIN,95672,0,15,85.7152,20,10000742,10000188,ADMIN,Shanghai
-103,22/02/2013,ABIN,43398,0,11,65.197,2,10000294,10007930,ADMIN,Shanghai
-104,20/02/2012,Others,175750,0,14,63.2187,4,10000364,10007008,MODELER,Hongkong
-105,06/08/2013,Others,32876,0,13,50.8111,2,10000542,10004538,MODELER,Hongkong
-106,27/06/2013,Others,13987,0,13,2.3165,6,10000979,10003960,ADMIN,Shanghai
-107,17/07/2012,Others,13987,0,16,25.2136,10,10000744,10006962,ADMIN,Shanghai
-108,12/10/2013,FP-non GTC,16509,0,5,15.3986,15,10000652,10000372,MODELER,Hongkong
-109,07/12/2012,Others,158798,0,5,1.6111,7,10000149,10007138,ANALYST,Beijing
-110,09/02/2013,Auction,32876,0,5,37.3237,14,10000467,10007844,ADMIN,Shanghai
-111,26/10/2012,Others,44079,0,12,18.187,2,10000420,10003664,ADMIN,Shanghai
-112,10/04/2012,ABIN,132939,0,13,88.595,5,10000103,10004100,MODELER,Hongkong
-113,08/04/2013,FP-non GTC,159184,0,5,92.4421,12,10000933,10005974,ADMIN,Shanghai
-114,30/05/2012,Others,80287,0,14,68.1663,17,10000195,10001912,ADMIN,Shanghai
-115,23/02/2012,FP-GTC,88750,0,11,19.4075,7,10000038,10000200,ADMIN,Shanghai
-116,25/10/2013,ABIN,11554,0,11,27.2512,10,10000228,10006680,ANALYST,Beijing
-117,02/12/2012,FP-GTC,41940,0,14,49.5504,7,10000927,10003981,ANALYST,Beijing
-118,16/09/2013,FP-GTC,314,0,-99,3.4785,13,10000304,10002124,MODELER,Hongkong
-119,01/11/2013,Others,31519,3,15,46.1783,5,10000771,10008920,MODELER,Hongkong
-120,21/10/2012,FP-non GTC,31519,0,15,1.7696,8,10000507,10001384,MODELER,Hongkong
-121,25/08/2013,Auction,95173,0,13,89.2083,5,10000868,10006103,MODELER,Hongkong
-122,09/11/2013,FP-non GTC,150047,3,12,58.2634,11,10000497,10004364,ANALYST,Beijing
-123,04/02/2012,FP-GTC,175750,0,11,41.385,6,10000947,10000922,ANALYST,Beijing
-124,17/10/2012,ABIN,26249,0,14,10.5789,9,10000965,10001034,ADMIN,Shanghai
-125,16/03/2012,FP-non GTC,963,0,5,17.8949,17,10000908,10009714,ANALYST,Beijing
-126,06/08/2012,Others,94847,0,13,96.0332,10,10000595,10002452,ANALYST,Beijing
-127,23/12/2013,FP-non GTC,24760,0,16,90.7096,18,10000700,10006693,ADMIN,Shanghai
-128,29/03/2013,FP-GTC,9426,3,13,50.9023,10,10000090,10004582,ADMIN,Shanghai
-129,25/03/2013,Others,20886,0,-99,35.0414,9,10000171,10002472,ANALYST,Beijing
-130,10/12/2013,Auction,95672,0,15,29.8872,16,10000216,10009995,MODELER,Hongkong
-131,14/03/2012,Others,12688,0,14,10.4067,17,10000626,10003664,MODELER,Hongkong
-132,29/11/2013,FP-non GTC,31519,0,13,89.2334,10,10000199,10008582,ANALYST,Beijing
-133,29/04/2013,ABIN,31519,0,-99,99.246,11,10000565,10005784,ADMIN,Shanghai
-134,01/02/2012,FP-GTC,161567,15,11,56.3989,9,10000651,10007644,ANALYST,Beijing
-135,17/03/2012,FP-GTC,4943,0,16,0.4422,1,10000829,10004772,MODELER,Hongkong
-136,06/11/2012,FP-GTC,156614,0,12,2.3454,1,10000370,10009337,ANALYST,Beijing
-137,03/12/2012,Auction,155226,0,13,51.1354,5,10000867,10008319,MODELER,Hongkong
-138,09/09/2012,FP-GTC,99985,0,12,9.1236,14,10000281,10001454,ANALYST,Beijing
-139,16/06/2012,ABIN,99985,0,16,16.407,14,10000143,10000141,MODELER,Hongkong
-140,06/05/2012,FP-GTC,99985,0,14,6.4304,13,10000514,10009863,MODELER,Hongkong
-141,20/10/2013,FP-GTC,36250,0,13,55.6895,8,10000390,10006419,ADMIN,Shanghai
-142,11/10/2012,ABIN,106246,0,15,30.6373,4,10000980,10003318,ADMIN,Shanghai
-143,08/08/2013,Others,103178,15,12,41.4904,13,10000751,10007021,ADMIN,Shanghai
-144,05/09/2012,Auction,314,0,16,3.2615,8,10000280,10008590,MODELER,Hongkong
-145,07/08/2012,ABIN,48904,0,13,57.7688,14,10000245,10005116,ANALYST,Beijing
-146,05/07/2013,ABIN,314,0,14,28.9444,15,10000888,10001907,MODELER,Hongkong
-147,10/09/2012,FP-non GTC,95672,0,-99,54.0914,10,10000601,10008909,ANALYST,Beijing
-148,18/09/2012,Auction,25147,0,12,74.4422,7,10000679,10000970,ANALYST,Beijing
-149,24/08/2012,FP-non GTC,175750,3,16,45.8282,6,10000397,10000959,ANALYST,Beijing
-150,29/05/2013,Others,26249,0,14,12.7681,16,10000967,10006295,ANALYST,Beijing
-151,17/11/2012,FP-GTC,94847,0,14,74.4181,2,10000175,10009505,ANALYST,Beijing
-152,22/09/2012,ABIN,175750,0,15,85.6879,14,10000453,10008084,ADMIN,Shanghai
-153,17/09/2013,ABIN,533,0,11,85.5838,12,10000642,10003672,ADMIN,Shanghai
-154,28/03/2013,Others,60340,0,12,72.6495,6,10000397,10003829,ADMIN,Shanghai
-155,09/08/2013,Others,216,0,15,88.6918,20,10000867,10000486,ADMIN,Shanghai
-156,19/12/2013,Others,43398,0,-99,88.0444,5,10000482,10004753,MODELER,Hongkong
-157,02/12/2013,FP-non GTC,15115,0,-99,29.6027,8,10000538,10007924,MODELER,Hongkong
-158,23/11/2012,Others,46575,0,5,49.3124,6,10000678,10003077,ANALYST,Beijing
-159,25/06/2013,FP-non GTC,16145,0,15,7.4413,3,10000897,10004554,MODELER,Hongkong
-160,22/01/2012,FP-GTC,164,0,14,71.8071,1,10000510,10000523,ANALYST,Beijing
-161,24/04/2013,FP-GTC,1504,0,12,95.5706,6,10000554,10002121,MODELER,Hongkong
-162,05/03/2012,FP-non GTC,12688,0,5,47.7646,12,10000175,10005221,ADMIN,Shanghai
-163,15/02/2013,FP-non GTC,16145,0,13,40.9511,6,10000647,10002696,MODELER,Hongkong
-164,18/01/2013,Others,36250,0,13,77.3572,10,10000687,10002478,ADMIN,Shanghai
-165,04/06/2012,Others,15115,0,15,25.34,7,10000493,10000370,MODELER,Hongkong
-166,13/10/2013,FP-GTC,165888,0,-99,58.1861,8,10000753,10003626,ANALYST,Beijing
-167,21/05/2012,Auction,145970,0,5,37.2504,11,10000292,10002719,ADMIN,Shanghai
-168,26/04/2012,Auction,11848,0,13,29.2954,2,10000804,10007432,MODELER,Hongkong
-169,26/09/2013,FP-GTC,61323,0,13,84.7095,9,10000761,10005178,ANALYST,Beijing
-170,06/03/2013,Auction,100847,0,-99,81.798,1,10000593,10006532,ANALYST,Beijing
-171,08/08/2012,ABIN,51582,0,15,15.979,1,10000138,10008335,ADMIN,Shanghai
-172,09/01/2012,Others,1504,0,14,34.0508,5,10000954,10003235,ANALYST,Beijing
-173,20/10/2013,ABIN,20886,0,12,36.9353,6,10000408,10007540,ADMIN,Shanghai
-174,25/06/2013,Auction,53064,0,11,99.7714,10,10000409,10003295,MODELER,Hongkong
-175,03/05/2013,FP-GTC,63861,0,5,39.2974,4,10000668,10003615,MODELER,Hongkong
-176,26/01/2013,FP-GTC,100847,0,13,59.5009,16,10000536,10005423,ADMIN,Shanghai
-177,19/09/2012,FP-GTC,164261,0,13,15.7583,5,10000231,10007013,ANALYST,Beijing
-178,16/11/2013,FP-GTC,164261,0,16,83.6234,20,10000927,10006597,ADMIN,Shanghai
-179,07/05/2013,FP-non GTC,20485,0,5,52.91,19,10000491,10005645,MODELER,Hongkong
-180,15/08/2013,Others,32996,15,12,54.7122,18,10000411,10008504,MODELER,Hongkong
-181,31/10/2013,FP-GTC,63861,0,14,0.3599,2,10000903,10005471,MODELER,Hongkong
-182,13/06/2012,FP-non GTC,159184,0,11,49.955,2,10000849,10000545,ANALYST,Beijing
-183,25/12/2012,Auction,20213,0,-99,11.4752,15,10000583,10004067,ANALYST,Beijing
-184,15/12/2013,Auction,759,0,13,33.6991,13,10000390,10004439,ADMIN,Shanghai
-185,30/04/2012,Others,87118,0,16,51.463,10,10000572,10009658,MODELER,Hongkong
-186,14/07/2012,Others,10866,0,11,36.9614,13,10000467,10001311,ANALYST,Beijing
-187,25/06/2013,FP-non GTC,1357,0,12,17.5618,12,10000426,10000482,ANALYST,Beijing
-188,06/02/2013,Auction,13836,0,12,74.3179,17,10000346,10005540,ANALYST,Beijing
-189,03/06/2012,ABIN,60340,0,12,82.3722,5,10000531,10005121,ANALYST,Beijing
-190,23/07/2013,FP-GTC,139973,0,-99,75.2548,20,10000385,10004656,ADMIN,Shanghai
-191,06/07/2013,Others,11554,0,14,40.1129,7,10000724,10000830,ADMIN,Shanghai
-192,10/08/2013,FP-non GTC,31519,0,-99,5.2432,4,10000356,10000921,ADMIN,Shanghai
-193,31/07/2013,Auction,31673,0,14,51.1089,11,10000598,10005431,ADMIN,Shanghai
-194,05/12/2013,Auction,314,0,13,19.9031,17,10000068,10007321,ADMIN,Shanghai
-195,30/12/2012,ABIN,50508,0,5,73.8659,5,10000482,10005414,MODELER,Hongkong
-196,22/09/2012,Auction,165888,0,12,81.9826,18,10000832,10000194,MODELER,Hongkong
-197,16/07/2012,Auction,223,0,5,76.0284,14,10000354,10002836,MODELER,Hongkong
-198,26/01/2013,Others,20485,0,12,71.6623,5,10000552,10000202,ANALYST,Beijing
-199,07/11/2012,FP-non GTC,57013,0,16,73.2931,5,10000801,10000942,ANALYST,Beijing
-200,26/01/2012,FP-non GTC,175750,0,13,18.897,10,10000649,10001073,ADMIN,Shanghai
-201,09/06/2012,FP-non GTC,60340,0,12,11.1614,13,10000830,10007167,ADMIN,Shanghai
-202,20/06/2012,Auction,57990,3,5,69.1183,18,10000557,10004252,MODELER,Hongkong
-203,01/01/2013,FP-non GTC,45333,0,13,94.0586,8,10000253,10003835,ANALYST,Beijing
-204,26/07/2013,FP-non GTC,139973,0,11,39.7239,14,10000441,10000478,ADMIN,Shanghai
-205,15/09/2012,FP-GTC,24541,0,15,0.3239,20,10000379,10009850,ANALYST,Beijing
-206,22/09/2013,Auction,31519,3,12,80.0315,16,10000436,10001924,ANALYST,Beijing
-207,27/10/2012,FP-GTC,223,0,15,39.379,8,10000420,10002426,ANALYST,Beijing
-208,20/11/2012,FP-non GTC,533,0,5,49.4951,3,10000390,10006171,ADMIN,Shanghai
-209,07/12/2013,FP-GTC,1349,0,15,93.3008,19,10000398,10008965,MODELER,Hongkong
-210,30/05/2012,FP-non GTC,46575,0,12,66.7652,6,10000915,10006004,ADMIN,Shanghai
-211,07/09/2012,FP-non GTC,95672,0,14,29.597,9,10000231,10004893,MODELER,Hongkong
-212,26/04/2013,Auction,31673,0,-99,15.8347,4,10000260,10002693,ANALYST,Beijing
-213,11/09/2012,FP-non GTC,50508,0,14,80.5444,1,10000263,10002496,ADMIN,Shanghai
-214,03/09/2013,FP-non GTC,95173,0,16,33.1487,19,10000493,10002289,ADMIN,Shanghai
-215,03/10/2012,FP-non GTC,44079,0,15,99.7408,13,10000828,10009523,ADMIN,Shanghai
-216,26/12/2012,ABIN,16145,0,14,40.7826,11,10000062,10005334,ANALYST,Beijing
-217,17/07/2012,Auction,26249,0,15,28.6663,9,10000534,10002400,ADMIN,Shanghai
-218,30/06/2013,ABIN,63861,0,14,75.6832,13,10000944,10007279,ADMIN,Shanghai
-219,09/01/2012,ABIN,156614,0,11,98.4996,3,10000353,10005539,ADMIN,Shanghai
-220,08/09/2013,FP-GTC,156614,0,13,12.7473,12,10000063,10002408,ADMIN,Shanghai
-221,23/09/2013,Auction,41940,0,5,37.5254,2,10000383,10001341,ADMIN,Shanghai
-222,11/11/2012,Auction,145970,0,5,77.1361,16,10000420,10000232,MODELER,Hongkong
-223,26/08/2013,Others,11848,0,16,28.2799,12,10000323,10007637,MODELER,Hongkong
-224,15/09/2012,Others,11554,0,13,63.7232,2,10000729,10008500,ADMIN,Shanghai
-225,09/01/2012,FP-non GTC,25147,0,11,7.3009,7,10000745,10006070,ADMIN,Shanghai
-226,16/08/2012,Others,87118,0,15,47.8997,1,10000240,10002299,MODELER,Hongkong
-227,26/12/2013,FP-non GTC,16145,0,12,51.0442,8,10000050,10000493,MODELER,Hongkong
-228,24/03/2012,ABIN,48027,0,5,26.2083,5,10000600,10007513,ANALYST,Beijing
-229,22/08/2012,ABIN,216,0,5,79.9268,7,10000706,10002465,ANALYST,Beijing
-230,03/11/2013,FP-GTC,24541,0,12,83.0235,18,10000293,10007217,ADMIN,Shanghai
-231,18/05/2013,FP-non GTC,88750,0,-99,84.8856,7,10000147,10007044,ADMIN,Shanghai
-232,15/01/2013,ABIN,95672,0,15,86.0386,5,10000590,10009080,MODELER,Hongkong
-233,03/12/2012,ABIN,50508,0,11,67.9981,17,10000582,10007971,MODELER,Hongkong
-234,19/06/2012,FP-GTC,148324,15,15,53.4811,10,10000793,10009995,ANALYST,Beijing
-235,06/10/2013,Auction,6762,0,14,73.5338,17,10000544,10009125,MODELER,Hongkong
-236,18/12/2013,ABIN,63861,0,5,50.2892,17,10000769,10005451,ADMIN,Shanghai
-237,04/12/2013,Auction,25147,0,5,48.3303,11,10000590,10001560,MODELER,Hongkong
-238,25/04/2012,FP-non GTC,62179,0,14,72.3156,17,10000102,10008456,ADMIN,Shanghai
-239,20/11/2013,Auction,24760,0,14,91.4883,5,10000370,10003137,ADMIN,Shanghai
-240,08/02/2013,Others,20485,0,11,29.1083,8,10000261,10005174,MODELER,Hongkong
-241,18/04/2013,Others,2023,0,5,48.4321,5,10000355,10009360,ANALYST,Beijing
-242,21/11/2012,Others,32876,0,12,16.7365,11,10000352,10008131,ANALYST,Beijing
-243,07/03/2012,FP-GTC,61323,0,-99,32.5836,5,10000494,10001632,ADMIN,Shanghai
-244,20/01/2012,FP-non GTC,175750,0,16,52.8153,9,10000025,10007893,ADMIN,Shanghai
-245,06/10/2013,FP-GTC,26249,0,-99,1.9539,2,10000660,10008166,MODELER,Hongkong
-246,10/01/2012,Auction,57990,0,14,57.58,5,10000830,10004472,ANALYST,Beijing
-247,13/09/2013,Others,46575,0,15,60.3315,12,10000528,10008869,MODELER,Hongkong
-248,19/11/2012,FP-non GTC,44079,0,11,73.0457,20,10000482,10006453,ANALYST,Beijing
-249,23/09/2012,Auction,95672,0,13,84.6345,11,10000586,10000622,MODELER,Hongkong
-250,31/08/2013,Others,216,0,12,34.7429,12,10000676,10003143,ANALYST,Beijing
-251,11/04/2013,FP-GTC,963,0,-99,77.7226,8,10000367,10003468,ADMIN,Shanghai
-252,28/03/2013,ABIN,15687,0,-99,8.2384,5,10000542,10002480,ANALYST,Beijing
-253,24/05/2012,ABIN,152801,0,15,39.0204,17,10000277,10008011,ANALYST,Beijing
-254,05/03/2013,ABIN,67698,0,16,40.6091,15,10000886,10008753,ADMIN,Shanghai
-255,06/08/2013,Auction,13987,0,13,36.5088,12,10000628,10000546,MODELER,Hongkong
-256,04/02/2013,ABIN,20213,0,11,73.5301,16,10000134,10003648,MODELER,Hongkong
-257,19/05/2012,FP-non GTC,38238,0,12,59.3237,7,10000500,10008314,ADMIN,Shanghai
-258,19/08/2013,ABIN,57990,0,12,10.1688,8,10000715,10007648,ANALYST,Beijing
-259,28/03/2012,FP-non GTC,46575,0,16,77.7396,11,10000756,10003617,ADMIN,Shanghai
-260,19/02/2012,Auction,43479,0,-99,45.245,12,10000049,10001668,ANALYST,Beijing
-261,02/04/2012,FP-GTC,41940,0,15,58.8415,3,10000195,10000014,ADMIN,Shanghai
-262,24/08/2013,Others,16145,0,11,5.4971,17,10000119,10005635,MODELER,Hongkong
-263,10/01/2013,Auction,106340,15,12,24.1336,9,10000603,10003847,MODELER,Hongkong
-264,23/09/2012,Auction,152801,0,5,70.4024,5,10000384,10002592,ANALYST,Beijing
-265,29/07/2013,FP-GTC,73506,0,16,14.5932,5,10000580,10005977,ADMIN,Shanghai
-266,16/02/2012,FP-non GTC,67698,0,5,4.4981,18,10000524,10003718,MODELER,Hongkong
-267,06/01/2013,Others,65,0,13,64.8343,10,10000731,10002914,MODELER,Hongkong
-268,02/03/2013,Auction,43398,0,16,22.6621,3,10000734,10005774,ADMIN,Shanghai
-269,25/10/2013,Auction,57990,0,13,64.3384,8,10000961,10007907,ADMIN,Shanghai
-270,07/07/2012,FP-GTC,156356,0,16,47.3744,13,10000795,10004560,ADMIN,Shanghai
-271,19/12/2013,FP-GTC,67698,0,16,96.9535,1,10000207,10005626,MODELER,Hongkong
-272,04/11/2013,Auction,63861,0,5,34.8173,19,10000964,10001539,MODELER,Hongkong
-273,18/01/2013,FP-non GTC,174053,3,5,94.6036,11,10000393,10003930,ANALYST,Beijing
-274,30/01/2013,Auction,43479,0,16,28.9982,9,10000854,10004230,ANALYST,Beijing
-275,16/05/2012,ABIN,24541,0,15,91.6347,16,10000536,10004139,MODELER,Hongkong
-276,03/06/2013,FP-non GTC,159184,0,15,27.9722,3,10000644,10005293,ADMIN,Shanghai
-277,24/08/2013,FP-GTC,2635,0,14,95.7894,16,10000503,10009687,ADMIN,Shanghai
-278,02/07/2012,FP-non GTC,16509,0,5,76.9646,11,10000320,10009405,ADMIN,Shanghai
-279,09/09/2013,ABIN,80287,0,16,68.962,13,10000934,10006877,MODELER,Hongkong
-280,15/04/2012,FP-GTC,156356,0,12,95.364,3,10000269,10000216,ANALYST,Beijing
-281,10/07/2013,FP-GTC,1349,0,13,82.7969,12,10000033,10000809,MODELER,Hongkong
-282,02/12/2012,FP-GTC,963,0,13,31.4196,16,10000717,10002029,ADMIN,Shanghai
-283,17/02/2013,Auction,26249,0,14,9.5202,6,10000543,10009090,ANALYST,Beijing
-284,17/02/2013,Others,87118,0,16,77.3899,20,10000647,10009664,ADMIN,Shanghai
-285,12/12/2012,Others,6762,0,15,76.3422,17,10000076,10001968,ADMIN,Shanghai
-286,08/07/2013,Others,80053,0,5,95.2412,4,10000723,10007824,MODELER,Hongkong
-287,19/08/2012,ABIN,32876,0,12,40.9587,12,10000352,10000219,ANALYST,Beijing
-288,24/07/2013,Auction,314,0,11,78.8147,11,10000996,10004912,ANALYST,Beijing
-289,10/10/2013,Others,57990,0,12,89.8124,4,10000549,10001719,ANALYST,Beijing
-290,13/11/2013,Others,139973,0,16,85.5451,5,10000089,10006506,ADMIN,Shanghai
-291,29/03/2013,FP-non GTC,25147,0,16,16.229,16,10000017,10001175,MODELER,Hongkong
-292,04/06/2012,FP-non GTC,44079,0,-99,0.1588,1,10000758,10007273,MODELER,Hongkong
-293,19/07/2012,FP-non GTC,57784,0,12,84.9441,18,10000553,10005711,ANALYST,Beijing
-294,08/01/2012,FP-GTC,155226,0,15,31.9705,20,10000778,10006227,ADMIN,Shanghai
-295,13/09/2012,FP-non GTC,963,0,16,39.6392,18,10000799,10008569,ADMIN,Shanghai
-296,24/12/2012,Others,175750,0,15,77.3617,3,10000164,10000878,MODELER,Hongkong
-297,31/08/2013,Auction,57990,0,12,76.7885,10,10000580,10008645,ANALYST,Beijing
-298,06/08/2013,FP-non GTC,156356,0,13,86.0218,11,10000445,10003631,ANALYST,Beijing
-299,29/05/2013,FP-non GTC,11848,0,11,23.0269,7,10000372,10000789,MODELER,Hongkong
-300,19/05/2013,Others,53064,0,13,66.399,9,10000390,10009388,MODELER,Hongkong
-301,01/11/2012,FP-non GTC,51582,0,14,57.6157,20,10000061,10007487,ADMIN,Shanghai
-302,01/01/2013,Auction,1357,0,-99,18.2117,17,10000860,10006079,ANALYST,Beijing
-303,17/05/2013,Others,121153,0,14,79.4534,17,10000155,10004663,MODELER,Hongkong
-304,04/07/2013,FP-GTC,80053,0,5,43.3271,16,10000456,10003591,ANALYST,Beijing
-305,17/12/2013,Others,165888,0,-99,9.8402,12,10000357,10007752,MODELER,Hongkong
-306,28/11/2013,FP-GTC,26262,0,-99,1.4013,19,10000731,10008257,ADMIN,Shanghai
-307,08/02/2013,FP-GTC,65,0,11,61.9604,7,10000176,10004097,ADMIN,Shanghai
-308,17/05/2012,ABIN,9426,3,12,68.017,14,10000895,10000672,ANALYST,Beijing
-309,29/08/2012,Auction,159184,0,13,43.7406,20,10000723,10005989,MODELER,Hongkong
-310,06/10/2013,FP-GTC,61323,0,14,85.9325,18,10000692,10008834,ANALYST,Beijing
-311,08/08/2013,FP-GTC,20886,0,14,88.0137,14,10000063,10009103,ADMIN,Shanghai
-312,06/04/2013,FP-non GTC,156356,0,16,67.1201,16,10000441,10002323,ADMIN,Shanghai
-313,09/03/2012,Auction,1504,0,13,45.8878,8,10000862,10003955,ANALYST,Beijing
-314,13/12/2013,FP-GTC,139973,3,16,58.7879,5,10000947,10004467,ADMIN,Shanghai
-315,03/02/2013,ABIN,95173,0,-99,33.2714,18,10000634,10002647,ANALYST,Beijing
-316,19/02/2013,Others,41940,0,5,32.2179,8,10000257,10002668,ANALYST,Beijing
-317,09/05/2012,ABIN,65,0,11,96.0579,5,10000675,10008559,ANALYST,Beijing
-318,30/10/2012,Auction,80135,0,11,55.5335,10,10000288,10009113,ANALYST,Beijing
-319,14/07/2013,ABIN,11554,0,16,9.7012,17,10000210,10000034,MODELER,Hongkong
-320,29/07/2012,Others,11554,0,5,91.5938,2,10000736,10006505,ANALYST,Beijing
-321,03/06/2012,ABIN,80287,0,13,99.7431,2,10000403,10002322,ADMIN,Shanghai
-322,10/10/2012,Auction,36250,0,13,14.7056,11,10000561,10000446,ANALYST,Beijing
-323,20/06/2013,ABIN,20485,0,5,23.912,6,10000798,10002306,ANALYST,Beijing
-324,26/12/2013,Others,11554,0,15,27.2028,6,10000154,10001499,MODELER,Hongkong
-325,07/05/2013,Auction,11848,0,-99,73.7354,4,10000448,10000791,MODELER,Hongkong
-326,14/12/2013,ABIN,61323,0,16,44.6086,3,10000555,10003798,ADMIN,Shanghai
-327,29/08/2012,FP-GTC,139973,0,14,34.7404,17,10000686,10000704,ANALYST,Beijing
-328,31/12/2013,FP-GTC,100847,0,15,70.6027,4,10000534,10007718,MODELER,Hongkong
-329,13/12/2013,FP-non GTC,16145,3,12,76.5802,5,10000864,10002571,ADMIN,Shanghai
-330,24/01/2012,FP-non GTC,57990,0,-99,54.7904,7,10000325,10001260,ADMIN,Shanghai
-331,26/07/2013,Others,20485,0,11,9.313,10,10000991,10004211,MODELER,Hongkong
-332,30/09/2013,ABIN,31519,0,15,95.7797,1,10000701,10007592,ADMIN,Shanghai
-333,30/03/2012,FP-non GTC,88750,0,16,34.6777,18,10000494,10000223,ANALYST,Beijing
-334,11/05/2012,Others,20485,0,16,2.2782,6,10000436,10003082,MODELER,Hongkong
-335,21/11/2012,FP-non GTC,156356,0,13,3.4276,20,10000574,10007758,MODELER,Hongkong
-336,19/11/2012,FP-non GTC,36250,0,16,42.1665,16,10000885,10005830,ADMIN,Shanghai
-337,07/01/2013,Others,20485,0,13,29.1424,12,10000931,10007644,ANALYST,Beijing
-338,16/01/2013,ABIN,88750,0,13,87.3858,3,10000276,10004791,ADMIN,Shanghai
-339,20/07/2012,ABIN,25147,0,-99,41.9809,10,10000928,10001382,ADMIN,Shanghai
-340,27/06/2013,FP-non GTC,75665,0,11,65.3889,17,10000587,10001522,ADMIN,Shanghai
-341,04/12/2013,FP-non GTC,45333,0,15,41.7577,5,10000979,10006485,ADMIN,Shanghai
-342,01/05/2012,Auction,4943,0,11,76.3978,17,10000385,10006699,ANALYST,Beijing
-343,03/05/2013,Others,66767,15,-99,69.062,17,10000186,10009550,ANALYST,Beijing
-344,14/02/2013,FP-non GTC,95672,0,14,15.6152,16,10000356,10006265,ANALYST,Beijing
-345,01/01/2012,Others,1357,0,13,16.7271,20,10000263,10005410,ANALYST,Beijing
-346,05/01/2012,FP-non GTC,67703,3,5,32.2005,6,10000141,10008271,ADMIN,Shanghai
-347,15/12/2013,Others,2023,0,-99,55.3349,5,10000920,10002066,ADMIN,Shanghai
-348,02/10/2012,FP-GTC,314,0,14,95.9217,10,10000647,10009465,ANALYST,Beijing
-349,27/10/2012,Others,64076,0,5,3.4266,15,10000486,10005406,ADMIN,Shanghai
-350,17/10/2013,Others,41940,0,15,3.0363,9,10000299,10001071,ADMIN,Shanghai
-351,09/02/2013,Others,13836,0,15,37.1746,17,10000799,10005197,MODELER,Hongkong
-352,03/04/2013,FP-GTC,31519,0,15,83.3031,17,10000656,10000175,ADMIN,Shanghai
-353,30/08/2013,Others,63861,0,13,37.862,16,10000958,10007038,ADMIN,Shanghai
-354,01/01/2012,Others,1349,0,13,0.7227,14,10000774,10008113,ADMIN,Shanghai
-355,24/03/2013,FP-non GTC,87118,0,13,32.5615,12,10000661,10006407,ADMIN,Shanghai
-356,03/12/2012,FP-non GTC,216,0,5,40.1447,5,10000923,10008590,ANALYST,Beijing
-357,12/07/2012,FP-GTC,94847,0,5,74.4586,18,10000161,10009803,MODELER,Hongkong
-358,18/03/2013,Others,1161,3,-99,70.1521,3,10000123,10000399,ANALYST,Beijing
-359,17/11/2013,Auction,165888,0,5,16.1344,10,10000029,10008638,ANALYST,Beijing
-360,25/11/2013,FP-non GTC,44079,0,5,18.0996,11,10000461,10001260,MODELER,Hongkong
-361,06/12/2012,FP-GTC,159184,0,15,63.6868,3,10000399,10005791,ANALYST,Beijing
-362,27/09/2013,FP-non GTC,161567,15,15,35.8966,12,10000283,10005103,MODELER,Hongkong
-363,02/11/2013,ABIN,20865,0,5,70.2863,18,10000724,10008591,ADMIN,Shanghai
-364,29/04/2013,Auction,314,0,13,53.6862,15,10000676,10007462,MODELER,Hongkong
-365,21/01/2013,Auction,20886,0,-99,32.5548,18,10000052,10007849,MODELER,Hongkong
-366,18/04/2012,FP-GTC,67698,0,14,90.8206,8,10000221,10007877,ANALYST,Beijing
-367,14/10/2013,Others,46575,0,13,92.8392,3,10000187,10005505,ANALYST,Beijing
-368,06/07/2013,FP-GTC,26249,0,-99,54.6729,7,10000651,10008340,MODELER,Hongkong
-369,15/09/2012,Others,16145,0,15,70.0461,8,10000279,10007484,ADMIN,Shanghai
-370,02/11/2013,Others,38238,0,11,22.9595,19,10000451,10008446,MODELER,Hongkong
-371,03/06/2012,ABIN,57013,0,11,28.9005,17,10000028,10007103,ADMIN,Shanghai
-372,08/10/2013,FP-GTC,20886,0,14,24.6958,5,10000676,10008871,ANALYST,Beijing
-373,11/04/2013,Auction,100847,0,14,42.1878,11,10000645,10008666,MODELER,Hongkong
-374,07/11/2013,Auction,45333,0,13,58.1089,2,10000182,10002861,ANALYST,Beijing
-375,18/10/2012,FP-non GTC,57990,0,15,37.9049,10,10000696,10006240,ADMIN,Shanghai
-376,28/11/2012,FP-non GTC,63889,0,5,59.1268,16,10000116,10008447,ADMIN,Shanghai
-377,02/03/2012,Auction,45333,0,13,37.4799,9,10000298,10005848,ANALYST,Beijing
-378,27/07/2013,FP-non GTC,75665,0,12,19.423,5,10000333,10000499,ADMIN,Shanghai
-379,01/12/2012,Auction,44079,0,11,95.4596,19,10000163,10003317,ANALYST,Beijing
-380,11/04/2012,FP-non GTC,161567,15,13,47.4582,13,10000082,10003929,ADMIN,Shanghai
-381,23/06/2013,ABIN,175750,3,13,23.9511,17,10000560,10008767,ANALYST,Beijing
-382,17/02/2012,Others,46575,0,15,25.7613,2,10000308,10002423,ADMIN,Shanghai
-383,19/05/2013,ABIN,223,0,15,64.7127,12,10000850,10000542,ANALYST,Beijing
-384,29/10/2012,FP-non GTC,159184,0,5,15.4188,16,10000995,10003958,ADMIN,Shanghai
-385,17/07/2012,Auction,4943,0,11,32.1734,13,10000820,10000943,ANALYST,Beijing
-386,03/01/2013,ABIN,57784,0,5,47.7542,20,10000563,10007077,MODELER,Hongkong
-387,17/07/2013,FP-GTC,150047,3,16,35.0905,14,10000545,10004393,ADMIN,Shanghai
-388,20/07/2013,Others,87118,0,14,24.6574,2,10000985,10008103,MODELER,Hongkong
-389,14/12/2013,Others,11848,0,14,80.9504,14,10000318,10002671,MODELER,Hongkong
-390,16/01/2013,Auction,106246,0,12,94.2201,1,10000113,10007603,ADMIN,Shanghai
-391,04/11/2012,Others,63861,0,12,71.8354,10,10000985,10002264,MODELER,Hongkong
-392,25/08/2012,FP-non GTC,100847,0,5,68.7418,6,10000581,10002691,ADMIN,Shanghai
-393,22/07/2013,Auction,156356,0,15,63.6314,8,10000140,10002949,ANALYST,Beijing
-394,04/11/2013,Auction,67698,0,16,15.6061,6,10000128,10003790,MODELER,Hongkong
-395,01/03/2013,FP-GTC,20485,0,-99,49.2253,4,10000722,10000653,MODELER,Hongkong
-396,03/12/2013,FP-GTC,46575,0,-99,71.5261,2,10000796,10003451,ADMIN,Shanghai
-397,25/03/2013,FP-non GTC,60340,0,11,23.0796,14,10000605,10009790,ADMIN,Shanghai
-398,16/11/2012,FP-non GTC,99985,0,11,91.6223,15,10000619,10001315,ANALYST,Beijing
-399,24/05/2012,Auction,16509,0,16,97.7044,14,10000002,10004133,ADMIN,Shanghai
-400,04/04/2012,ABIN,139973,3,13,16.7393,17,10000923,10001413,MODELER,Hongkong
-401,20/07/2012,FP-GTC,174106,3,13,8.8874,15,10000134,10007563,MODELER,Hongkong
-402,12/09/2013,Auction,33038,15,13,46.7475,3,10000154,10001737,MODELER,Hongkong
-403,12/06/2012,ABIN,57990,0,11,33.1479,11,10000036,10005775,ADMIN,Shanghai
-404,31/08/2013,ABIN,46575,0,12,41.0635,17,10000310,10008468,ADMIN,Shanghai
-405,18/12/2012,Auction,1349,0,5,14.6328,12,10000296,10002985,MODELER,Hongkong
-406,18/02/2012,ABIN,16509,0,-99,76.3163,18,10000486,10007612,MODELER,Hongkong
-407,02/07/2013,Auction,3838,0,11,95.7402,5,10000963,10008136,ANALYST,Beijing
-408,28/06/2012,Auction,145970,0,16,41.1425,16,10000646,10001273,ADMIN,Shanghai
-409,03/11/2013,FP-GTC,12688,0,5,73.9106,7,10000357,10009931,MODELER,Hongkong
-410,30/03/2012,FP-non GTC,16509,0,5,39.5703,2,10000495,10005493,MODELER,Hongkong
-411,18/08/2012,FP-non GTC,75665,0,11,61.4928,1,10000522,10007332,MODELER,Hongkong
-412,29/07/2013,Others,156356,0,16,98.2405,14,10000417,10002579,ANALYST,Beijing
-413,19/07/2013,Others,32876,0,12,68.5148,6,10000306,10000694,ANALYST,Beijing
-414,13/02/2012,ABIN,175750,0,16,64.5444,5,10000274,10001204,ANALYST,Beijing
-415,04/02/2013,FP-non GTC,13836,0,-99,65.7663,5,10000511,10006170,ADMIN,Shanghai
-416,24/07/2013,FP-GTC,165888,0,16,91.7808,12,10000330,10000833,ADMIN,Shanghai
-417,25/04/2012,Others,41940,0,-99,6.5053,10,10000357,10003161,MODELER,Hongkong
-418,28/11/2013,ABIN,106340,15,12,71.4264,11,10000155,10004256,ANALYST,Beijing
-419,16/09/2012,FP-non GTC,170083,3,-99,80.2442,11,10000690,10007308,ANALYST,Beijing
-420,10/05/2012,FP-GTC,16145,0,14,11.0775,17,10000372,10008952,ANALYST,Beijing
-421,28/10/2013,FP-non GTC,15687,0,16,65.9436,16,10000032,10004950,ANALYST,Beijing
-422,17/05/2013,FP-non GTC,94847,0,13,37.352,7,10000044,10004245,ADMIN,Shanghai
-423,31/05/2013,FP-GTC,223,0,5,93.3181,12,10000365,10006098,MODELER,Hongkong
-424,07/01/2012,FP-non GTC,73506,0,13,13.1225,1,10000218,10000017,ANALYST,Beijing
-425,01/07/2013,FP-GTC,64076,0,15,78.1094,3,10000081,10009186,ADMIN,Shanghai
-426,26/07/2013,Others,67698,0,11,14.9245,7,10000775,10002428,ANALYST,Beijing
-427,02/03/2013,Auction,75665,0,5,84.2445,2,10000162,10007586,ANALYST,Beijing
-428,13/01/2012,FP-GTC,1120,3,16,43.8855,12,10000157,10009306,MODELER,Hongkong
-429,02/06/2012,ABIN,53064,0,12,17.36,9,10000091,10003461,ADMIN,Shanghai
-430,23/02/2012,ABIN,31387,3,14,43.1757,9,10000071,10009080,ADMIN,Shanghai
-431,22/10/2013,FP-non GTC,158798,0,15,10.9759,6,10000341,10006910,MODELER,Hongkong
-432,23/05/2013,FP-GTC,11848,0,-99,7.523,2,10000277,10004380,MODELER,Hongkong
-433,25/07/2013,FP-GTC,43479,0,14,13.262,20,10000500,10003341,MODELER,Hongkong
-434,03/03/2012,FP-non GTC,11848,0,14,95.0683,19,10000164,10009852,ADMIN,Shanghai
-435,09/01/2012,ABIN,11554,0,15,6.8777,15,10000785,10008587,ANALYST,Beijing
-436,08/05/2013,ABIN,80287,0,14,95.532,4,10000595,10003020,ANALYST,Beijing
-437,26/07/2013,FP-GTC,121153,0,12,28.7213,3,10000829,10008342,MODELER,Hongkong
-438,27/11/2012,FP-GTC,31673,0,14,7.8033,15,10000498,10003515,ANALYST,Beijing
-439,04/06/2013,ABIN,43398,0,15,76.0187,16,10000061,10009033,ANALYST,Beijing
-440,05/10/2012,ABIN,15687,0,13,87.1235,16,10000649,10006843,MODELER,Hongkong
-441,29/06/2013,Auction,61323,0,14,78.4,6,10000765,10000686,ANALYST,Beijing
-442,25/07/2012,ABIN,216,0,12,43.1556,4,10000597,10007703,ANALYST,Beijing
-443,03/01/2013,FP-non GTC,121153,0,12,70.1009,3,10000201,10008803,MODELER,Hongkong
-444,22/10/2013,FP-non GTC,170302,15,12,92.5959,2,10000151,10001464,MODELER,Hongkong
-445,03/02/2012,FP-non GTC,155226,0,12,68.6486,10,10000870,10007848,MODELER,Hongkong
-446,30/03/2013,FP-GTC,139973,0,11,93.3324,16,10000585,10004571,ANALYST,Beijing
-447,21/11/2012,Others,20865,0,11,68.0525,15,10000115,10004485,ADMIN,Shanghai
-448,09/11/2012,ABIN,11848,0,13,66.3745,2,10000652,10009670,ADMIN,Shanghai
-449,22/10/2012,Auction,2023,0,13,52.4918,10,10000836,10002112,ANALYST,Beijing
-450,07/01/2013,Auction,164261,0,-99,56.468,3,10000560,10000356,ANALYST,Beijing
-451,05/01/2012,Others,80053,0,-99,34.1305,18,10000905,10008693,ADMIN,Shanghai
-452,27/08/2012,FP-GTC,64076,0,14,3.0691,11,10000389,10008952,ADMIN,Shanghai
-453,23/06/2013,Others,139973,0,16,89.7673,8,10000636,10006901,ANALYST,Beijing
-454,13/09/2013,Auction,67698,0,15,62.9192,1,10000596,10000522,MODELER,Hongkong
-455,18/08/2012,FP-GTC,43398,0,12,45.3384,4,10000382,10003920,ADMIN,Shanghai
-456,27/10/2012,FP-non GTC,1120,3,5,84.6466,7,10000487,10001720,ANALYST,Beijing
-457,24/05/2012,Auction,24541,0,14,56.7041,4,10000366,10002296,ADMIN,Shanghai
-458,12/02/2013,Auction,64076,0,13,1.136,1,10000089,10004796,ADMIN,Shanghai
-459,19/02/2012,FP-non GTC,43398,0,14,7.2416,11,10000129,10005474,ANALYST,Beijing
-460,30/07/2012,ABIN,48027,0,11,41.6704,12,10000761,10009175,MODELER,Hongkong
-461,04/10/2013,FP-non GTC,37831,0,16,31.9567,6,10000610,10007936,MODELER,Hongkong
-462,18/11/2013,Auction,223,0,16,21.8155,2,10000079,10001417,ANALYST,Beijing
-463,08/06/2012,Others,80135,0,16,90.0166,15,10000478,10009633,MODELER,Hongkong
-464,25/10/2012,Others,159184,0,16,92.0436,2,10000166,10007086,ANALYST,Beijing
-465,22/10/2013,ABIN,13987,0,15,62.5546,15,10000592,10008119,ADMIN,Shanghai
-466,01/11/2013,ABIN,103324,15,16,14.9475,11,10000924,10006715,ADMIN,Shanghai
-467,21/03/2013,FP-GTC,1120,3,-99,29.5252,3,10000758,10000262,ADMIN,Shanghai
-468,13/03/2012,Others,759,0,16,62.4158,17,10000810,10000963,MODELER,Hongkong
-469,14/05/2012,Auction,95672,0,16,92.0781,17,10000609,10000260,ANALYST,Beijing
-470,28/09/2013,FP-GTC,64076,0,12,37.3599,19,10000487,10001678,ADMIN,Shanghai
-471,09/10/2013,ABIN,60606,3,13,48.9467,17,10000007,10004697,ANALYST,Beijing
-472,21/04/2013,ABIN,26249,0,14,67.152,6,10000257,10008616,MODELER,Hongkong
-473,08/05/2013,FP-GTC,20886,0,13,19.5685,17,10000122,10004150,ANALYST,Beijing
-474,29/12/2013,ABIN,1357,0,12,89.9069,2,10000900,10007759,MODELER,Hongkong
-475,14/02/2013,FP-GTC,63889,0,16,74.5657,1,10000676,10009640,ANALYST,Beijing
-476,25/02/2012,Auction,88750,0,-99,92.3939,19,10000769,10000718,MODELER,Hongkong
-477,22/06/2012,Others,13836,0,14,77.1277,2,10000795,10002041,MODELER,Hongkong
-478,30/01/2012,FP-GTC,43479,0,15,48.2346,6,10000013,10005919,ANALYST,Beijing
-479,17/08/2013,ABIN,57990,3,-99,87.0638,20,10000277,10006947,ADMIN,Shanghai
-480,24/06/2013,FP-GTC,1357,0,15,75.0985,13,10000056,10003367,ANALYST,Beijing
-481,08/04/2013,ABIN,57013,0,11,21.4485,11,10000195,10002888,ANALYST,Beijing
-482,07/04/2012,FP-non GTC,24541,0,-99,40.6197,3,10000313,10003959,ANALYST,Beijing
-483,26/07/2012,Others,963,0,16,99.8137,18,10000257,10009703,ANALYST,Beijing
-484,18/07/2012,Others,15115,0,-99,9.9153,5,10000937,10007448,MODELER,Hongkong
-485,04/01/2012,Auction,63889,0,13,84.5004,5,10000467,10002512,MODELER,Hongkong
-486,25/05/2013,Auction,1120,3,14,40.3618,15,10000076,10005541,MODELER,Hongkong
-487,30/03/2013,FP-GTC,38238,0,12,94.987,4,10000997,10005686,ANALYST,Beijing
-488,12/05/2012,Auction,32876,0,-99,38.58,10,10000943,10008237,ANALYST,Beijing
-489,21/11/2012,FP-GTC,100847,0,14,7.7495,1,10000588,10009021,MODELER,Hongkong
-490,02/10/2013,Auction,15868,0,5,92.2309,14,10000520,10000500,ADMIN,Shanghai
-491,05/11/2013,ABIN,31673,0,-99,75.9838,20,10000344,10003167,ANALYST,Beijing
-492,23/03/2013,FP-non GTC,48904,0,-99,15.426,2,10000090,10003494,MODELER,Hongkong
-493,27/12/2012,FP-non GTC,43479,0,5,35.7689,16,10000066,10002163,ANALYST,Beijing
-494,11/01/2013,FP-non GTC,223,0,12,5.9423,15,10000174,10005802,ANALYST,Beijing
-495,25/05/2012,Auction,50508,0,16,81.5853,1,10000337,10003087,MODELER,Hongkong
-496,08/09/2012,Others,6762,0,15,11.8473,8,10000718,10005439,ANALYST,Beijing
-497,08/07/2013,Auction,139973,0,5,11.7025,2,10000537,10004155,ANALYST,Beijing
-498,23/04/2013,ABIN,32876,0,12,40.4116,5,10000798,10004170,ADMIN,Shanghai
-499,22/03/2013,Auction,2023,0,15,89.8093,16,10000662,10004308,ANALYST,Beijing
-500,01/11/2012,Others,75708,3,11,95.9527,12,10000776,10000062,MODELER,Hongkong
-501,29/01/2012,ABIN,156356,0,16,64.6181,11,10000910,10009800,ADMIN,Shanghai
-502,20/01/2012,Others,2635,0,11,81.6196,2,10000893,10001706,ANALYST,Beijing
-503,27/05/2012,FP-non GTC,94847,0,15,87.1713,12,10000121,10005066,ANALYST,Beijing
-504,17/12/2013,Others,156614,0,14,87.7791,18,10000531,10000981,MODELER,Hongkong
-505,05/03/2013,ABIN,80135,0,16,64.8966,12,10000452,10009861,MODELER,Hongkong
-506,26/01/2013,ABIN,156356,0,16,59.0108,11,10000671,10008048,ADMIN,Shanghai
-507,08/01/2012,FP-non GTC,57990,0,15,56.347,3,10000028,10002420,ADMIN,Shanghai
-508,04/08/2013,ABIN,13836,0,11,98.5679,6,10000481,10007240,ADMIN,Shanghai
-509,04/04/2013,Others,63861,0,13,14.3911,15,10000944,10009613,ANALYST,Beijing
-510,15/02/2013,FP-GTC,2023,0,5,5.5878,17,10000816,10006549,MODELER,Hongkong
-511,05/05/2013,FP-non GTC,175750,0,-99,80.9471,9,10000727,10007640,MODELER,Hongkong
-512,24/06/2013,FP-GTC,38238,0,12,96.7825,11,10000202,10004824,ANALYST,Beijing
-513,04/11/2013,ABIN,45333,0,14,57.954,4,10000767,10002230,ANALYST,Beijing
-514,23/02/2012,FP-non GTC,57990,0,5,16.479,3,10000146,10002203,ADMIN,Shanghai
-515,13/01/2013,ABIN,158798,0,11,45.5371,13,10000133,10005287,ANALYST,Beijing
-516,16/11/2013,ABIN,73506,0,16,35.7207,19,10000304,10003454,ANALYST,Beijing
-517,15/11/2012,Auction,64076,0,15,66.0059,9,10000375,10009226,ANALYST,Beijing
-518,03/01/2012,ABIN,156614,0,15,84.8759,8,10000095,10004187,ADMIN,Shanghai
-519,22/09/2012,Auction,57990,0,14,42.7313,10,10000953,10005015,MODELER,Hongkong
-520,03/11/2013,FP-non GTC,67698,0,11,8.9322,2,10000913,10008723,MODELER,Hongkong
-521,31/08/2012,ABIN,1504,0,5,34.3706,12,10000748,10008788,ANALYST,Beijing
-522,13/12/2013,FP-GTC,15868,0,-99,18.3204,8,10000495,10005383,MODELER,Hongkong
-523,20/07/2012,ABIN,533,0,11,24.9451,13,10000237,10005440,MODELER,Hongkong
-524,14/07/2012,ABIN,41940,0,11,42.9796,1,10000328,10004188,ANALYST,Beijing
-525,06/11/2013,Others,32876,0,15,4.2693,1,10000383,10001156,ADMIN,Shanghai
-526,02/07/2012,Auction,41940,0,12,19.2987,1,10000196,10002110,ADMIN,Shanghai
-527,21/06/2012,FP-GTC,20865,0,-99,36.5029,14,10000098,10000543,ANALYST,Beijing
-528,19/09/2012,ABIN,159184,0,15,65.9768,15,10000517,10004195,ANALYST,Beijing
-529,11/02/2012,Others,16145,0,14,41.4762,2,10000733,10009727,ANALYST,Beijing
-530,19/07/2012,Auction,63861,0,11,30.7294,15,10000351,10001645,ANALYST,Beijing
-531,24/05/2013,ABIN,1349,0,12,61.0346,17,10000078,10006458,MODELER,Hongkong
-532,26/06/2013,Auction,139973,3,11,49.3849,5,10000537,10000240,ADMIN,Shanghai
-533,30/11/2013,Others,1357,0,16,62.0388,20,10000532,10003270,MODELER,Hongkong
-534,01/11/2012,Others,121153,0,5,64.8814,13,10000820,10007027,ANALYST,Beijing
-535,20/10/2013,Auction,20485,0,5,88.9093,4,10000467,10000657,MODELER,Hongkong
-536,15/04/2013,FP-GTC,43479,0,13,72.5969,12,10000352,10006790,MODELER,Hongkong
-537,02/04/2012,FP-GTC,33977,15,16,42.1845,8,10000614,10008912,ANALYST,Beijing
-538,20/03/2012,ABIN,2023,0,16,95.0254,2,10000014,10009645,ADMIN,Shanghai
-539,21/02/2013,FP-non GTC,57990,0,5,25.0404,12,10000386,10006749,ANALYST,Beijing
-540,02/06/2013,FP-non GTC,6762,0,12,60.3684,4,10000783,10002330,ADMIN,Shanghai
-541,07/01/2012,FP-GTC,759,0,11,80.172,12,10000952,10007127,MODELER,Hongkong
-542,01/12/2012,FP-non GTC,533,0,12,47.1653,15,10000550,10002624,ADMIN,Shanghai
-543,03/11/2012,ABIN,99985,0,16,40.828,16,10000629,10009256,MODELER,Hongkong
-544,15/10/2013,FP-GTC,64076,0,15,55.9541,9,10000531,10004447,ANALYST,Beijing
-545,30/12/2013,Auction,80135,0,13,13.2503,13,10000425,10001981,ADMIN,Shanghai
-546,28/06/2012,FP-GTC,15868,0,12,76.836,4,10000557,10006646,MODELER,Hongkong
-547,07/04/2013,FP-GTC,10866,0,16,37.754,17,10000810,10006760,ADMIN,Shanghai
-548,03/08/2013,FP-GTC,139255,15,13,70.8704,2,10000514,10004045,MODELER,Hongkong
-549,23/07/2012,ABIN,121153,0,5,65.6362,2,10000166,10009384,MODELER,Hongkong
-550,04/05/2013,FP-GTC,31519,0,5,38.9751,5,10000932,10001903,ANALYST,Beijing
-551,27/01/2013,ABIN,175750,3,-99,81.7736,17,10000529,10009081,ADMIN,Shanghai
-552,30/03/2013,ABIN,11554,0,15,37.8703,13,10000509,10009280,MODELER,Hongkong
-553,03/06/2012,FP-GTC,20485,0,-99,62.4729,9,10000404,10002401,MODELER,Hongkong
-554,09/05/2012,FP-non GTC,63861,0,11,35.1334,2,10000175,10008536,ADMIN,Shanghai
-555,25/12/2013,FP-non GTC,6762,0,12,95.0363,20,10000101,10005222,ANALYST,Beijing
-556,18/11/2012,FP-GTC,175750,0,5,95.8938,1,10000537,10003708,ADMIN,Shanghai
-557,07/01/2013,FP-non GTC,57784,0,14,0.2995,11,10000847,10001989,ADMIN,Shanghai
-558,11/03/2012,Auction,63861,0,12,58.1028,5,10000912,10006009,ANALYST,Beijing
-559,23/03/2012,Others,80287,0,14,24.79,9,10000967,10003972,ADMIN,Shanghai
-560,19/03/2012,ABIN,99985,0,16,5.9564,16,10000684,10007469,ADMIN,Shanghai
-561,24/11/2013,Auction,20886,0,15,28.1384,10,10000239,10009910,ADMIN,Shanghai
-562,04/04/2012,FP-GTC,100847,0,11,91.4446,11,10000743,10009881,ADMIN,Shanghai
-563,15/12/2013,ABIN,44079,0,16,39.686,14,10000476,10000011,MODELER,Hongkong
-564,06/10/2013,FP-GTC,132939,0,12,20.8855,6,10000854,10005709,ADMIN,Shanghai
-565,07/12/2013,Others,25147,0,14,16.4357,1,10000198,10007137,ADMIN,Shanghai
-566,04/02/2012,Others,158798,0,13,44.5451,14,10000543,10000252,MODELER,Hongkong
-567,07/11/2013,FP-GTC,10866,0,16,8.5249,8,10000105,10008626,ADMIN,Shanghai
-568,04/08/2012,FP-GTC,2635,0,12,11.7572,16,10000382,10009967,ADMIN,Shanghai
-569,14/08/2012,Auction,152801,0,15,46.6818,5,10000205,10008668,ADMIN,Shanghai
-570,25/02/2013,Auction,50508,0,15,37.8443,13,10000644,10000524,MODELER,Hongkong
-571,09/10/2013,ABIN,223,0,13,58.8049,7,10000768,10003846,ANALYST,Beijing
-572,10/01/2012,Auction,20485,0,11,92.2997,2,10000975,10007593,ADMIN,Shanghai
-573,03/05/2013,FP-non GTC,37831,0,12,15.9399,20,10000280,10006028,MODELER,Hongkong
-574,12/11/2013,ABIN,73506,0,12,13.9272,8,10000332,10004101,ANALYST,Beijing
-575,16/08/2013,Others,164,0,12,39.2371,19,10000887,10004134,MODELER,Hongkong
-576,27/10/2013,FP-non GTC,15868,0,16,51.4769,17,10000303,10004782,MODELER,Hongkong
-577,27/04/2013,FP-non GTC,80287,0,15,90.8971,19,10000207,10000144,MODELER,Hongkong
-578,06/11/2013,FP-non GTC,11554,0,12,26.0123,15,10000855,10007610,ADMIN,Shanghai
-579,25/11/2013,Others,6762,0,14,96.0198,12,10000418,10000846,ANALYST,Beijing
-580,09/08/2012,ABIN,80135,0,15,89.0544,10,10000861,10007561,MODELER,Hongkong
-581,28/09/2013,ABIN,16509,0,12,90.2997,9,10000520,10003310,MODELER,Hongkong
-582,19/03/2012,FP-GTC,43479,0,-99,93.8319,8,10000045,10002310,MODELER,Hongkong
-583,25/05/2012,FP-GTC,139973,0,16,8.9816,5,10000684,10006239,ADMIN,Shanghai
-584,04/04/2013,Auction,25147,0,11,43.4557,10,10000801,10002822,MODELER,Hongkong
-585,07/07/2013,Auction,67698,0,14,59.3634,16,10000305,10006424,ADMIN,Shanghai
-586,27/10/2013,ABIN,63861,0,14,48.1526,11,10000963,10001757,MODELER,Hongkong
-587,31/05/2012,Auction,130,0,5,1.056,17,10000357,10003239,MODELER,Hongkong
-588,21/08/2012,Auction,13836,0,16,14.4247,1,10000278,10003258,ADMIN,Shanghai
-589,24/07/2013,Auction,15568,15,14,14.5748,12,10000590,10006155,ADMIN,Shanghai
-590,05/04/2013,ABIN,12688,0,12,2.0636,14,10000951,10001457,ADMIN,Shanghai
-591,27/02/2013,FP-non GTC,73506,0,14,2.7139,10,10000134,10008130,ADMIN,Shanghai
-592,16/06/2013,FP-non GTC,164,0,5,59.4198,19,10000745,10008243,ANALYST,Beijing
-593,01/04/2012,Others,20213,0,11,77.849,13,10000929,10006004,ANALYST,Beijing
-594,02/12/2013,FP-non GTC,155226,0,16,71.11,20,10000433,10005008,ADMIN,Shanghai
-595,07/06/2013,Auction,158798,0,15,98.6517,14,10000320,10009500,ADMIN,Shanghai
-596,25/11/2012,ABIN,31519,0,12,68.8974,9,10000726,10000921,ADMIN,Shanghai
-597,17/04/2013,FP-GTC,57013,0,14,49.4405,5,10000050,10007518,ADMIN,Shanghai
-598,18/03/2013,FP-non GTC,175750,0,13,2.3784,16,10000559,10005674,ANALYST,Beijing
-599,13/08/2013,Others,62179,0,11,36.494,16,10000652,10003841,MODELER,Hongkong
-600,15/10/2012,ABIN,175750,0,14,42.7269,7,10000560,10007156,MODELER,Hongkong
-601,08/01/2012,Others,6762,0,5,58.5212,16,10000829,10005036,ADMIN,Shanghai
-602,20/05/2013,FP-non GTC,216,0,14,16.9644,3,10000104,10003344,ADMIN,Shanghai
-603,07/12/2012,Others,51582,0,5,6.0617,10,10000228,10008500,ANALYST,Beijing
-604,04/12/2013,ABIN,57013,0,16,48.2921,10,10000347,10008359,MODELER,Hongkong
-605,17/12/2013,ABIN,759,0,14,12.8117,20,10000161,10005247,ANALYST,Beijing
-606,08/02/2012,FP-non GTC,24760,0,13,11.319,3,10000938,10005304,MODELER,Hongkong
-607,03/01/2013,FP-non GTC,15568,15,14,0.3508,1,10000964,10003128,ADMIN,Shanghai
-608,21/04/2013,Others,87118,0,12,9.5385,19,10000949,10000218,ADMIN,Shanghai
-609,13/11/2013,Others,36250,0,12,0.9029,1,10000532,10000421,ANALYST,Beijing
-610,18/03/2013,FP-GTC,164,0,12,3.7727,19,10000237,10004397,ADMIN,Shanghai
-611,29/06/2013,Others,15868,0,11,23.3421,1,10000671,10003358,MODELER,Hongkong
-612,25/12/2013,Others,25147,0,5,99.4168,20,10000708,10008710,ANALYST,Beijing
-613,06/10/2012,FP-GTC,533,0,12,40.5069,5,10000295,10007759,MODELER,Hongkong
-614,21/02/2013,ABIN,216,0,14,84.3818,11,10000637,10004755,ADMIN,Shanghai
-615,27/08/2012,FP-non GTC,67698,0,16,7.0299,13,10000671,10009708,MODELER,Hongkong
-616,13/04/2013,Others,51582,0,13,67.973,20,10000778,10003242,ANALYST,Beijing
-617,10/07/2013,FP-GTC,139973,0,13,24.8423,15,10000253,10009806,ADMIN,Shanghai
-618,02/11/2013,ABIN,1349,0,14,56.631,20,10000922,10001129,ANALYST,Beijing
-619,11/05/2013,Others,26249,0,16,49.7279,9,10000610,10003206,ANALYST,Beijing
-620,17/01/2013,FP-non GTC,118687,3,11,34.6913,1,10000002,10003226,MODELER,Hongkong
-621,02/07/2012,Others,148324,15,-99,90.6687,2,10000613,10002537,ANALYST,Beijing
-622,14/07/2013,ABIN,41940,0,-99,60.2733,3,10000145,10004935,MODELER,Hongkong
-623,13/06/2013,Auction,63889,0,11,87.9644,3,10000920,10007836,MODELER,Hongkong
-624,05/05/2013,FP-non GTC,1120,3,16,99.4791,12,10000720,10004775,ADMIN,Shanghai
-625,30/03/2012,Others,4943,0,14,56.9643,12,10000105,10001645,ANALYST,Beijing
-626,13/01/2012,FP-non GTC,139973,3,-99,15.838,5,10000587,10009527,ANALYST,Beijing
-627,02/09/2012,ABIN,12688,0,13,69.4513,9,10000756,10008016,ANALYST,Beijing
-628,08/08/2013,Others,121153,0,11,29.2318,6,10000289,10000409,ADMIN,Shanghai
-629,09/03/2013,Others,99985,0,15,14.7661,3,10000453,10003921,MODELER,Hongkong
-630,17/10/2012,FP-non GTC,25147,0,11,96.1909,3,10000054,10004624,ADMIN,Shanghai
-631,20/01/2012,Others,73506,0,14,56.0157,8,10000909,10009618,ADMIN,Shanghai
-632,15/04/2012,FP-GTC,16145,0,12,94.6354,16,10000551,10008904,ANALYST,Beijing
-633,08/08/2013,Others,3838,0,13,85.7695,15,10000464,10003300,ANALYST,Beijing
-634,21/01/2013,Others,139973,0,13,64.5021,12,10000107,10004525,MODELER,Hongkong
-635,17/06/2012,Others,139973,0,-99,72.8065,1,10000495,10009258,MODELER,Hongkong
-636,31/12/2013,Others,20886,0,15,91.2326,8,10000746,10006331,MODELER,Hongkong
-637,11/02/2012,FP-non GTC,4943,0,14,28.3146,4,10000649,10003718,ANALYST,Beijing
-638,01/10/2013,Auction,155226,0,15,72.2196,20,10000576,10003692,MODELER,Hongkong
-639,25/08/2012,ABIN,223,0,15,17.6408,16,10000661,10004113,ANALYST,Beijing
-640,10/07/2012,Others,67698,0,15,70.9451,6,10000115,10000167,ANALYST,Beijing
-641,27/12/2013,FP-non GTC,26262,0,13,64.5694,18,10000740,10005696,MODELER,Hongkong
-642,05/02/2012,Auction,32996,15,12,9.9521,1,10000833,10009650,ANALYST,Beijing
-643,07/11/2012,FP-non GTC,25147,0,16,83.3886,6,10000494,10005692,MODELER,Hongkong
-644,01/01/2014,Others,45333,0,16,19.2606,1,10000869,10002121,ADMIN,Shanghai
-645,05/10/2013,Auction,50508,0,14,39.1465,16,10000904,10001424,ADMIN,Shanghai
-646,25/02/2012,FP-GTC,37831,0,11,12.4616,1,10000680,10003627,ANALYST,Beijing
-647,16/05/2012,Others,31519,0,12,66.9268,15,10000548,10008997,ADMIN,Shanghai
-648,08/11/2013,FP-GTC,100847,0,11,47.751,8,10000699,10006495,ADMIN,Shanghai
-649,15/03/2012,ABIN,106246,0,16,96.1592,14,10000922,10000661,ADMIN,Shanghai
-650,15/05/2013,Auction,64076,0,15,37.743,18,10000148,10009040,ADMIN,Shanghai
-651,03/11/2012,Auction,61323,0,15,14.3851,4,10000353,10002536,ADMIN,Shanghai
-652,28/07/2012,ABIN,175750,0,-99,45.5415,9,10000869,10009969,ANALYST,Beijing
-653,26/06/2013,FP-non GTC,95672,0,14,90.732,9,10000507,10005796,ANALYST,Beijing
-654,11/09/2012,Others,223,0,15,10.5185,9,10000705,10004209,ANALYST,Beijing
-655,24/09/2012,FP-GTC,43479,0,16,3.4611,6,10000991,10006760,ANALYST,Beijing
-656,13/09/2013,FP-GTC,100847,0,-99,51.0213,18,10000871,10003181,ANALYST,Beijing
-657,01/07/2013,Others,132939,0,16,14.6584,20,10000859,10006357,ADMIN,Shanghai
-658,13/06/2012,Others,26262,0,-99,46.0446,13,10000041,10006338,ANALYST,Beijing
-659,26/10/2013,Auction,3838,0,12,24.8196,18,10000561,10006508,ADMIN,Shanghai
-660,05/03/2012,Others,26262,0,13,66.7748,17,10000859,10005152,ANALYST,Beijing
-661,10/12/2012,FP-GTC,20485,0,12,52.3473,8,10000171,10002029,ANALYST,Beijing
-662,15/11/2013,Others,2023,0,5,84.6955,16,10000451,10006083,ADMIN,Shanghai
-663,12/12/2013,ABIN,80135,0,11,95.0112,11,10000718,10000889,ANALYST,Beijing
-664,16/04/2013,ABIN,106340,15,16,14.6367,19,10000822,10001648,ANALYST,Beijing
-665,10/03/2012,Auction,161567,15,16,5.28,13,10000081,10006504,ANALYST,Beijing
-666,19/09/2013,Auction,11848,0,11,27.6166,2,10000388,10008310,ADMIN,Shanghai
-667,28/12/2013,FP-GTC,216,0,15,90.1257,18,10000122,10004220,MODELER,Hongkong
-668,26/01/2012,ABIN,314,0,12,3.5634,8,10000912,10002566,ANALYST,Beijing
-669,13/12/2013,FP-GTC,158798,0,5,41.6652,8,10000485,10004610,MODELER,Hongkong
-670,26/09/2013,FP-non GTC,80135,0,11,20.5746,9,10000138,10008645,ANALYST,Beijing
-671,20/01/2012,Auction,62179,0,14,94.2904,1,10000310,10006436,ANALYST,Beijing
-672,20/04/2013,ABIN,963,0,5,68.3604,1,10000571,10003785,ADMIN,Shanghai
-673,03/02/2012,FP-GTC,164,0,11,94.0501,18,10000120,10004595,MODELER,Hongkong
-674,14/10/2013,Others,2023,0,12,84.9792,2,10000371,10008636,MODELER,Hongkong
-675,19/08/2013,FP-GTC,57990,3,14,57.0343,2,10000171,10008666,MODELER,Hongkong
-676,18/06/2012,Others,963,0,12,44.4186,14,10000343,10004277,MODELER,Hongkong
-677,18/10/2012,Others,31519,0,12,50.6753,20,10000875,10004037,MODELER,Hongkong
-678,09/07/2013,Others,36250,0,16,99.5251,1,10000446,10007730,MODELER,Hongkong
-679,19/01/2012,ABIN,38238,0,14,0.1259,20,10000242,10001967,ADMIN,Shanghai
-680,19/05/2013,FP-non GTC,13836,0,16,92.7668,16,10000895,10008811,MODELER,Hongkong
-681,29/01/2013,Others,95672,0,5,13.6012,5,10000843,10005299,ADMIN,Shanghai
-682,30/01/2013,ABIN,67698,0,5,13.9968,13,10000424,10008453,MODELER,Hongkong
-683,02/09/2012,FP-GTC,223,0,-99,38.0021,4,10000803,10006720,ADMIN,Shanghai
-684,11/05/2012,Auction,11848,0,13,7.191,19,10000964,10004802,ADMIN,Shanghai
-685,14/05/2013,FP-non GTC,759,0,16,43.172,16,10000123,10004560,ANALYST,Beijing
-686,06/03/2013,ABIN,3838,0,16,59.8866,1,10000653,10007059,ADMIN,Shanghai
-687,23/08/2012,FP-GTC,20213,0,16,91.3442,10,10000615,10001288,ANALYST,Beijing
-688,24/03/2013,FP-GTC,4943,0,-99,18.4365,5,10000718,10003403,MODELER,Hongkong
-689,26/01/2012,FP-non GTC,11554,0,14,41.6729,11,10000174,10006864,ADMIN,Shanghai
-690,07/10/2013,Others,108782,15,14,14.913,7,10000643,10005077,ADMIN,Shanghai
-691,25/07/2013,Others,13987,0,-99,28.7021,4,10000154,10008498,MODELER,Hongkong
-692,08/11/2013,FP-non GTC,31673,0,13,83.8884,19,10000059,10006354,MODELER,Hongkong
-693,06/01/2013,ABIN,82494,15,12,74.4582,9,10000023,10002631,MODELER,Hongkong
-694,21/01/2013,FP-non GTC,13987,0,11,72.4276,9,10000541,10008262,ANALYST,Beijing
-695,22/11/2013,Auction,67698,0,11,53.1908,6,10000553,10008665,ANALYST,Beijing
-696,02/02/2012,FP-non GTC,175750,0,12,79.116,4,10000121,10003008,ANALYST,Beijing
-697,14/10/2012,FP-GTC,24760,0,11,70.5894,13,10000677,10000508,ADMIN,Shanghai
-698,20/10/2012,ABIN,73506,0,11,3.7214,20,10000076,10001297,ANALYST,Beijing
-699,01/12/2013,Others,16145,0,15,54.8674,6,10000092,10003183,MODELER,Hongkong
-700,07/11/2012,FP-non GTC,53064,0,14,53.4385,13,10000267,10009566,MODELER,Hongkong
-701,20/04/2012,FP-non GTC,53064,0,15,76.5825,10,10000551,10002058,ANALYST,Beijing
-702,18/11/2012,Auction,50677,0,11,44.5149,7,10000924,10000200,MODELER,Hongkong
-703,31/07/2013,FP-GTC,51582,0,14,86.5325,10,10000652,10000550,MODELER,Hongkong
-704,25/12/2012,Others,31387,3,13,79.7624,6,10000495,10007108,ADMIN,Shanghai
-705,10/08/2012,ABIN,2635,0,12,54.5975,2,10000453,10000115,ADMIN,Shanghai
-706,09/06/2012,Others,31519,0,15,20.1828,17,10000226,10002561,ANALYST,Beijing
-707,16/09/2012,Others,44079,0,16,46.0626,12,10000193,10008238,ANALYST,Beijing
-708,23/04/2012,Others,16145,0,11,60.2064,9,10000058,10002015,ANALYST,Beijing
-709,03/01/2012,Others,164,0,15,94.1749,15,10000509,10000641,ADMIN,Shanghai
-710,16/05/2012,FP-GTC,13987,0,15,5.6834,8,10000052,10002235,ADMIN,Shanghai
-711,24/07/2012,FP-non GTC,16145,0,15,8.1625,14,10000036,10000988,ADMIN,Shanghai
-712,01/01/2013,FP-non GTC,32996,15,15,75.5057,7,10000393,10003712,ADMIN,Shanghai
-713,28/01/2013,FP-non GTC,155226,0,16,49.2395,17,10000966,10004827,ANALYST,Beijing
-714,19/03/2013,FP-GTC,20485,0,5,57.4433,1,10000138,10003150,ADMIN,Shanghai
-715,03/02/2013,FP-GTC,57784,0,5,11.8152,9,10000731,10007587,ADMIN,Shanghai
-716,02/01/2012,Others,53064,0,-99,45.8162,3,10000785,10006641,ANALYST,Beijing
-717,23/02/2013,ABIN,16145,0,14,25.1092,14,10000582,10000432,ANALYST,Beijing
-718,19/04/2013,Auction,159184,0,12,85.351,10,10000422,10005472,ANALYST,Beijing
-719,12/10/2012,FP-non GTC,139973,0,11,95.7331,6,10000791,10008481,ANALYST,Beijing
-720,20/04/2012,FP-non GTC,87118,0,16,15.2148,19,10000621,10001203,ADMIN,Shanghai
-721,30/04/2013,Auction,53064,0,5,89.5035,17,10000377,10006787,MODELER,Hongkong
-722,24/10/2013,FP-non GTC,73506,0,12,84.0199,17,10000503,10007044,ANALYST,Beijing
-723,08/01/2012,Auction,63861,0,13,74.886,11,10000589,10006905,ADMIN,Shanghai
-724,21/05/2012,Others,63861,3,16,63.7247,14,10000730,10007764,ANALYST,Beijing
-725,29/01/2013,Auction,24541,0,15,22.2189,18,10000167,10005939,ANALYST,Beijing
-726,08/02/2012,Auction,533,0,5,53.5867,15,10000564,10008398,MODELER,Hongkong
-727,11/08/2012,FP-non GTC,174053,3,13,0.5792,1,10000568,10009768,ANALYST,Beijing
-728,05/07/2012,FP-GTC,20886,0,12,31.1464,2,10000806,10007756,MODELER,Hongkong
-729,08/01/2012,Auction,156356,0,5,7.4293,9,10000638,10003893,MODELER,Hongkong
-730,14/02/2012,Others,65,0,14,62.6745,9,10000253,10000065,ADMIN,Shanghai
-731,25/03/2013,FP-non GTC,48027,0,-99,53.489,2,10000899,10004566,MODELER,Hongkong
-732,30/09/2012,Others,57013,0,12,75.9109,3,10000274,10004646,MODELER,Hongkong
-733,01/09/2013,ABIN,32876,0,12,75.0498,12,10000086,10004192,ADMIN,Shanghai
-734,11/11/2013,FP-GTC,164,0,12,36.6051,19,10000521,10001321,MODELER,Hongkong
-735,18/02/2012,FP-GTC,57013,0,12,14.3117,3,10000763,10008044,ANALYST,Beijing
-736,23/11/2013,ABIN,139973,0,15,58.3344,16,10000787,10008203,ANALYST,Beijing
-737,21/03/2012,FP-non GTC,41940,0,15,38.2502,1,10000342,10004965,MODELER,Hongkong
-738,24/02/2012,Auction,57990,0,5,73.5795,16,10000175,10007860,ANALYST,Beijing
-739,20/07/2013,Auction,88750,0,-99,58.2532,16,10000122,10002332,ANALYST,Beijing
-740,17/10/2012,Others,43479,0,13,67.9585,2,10000516,10003865,ANALYST,Beijing
-741,03/11/2012,FP-non GTC,118687,3,13,90.9173,16,10000957,10004057,ADMIN,Shanghai
-742,22/10/2012,FP-GTC,60340,0,14,68.5877,18,10000502,10003402,ANALYST,Beijing
-743,25/04/2013,FP-GTC,11554,0,5,60.828,18,10000074,10000561,ADMIN,Shanghai
-744,23/10/2012,FP-GTC,24760,0,13,6.0732,10,10000138,10001443,ANALYST,Beijing
-745,29/04/2013,ABIN,175750,0,11,8.0341,15,10000077,10006577,ADMIN,Shanghai
-746,06/05/2012,Auction,106340,15,14,8.0384,8,10000234,10008206,MODELER,Hongkong
-747,15/07/2012,ABIN,57784,0,14,27.1882,7,10000729,10004164,MODELER,Hongkong
-748,15/10/2012,Auction,67698,0,-99,63.179,6,10000226,10006851,MODELER,Hongkong
-749,22/03/2013,FP-non GTC,57013,0,14,27.7251,11,10000658,10009513,ADMIN,Shanghai
-750,12/08/2013,FP-non GTC,11848,0,13,38.5597,20,10000020,10004285,ANALYST,Beijing
-751,14/07/2012,FP-GTC,80053,0,-99,81.0002,18,10000850,10002887,ADMIN,Shanghai
-752,12/08/2012,FP-non GTC,38238,0,11,58.6987,17,10000377,10009099,ANALYST,Beijing
-753,09/09/2013,FP-GTC,106246,0,11,5.0352,7,10000594,10003969,MODELER,Hongkong
-754,11/02/2013,FP-non GTC,57784,0,12,80.842,10,10000657,10007844,ADMIN,Shanghai
-755,09/03/2013,Others,175750,0,14,87.9884,10,10000152,10002498,MODELER,Hongkong
-756,18/12/2013,FP-non GTC,20485,0,14,74.4627,16,10000272,10007976,MODELER,Hongkong
-757,16/03/2013,FP-non GTC,106246,0,5,64.1494,16,10000459,10002996,MODELER,Hongkong
-758,06/09/2012,FP-non GTC,80135,0,13,30.1506,12,10000269,10003262,MODELER,Hongkong
-759,18/06/2013,FP-non GTC,80135,0,-99,75.5826,14,10000706,10004201,ANALYST,Beijing
-760,16/03/2013,FP-GTC,156614,0,15,36.7157,19,10000761,10005335,MODELER,Hongkong
-761,07/11/2012,Others,1349,0,15,96.7575,10,10000612,10005478,ANALYST,Beijing
-762,25/11/2012,ABIN,95173,0,5,61.1186,12,10000364,10001034,MODELER,Hongkong
-763,10/08/2013,ABIN,31673,0,11,63.4095,15,10000421,10009510,ADMIN,Shanghai
-764,17/02/2012,ABIN,88750,0,14,87.8624,9,10000306,10006994,MODELER,Hongkong
-765,29/03/2013,ABIN,139973,0,16,9.1685,10,10000089,10003257,MODELER,Hongkong
-766,27/12/2012,ABIN,121153,0,12,88.7482,20,10000726,10003899,MODELER,Hongkong
-767,13/10/2013,FP-GTC,67698,0,16,20.8225,18,10000504,10008744,MODELER,Hongkong
-768,26/09/2012,ABIN,2023,0,11,83.6499,12,10000641,10007443,MODELER,Hongkong
-769,01/08/2012,Others,139973,0,-99,1.0542,11,10000202,10008156,MODELER,Hongkong
-770,02/03/2012,ABIN,113802,15,5,23.4439,9,10000068,10003049,MODELER,Hongkong
-771,25/10/2012,Others,10866,0,11,60.7453,8,10000770,10008471,ADMIN,Shanghai
-772,07/01/2013,FP-non GTC,20485,0,16,16.4177,17,10000698,10005385,ADMIN,Shanghai
-773,18/03/2012,FP-non GTC,46575,0,15,55.2768,18,10000962,10007105,MODELER,Hongkong
-774,09/07/2012,FP-GTC,106246,0,14,99.6346,2,10000563,10009652,MODELER,Hongkong
-775,26/09/2012,FP-GTC,314,0,13,45.8302,5,10000769,10002802,ANALYST,Beijing
-776,14/01/2012,FP-non GTC,15868,0,16,47.5496,11,10000869,10006392,ADMIN,Shanghai
-777,01/01/2012,ABIN,63861,0,13,76.1393,14,10000028,10003113,ADMIN,Shanghai
-778,15/08/2012,FP-GTC,26249,0,5,40.7448,14,10000419,10005625,ADMIN,Shanghai
-779,24/09/2012,FP-non GTC,43398,0,12,54.0864,19,10000688,10004711,MODELER,Hongkong
-780,13/10/2013,Others,533,0,-99,58.0312,8,10000658,10005979,MODELER,Hongkong
-781,13/07/2012,FP-GTC,148324,15,11,71.6067,18,10000006,10009327,ADMIN,Shanghai
-782,21/01/2012,FP-GTC,32996,15,11,86.8077,13,10000928,10006222,ADMIN,Shanghai
-783,04/04/2012,Others,63861,3,16,85.2671,2,10000648,10007439,MODELER,Hongkong
-784,20/10/2012,FP-non GTC,31519,0,13,99.193,13,10000469,10004261,ANALYST,Beijing
-785,28/12/2013,FP-GTC,1504,0,5,14.9821,8,10000342,10004429,ANALYST,Beijing
-786,19/06/2013,Others,25147,0,5,77.4671,10,10000212,10007671,ANALYST,Beijing
-787,04/05/2013,Others,13836,0,11,13.4655,11,10000331,10000004,MODELER,Hongkong
-788,28/06/2012,ABIN,145970,0,15,74.3304,2,10000900,10009524,ANALYST,Beijing
-789,24/12/2012,FP-non GTC,156614,0,12,79.4508,3,10000743,10003295,MODELER,Hongkong
-790,17/07/2013,FP-GTC,139973,0,13,30.0555,2,10000949,10007067,ADMIN,Shanghai
-791,23/11/2013,Others,31519,3,11,9.0845,4,10000883,10008607,MODELER,Hongkong
-792,15/01/2013,Others,164261,0,12,31.5563,6,10000029,10005351,ANALYST,Beijing
-793,15/04/2012,FP-non GTC,20213,0,-99,55.6917,15,10000513,10006690,ANALYST,Beijing
-794,20/01/2013,FP-non GTC,24760,0,15,71.7279,13,10000274,10003552,ANALYST,Beijing
-795,13/11/2012,FP-GTC,95173,0,5,95.7043,3,10000418,10004402,ANALYST,Beijing
-796,03/09/2012,ABIN,57990,0,12,1.5845,19,10000703,10007782,ANALYST,Beijing
-797,02/01/2012,FP-non GTC,57990,3,16,12.3706,1,10000090,10006504,ADMIN,Shanghai
-798,06/12/2013,Auction,67703,3,5,70.4134,14,10000753,10007346,MODELER,Hongkong
-799,09/06/2012,Others,50677,0,12,88.7297,10,10000990,10004575,MODELER,Hongkong
-800,17/05/2012,ABIN,31519,0,14,37.707,13,10000326,10001200,ADMIN,Shanghai
-801,03/03/2012,Others,156356,0,16,50.7844,2,10000693,10008188,MODELER,Hongkong
-802,21/05/2012,Auction,44079,0,5,86.759,1,10000795,10003443,ADMIN,Shanghai
-803,15/10/2013,Auction,26262,0,15,56.3327,16,10000077,10000771,MODELER,Hongkong
-804,27/12/2013,ABIN,20213,0,13,96.1815,9,10000195,10005442,ADMIN,Shanghai
-805,15/02/2012,ABIN,314,0,5,6.5869,6,10000753,10002892,ANALYST,Beijing
-806,17/08/2012,FP-non GTC,6762,0,15,41.0782,19,10000797,10005277,ADMIN,Shanghai
-807,29/04/2013,Auction,145970,0,15,11.0876,6,10000208,10001555,ADMIN,Shanghai
-808,27/10/2013,Auction,80053,0,14,81.7435,19,10000310,10001507,ADMIN,Shanghai
-809,21/12/2012,FP-GTC,63861,0,13,57.9491,9,10000971,10000647,MODELER,Hongkong
-810,29/07/2013,ABIN,45333,0,12,33.5405,5,10000475,10009395,ADMIN,Shanghai
-811,18/04/2012,Auction,64076,0,13,83.3621,11,10000697,10004193,ANALYST,Beijing
-812,14/12/2012,FP-non GTC,63889,0,16,0.964,10,10000607,10000685,ANALYST,Beijing
-813,07/06/2013,Auction,145970,0,13,6.1316,8,10000450,10000965,ANALYST,Beijing
-814,27/11/2012,Others,164,0,15,53.7718,8,10000305,10002326,ADMIN,Shanghai
-815,03/08/2012,ABIN,61323,0,15,62.2945,15,10000214,10006731,ANALYST,Beijing
-816,07/07/2012,ABIN,75665,0,15,77.0551,10,10000094,10002517,ANALYST,Beijing
-817,06/08/2012,FP-non GTC,99985,0,13,39.1253,14,10000609,10004893,MODELER,Hongkong
-818,01/10/2012,Auction,759,0,15,69.179,16,10000199,10003906,MODELER,Hongkong
-819,22/03/2012,Auction,20865,0,5,40.78,11,10000392,10003136,MODELER,Hongkong
-820,06/02/2012,FP-non GTC,132939,0,11,66.8705,2,10000207,10008892,ADMIN,Shanghai
-821,17/10/2012,FP-non GTC,63861,3,14,35.8018,18,10000096,10008663,ANALYST,Beijing
-822,21/02/2013,Others,41940,0,5,87.4721,12,10000858,10006568,MODELER,Hongkong
-823,16/11/2012,FP-GTC,57784,0,15,96.2799,4,10000537,10003039,ADMIN,Shanghai
-824,10/01/2012,Others,158666,15,11,7.4077,10,10000190,10005422,ADMIN,Shanghai
-825,29/10/2013,FP-non GTC,65,0,-99,21.3439,6,10000182,10003311,ADMIN,Shanghai
-826,30/07/2013,Auction,103178,15,-99,94.8679,5,10000943,10007911,ADMIN,Shanghai
-827,05/08/2012,FP-non GTC,175750,0,14,10.098,9,10000212,10006111,ADMIN,Shanghai
-828,29/07/2012,ABIN,24760,0,13,44.527,10,10000168,10008551,ANALYST,Beijing
-829,27/04/2013,FP-GTC,57990,3,-99,75.6664,19,10000878,10005106,ANALYST,Beijing
-830,06/12/2012,FP-GTC,37831,0,11,57.1191,7,10000090,10006324,ADMIN,Shanghai
-831,29/08/2013,FP-GTC,164,0,16,75.2345,15,10000689,10007653,ANALYST,Beijing
-832,30/08/2012,FP-GTC,63861,0,12,21.2099,12,10000643,10009845,MODELER,Hongkong
-833,30/05/2013,ABIN,57013,0,15,28.841,14,10000290,10009369,ADMIN,Shanghai
-834,17/10/2013,ABIN,50677,0,-99,16.3836,17,10000705,10006136,ANALYST,Beijing
-835,25/04/2012,FP-GTC,67698,0,11,52.3502,17,10000257,10002325,ANALYST,Beijing
-836,18/08/2013,ABIN,759,0,16,23.8258,20,10000911,10009193,ADMIN,Shanghai
-837,24/02/2013,Others,13836,0,-99,40.6826,11,10000399,10004991,ADMIN,Shanghai
-838,22/07/2012,FP-non GTC,158798,0,15,94.1886,16,10000705,10005096,ANALYST,Beijing
-839,05/08/2012,FP-GTC,100847,0,-99,74.15,18,10000797,10006264,ADMIN,Shanghai
-840,30/12/2012,Auction,31519,0,11,69.1499,5,10000576,10002471,ANALYST,Beijing
-841,18/05/2013,Auction,63864,3,14,44.5408,6,10000232,10006928,ANALYST,Beijing
-842,01/11/2012,ABIN,53064,0,15,19.9817,9,10000065,10004624,ADMIN,Shanghai
-843,25/06/2013,FP-GTC,156614,0,16,91.4652,2,10000326,10006437,ANALYST,Beijing
-844,11/07/2013,FP-non GTC,10058,3,5,92.6671,18,10000787,10002813,ADMIN,Shanghai
-845,19/01/2013,FP-non GTC,13987,0,12,49.0313,3,10000556,10005696,ADMIN,Shanghai
-846,17/06/2013,Auction,94847,0,13,95.5377,11,10000047,10001497,ANALYST,Beijing
-847,23/12/2012,Others,64076,0,-99,50.1173,10,10000063,10003567,MODELER,Hongkong
-848,07/01/2013,FP-non GTC,67698,0,5,6.5979,4,10000412,10003921,ADMIN,Shanghai
-849,20/01/2012,ABIN,175750,3,16,79.6931,6,10000270,10000997,ADMIN,Shanghai
-850,14/01/2013,Others,95672,3,12,31.7315,19,10000044,10002298,ANALYST,Beijing
-851,22/02/2013,Others,26249,0,16,43.7969,11,10000634,10003500,ADMIN,Shanghai
-852,13/04/2013,FP-non GTC,156614,0,16,79.3213,8,10000321,10003186,ADMIN,Shanghai
-853,18/09/2013,FP-non GTC,43398,0,5,17.3702,9,10000621,10002418,MODELER,Hongkong
-854,05/09/2013,ABIN,67703,3,11,55.6439,12,10000784,10004250,ANALYST,Beijing
-855,25/05/2013,FP-GTC,43398,0,15,73.0548,9,10000921,10007198,MODELER,Hongkong
-856,06/10/2012,Others,100847,0,16,77.0258,6,10000606,10009841,MODELER,Hongkong
-857,08/11/2013,Others,164261,0,11,16.6959,11,10000136,10006656,ANALYST,Beijing
-858,13/04/2012,FP-non GTC,3838,0,5,68.4347,17,10000451,10000115,ADMIN,Shanghai
-859,03/07/2012,FP-GTC,63861,0,-99,89.5962,5,10000893,10006463,ADMIN,Shanghai
-860,21/02/2012,Auction,40059,3,-99,84.7455,13,10000613,10009000,MODELER,Hongkong
-861,25/02/2012,FP-non GTC,759,0,13,23.9258,11,10000826,10005739,MODELER,Hongkong
-862,30/10/2013,FP-non GTC,57990,0,16,88.4859,2,10000782,10006594,MODELER,Hongkong
-863,12/10/2013,FP-non GTC,57784,0,14,85.8986,20,10000220,10004445,MODELER,Hongkong
-864,15/08/2012,FP-GTC,223,0,12,32.1974,8,10000316,10002696,ANALYST,Beijing
-865,18/10/2012,FP-GTC,15115,0,16,52.5243,13,10000353,10009319,ADMIN,Shanghai
-866,09/11/2013,FP-GTC,20886,0,13,94.0128,8,10000690,10004603,MODELER,Hongkong
-867,27/09/2012,ABIN,31673,0,12,24.1332,1,10000404,10009648,ANALYST,Beijing
-868,08/05/2012,FP-GTC,95672,0,16,88.8853,12,10000984,10002017,ADMIN,Shanghai
-869,14/01/2013,Others,95173,0,12,86.3987,7,10000501,10002846,ADMIN,Shanghai
-870,02/12/2012,Auction,152801,0,-99,71.8799,17,10000021,10009050,ADMIN,Shanghai
-871,13/03/2012,ABIN,31519,3,15,96.8998,1,10000188,10009876,ADMIN,Shanghai
-872,15/06/2013,Auction,121153,0,14,66.2729,16,10000101,10001578,ANALYST,Beijing
-873,11/04/2012,Others,63861,0,15,88.0097,4,10000536,10005703,ADMIN,Shanghai
-874,25/01/2013,FP-non GTC,1357,0,16,49.8991,2,10000637,10008932,ANALYST,Beijing
-875,17/10/2013,FP-GTC,106246,0,15,92.2507,14,10000046,10005384,MODELER,Hongkong
-876,04/10/2012,Others,4943,0,13,88.8498,19,10000215,10001232,ADMIN,Shanghai
-877,14/05/2013,Auction,314,0,13,77.3441,6,10000900,10008675,ANALYST,Beijing
-878,17/03/2013,FP-non GTC,759,0,16,17.9719,6,10000759,10000828,MODELER,Hongkong
-879,20/01/2013,FP-non GTC,51582,0,16,3.6644,5,10000961,10000593,ADMIN,Shanghai
-880,20/03/2013,FP-non GTC,57990,0,12,9.368,16,10000846,10009003,ANALYST,Beijing
-881,06/06/2012,FP-non GTC,165888,0,16,9.4584,10,10000312,10004071,ADMIN,Shanghai
-882,13/12/2012,Auction,43479,0,16,84.0632,6,10000349,10004913,MODELER,Hongkong
-883,13/05/2012,FP-non GTC,139973,3,5,37.351,17,10000790,10000443,MODELER,Hongkong
-884,27/03/2013,Auction,48027,0,5,75.7167,1,10000424,10006108,MODELER,Hongkong
-885,08/08/2013,FP-GTC,37831,0,-99,27.78,16,10000637,10000193,ANALYST,Beijing
-886,06/04/2012,ABIN,314,0,12,18.3274,5,10000749,10007076,MODELER,Hongkong
-887,11/10/2013,FP-non GTC,57013,0,14,39.1868,14,10000599,10004702,MODELER,Hongkong
-888,18/01/2012,FP-GTC,41940,0,12,94.5165,2,10000102,10003299,MODELER,Hongkong
-889,20/05/2013,ABIN,50677,0,16,25.6136,20,10000423,10001434,ANALYST,Beijing
-890,24/06/2012,FP-GTC,963,0,13,20.9442,9,10000069,10005352,ADMIN,Shanghai
-891,01/04/2013,ABIN,75665,0,12,65.3307,6,10000543,10006729,MODELER,Hongkong
-892,05/02/2012,ABIN,50508,0,-99,69.4516,7,10000623,10004814,ANALYST,Beijing
-893,24/06/2012,ABIN,314,0,5,13.8612,16,10000347,10003703,ANALYST,Beijing
-894,07/12/2013,Others,1349,0,16,10.8619,19,10000784,10000834,MODELER,Hongkong
-895,01/03/2013,ABIN,73506,0,11,7.7818,3,10000882,10001213,ADMIN,Shanghai
-896,24/08/2012,Auction,45333,0,16,94.9583,6,10000495,10002795,ANALYST,Beijing
-897,04/08/2012,FP-non GTC,20886,0,15,69.73,10,10000428,10004891,MODELER,Hongkong
-898,08/11/2012,Auction,31519,0,15,41.4423,10,10000431,10007344,MODELER,Hongkong
-899,07/08/2013,FP-GTC,156356,0,13,44.6853,4,10000305,10007928,ANALYST,Beijing
-900,25/09/2013,ABIN,43398,0,16,46.5536,12,10000387,10005678,ANALYST,Beijing
-901,26/04/2012,FP-non GTC,95672,0,13,30.9202,8,10000591,10009900,MODELER,Hongkong
-902,30/03/2012,Others,159184,0,12,58.3993,16,10000381,10004902,ADMIN,Shanghai
-903,25/04/2012,Auction,73506,0,14,46.1501,9,10000010,10005565,ANALYST,Beijing
-904,13/09/2012,FP-GTC,20485,0,5,2.4112,18,10000959,10005950,ADMIN,Shanghai
-905,27/09/2013,ABIN,80287,0,-99,64.4361,7,10000770,10006064,ANALYST,Beijing
-906,17/01/2013,Others,36250,0,11,73.5773,14,10000948,10009683,ADMIN,Shanghai
-907,07/07/2012,Auction,103324,15,16,4.8559,20,10000123,10000205,MODELER,Hongkong
-908,15/08/2013,ABIN,13836,0,16,25.8438,9,10000851,10008521,ANALYST,Beijing
-909,07/07/2012,Others,12688,0,5,3.4934,7,10000096,10007124,MODELER,Hongkong
-910,03/06/2012,Auction,80135,0,12,17.8258,15,10000144,10002338,ANALYST,Beijing
-911,24/12/2013,FP-GTC,60340,0,11,22.6409,18,10000341,10002484,MODELER,Hongkong
-912,06/10/2012,ABIN,36250,0,5,7.4322,16,10000384,10004712,ADMIN,Shanghai
-913,03/01/2013,Auction,175750,0,15,14.1976,14,10000366,10000904,ADMIN,Shanghai
-914,15/04/2013,Auction,88750,0,11,98.6057,19,10000584,10004925,MODELER,Hongkong
-915,03/06/2013,FP-non GTC,1349,0,15,11.0419,16,10000866,10004642,ANALYST,Beijing
-916,30/01/2013,FP-GTC,2635,0,14,54.5175,11,10000959,10002206,ANALYST,Beijing
-917,27/08/2013,FP-non GTC,26249,0,13,50.0365,1,10000289,10001579,MODELER,Hongkong
-918,14/05/2012,FP-non GTC,9426,3,5,60.0224,1,10000730,10008731,ADMIN,Shanghai
-919,16/10/2012,FP-non GTC,16145,0,-99,53.0896,10,10000195,10008596,MODELER,Hongkong
-920,24/09/2012,FP-GTC,57013,0,14,81.7929,19,10000498,10000826,MODELER,Hongkong
-921,20/08/2012,FP-non GTC,25147,0,-99,30.7605,17,10000242,10007503,ADMIN,Shanghai
-922,19/02/2012,Others,94847,0,13,20.6395,12,10000707,10009034,ANALYST,Beijing
-923,14/11/2013,ABIN,50508,0,15,91.9217,20,10000208,10003230,ADMIN,Shanghai
-924,10/07/2012,Others,95672,0,13,21.1886,8,10000762,10006779,ADMIN,Shanghai
-925,14/03/2012,ABIN,44079,0,13,89.6838,17,10000227,10003305,ADMIN,Shanghai
-926,17/01/2012,FP-non GTC,314,0,14,44.3066,17,10000251,10003512,ADMIN,Shanghai
-927,27/02/2012,FP-non GTC,15868,0,14,80.8427,3,10000741,10008344,ANALYST,Beijing
-928,23/10/2012,Others,1161,3,5,54.0104,11,10000289,10008643,ADMIN,Shanghai
-929,12/06/2013,Others,53064,0,12,87.1336,1,10000871,10007639,ANALYST,Beijing
-930,06/03/2013,Auction,34273,100,15,6.7044,2,10000269,10006027,MODELER,Hongkong
-931,16/06/2012,Others,533,0,14,43.7104,17,10000172,10005634,ADMIN,Shanghai
-932,19/12/2012,FP-GTC,15868,0,15,94.1727,16,10000366,10002612,ANALYST,Beijing
-933,21/07/2012,ABIN,121153,0,12,42.3437,4,10000688,10001422,ANALYST,Beijing
-934,19/08/2013,FP-non GTC,65,0,11,3.0275,2,10000055,10008305,MODELER,Hongkong
-935,08/08/2012,Auction,15687,0,14,72.9505,11,10000725,10008549,ADMIN,Shanghai
-936,13/12/2013,FP-GTC,4943,0,-99,37.7781,3,10000932,10002759,MODELER,Hongkong
-937,18/05/2013,FP-GTC,3838,0,14,45.1642,11,10000370,10000382,ANALYST,Beijing
-938,31/01/2012,Auction,25147,0,14,78.658,8,10000721,10005361,MODELER,Hongkong
-939,05/12/2013,FP-GTC,62179,0,12,76.4692,8,10000423,10007624,MODELER,Hongkong
-940,07/03/2012,Auction,175750,0,15,19.5357,2,10000053,10004204,MODELER,Hongkong
-941,30/09/2013,Others,100847,0,12,38.2299,11,10000772,10005476,ADMIN,Shanghai
-942,07/01/2013,Others,15115,0,12,82.5099,3,10000864,10003344,ADMIN,Shanghai
-943,04/11/2013,FP-non GTC,57013,0,-99,61.3731,20,10000335,10008636,MODELER,Hongkong
-944,11/08/2012,FP-non GTC,31673,0,12,12.7001,5,10000303,10001980,MODELER,Hongkong
-945,30/12/2012,ABIN,156356,0,14,99.3562,4,10000427,10001133,ANALYST,Beijing
-946,19/05/2013,Others,113593,15,11,31.1634,2,10000361,10006272,ADMIN,Shanghai
-947,03/06/2013,ABIN,314,0,13,32.6968,11,10000373,10008996,ANALYST,Beijing
-948,29/06/2013,Auction,15808,15,12,18.3967,13,10000484,10007850,ADMIN,Shanghai
-949,10/06/2012,FP-GTC,61323,0,13,10.8143,17,10000403,10006130,ADMIN,Shanghai
-950,29/01/2012,FP-GTC,1349,0,15,0.4358,14,10000639,10000947,ANALYST,Beijing
-951,30/05/2013,FP-non GTC,759,0,14,19.592,6,10000116,10002996,MODELER,Hongkong
-952,28/07/2013,ABIN,15868,0,14,59.7123,7,10000668,10009144,ADMIN,Shanghai
-953,23/03/2013,Auction,165888,0,12,67.1201,9,10000063,10003845,MODELER,Hongkong
-954,29/05/2012,FP-GTC,118687,3,-99,98.203,18,10000886,10004781,ADMIN,Shanghai
-955,15/09/2013,FP-non GTC,156614,0,5,43.1795,9,10000302,10007429,ANALYST,Beijing
-956,21/05/2012,FP-GTC,223,0,-99,66.285,13,10000118,10000885,ANALYST,Beijing
-957,03/04/2012,FP-non GTC,26262,0,12,17.5241,13,10000440,10008021,ADMIN,Shanghai
-958,22/05/2012,ABIN,44079,0,16,25.2405,7,10000183,10001255,ANALYST,Beijing
-959,01/01/2014,Others,50508,0,16,92.5843,3,10000911,10004209,MODELER,Hongkong
-960,08/04/2012,Others,16145,3,12,90.2242,6,10000825,10003876,ADMIN,Shanghai
-961,27/11/2013,ABIN,95672,0,16,90.938,17,10000829,10004065,ANALYST,Beijing
-962,22/03/2013,Auction,41940,0,15,72.0648,7,10000795,10008608,ANALYST,Beijing
-963,06/04/2012,Others,64076,0,14,86.9185,5,10000356,10008162,ADMIN,Shanghai
-964,02/05/2013,Others,156356,0,11,41.6871,14,10000122,10003384,ANALYST,Beijing
-965,02/04/2012,Others,132939,0,12,75.8372,7,10000395,10007751,ADMIN,Shanghai
-966,29/06/2012,ABIN,26262,0,13,83.6043,17,10000120,10009086,ADMIN,Shanghai
-967,06/07/2012,FP-GTC,11554,0,16,38.4329,4,10000352,10001074,MODELER,Hongkong
-968,10/06/2012,ABIN,156614,0,-99,94.1812,10,10000280,10003259,ANALYST,Beijing
-969,05/04/2012,Auction,164261,0,12,0.9691,13,10000480,10008760,ANALYST,Beijing
-970,02/11/2012,Auction,31519,0,11,98.0505,1,10000893,10008194,MODELER,Hongkong
-971,20/05/2012,Auction,50508,0,15,61.9439,7,10000363,10006222,ANALYST,Beijing
-972,31/03/2012,Auction,80135,0,16,11.1543,2,10000692,10009054,MODELER,Hongkong
-973,01/07/2012,FP-non GTC,63889,0,-99,86.6557,8,10000402,10009722,MODELER,Hongkong
-974,04/01/2013,ABIN,41940,0,-99,48.4505,12,10000296,10008506,ANALYST,Beijing
-975,10/02/2013,FP-non GTC,63861,0,13,60.3535,8,10000214,10006412,ADMIN,Shanghai
-976,25/03/2012,Others,73506,0,11,74.2113,5,10000801,10006619,ANALYST,Beijing
-977,23/07/2013,FP-non GTC,53064,0,16,69.2368,11,10000542,10003961,MODELER,Hongkong
-978,09/04/2012,Auction,11848,0,15,76.172,11,10000555,10009344,ADMIN,Shanghai
-979,01/08/2012,Others,12688,0,5,85.4589,14,10000130,10005142,ADMIN,Shanghai
-980,15/07/2012,FP-non GTC,31519,3,-99,17.8485,14,10000045,10007670,MODELER,Hongkong
-981,06/10/2013,ABIN,1349,0,11,52.278,4,10000622,10005595,MODELER,Hongkong
-982,26/10/2012,Auction,25147,0,13,60.9912,10,10000379,10009391,ADMIN,Shanghai
-983,04/01/2013,FP-non GTC,37831,0,15,61.0134,10,10000785,10003394,ADMIN,Shanghai
-984,24/03/2012,Auction,139973,0,-99,8.1533,20,10000487,10007074,ADMIN,Shanghai
-985,30/12/2012,Auction,6762,0,-99,77.8179,2,10000645,10001838,ADMIN,Shanghai
-986,03/10/2012,ABIN,103178,15,12,15.2201,11,10000824,10002677,MODELER,Hongkong
-987,18/04/2012,FP-non GTC,50508,0,14,11.9715,11,10000012,10006191,MODELER,Hongkong
-988,30/09/2012,FP-non GTC,31673,0,12,90.0864,12,10000281,10002287,MODELER,Hongkong
-989,24/03/2013,ABIN,4943,0,12,11.0294,11,10000785,10001441,ADMIN,Shanghai
-990,02/05/2012,Others,87118,0,14,52.6549,6,10000924,10004868,ADMIN,Shanghai
-991,03/05/2013,FP-GTC,24760,0,5,8.9077,19,10000758,10004142,MODELER,Hongkong
-992,05/08/2012,FP-non GTC,38238,0,11,0.8463,12,10000066,10003073,MODELER,Hongkong
-993,15/08/2012,Auction,67698,0,11,23.6278,12,10000085,10007036,ADMIN,Shanghai
-994,04/04/2013,Auction,164261,0,13,52.3718,12,10000965,10008307,ADMIN,Shanghai
-995,18/04/2013,ABIN,25147,0,14,71.7687,14,10000157,10001658,ANALYST,Beijing
-996,09/11/2013,ABIN,43479,0,12,39.9269,11,10000024,10007805,ANALYST,Beijing
-997,02/05/2013,FP-non GTC,175750,0,12,17.6149,12,10000862,10001964,ANALYST,Beijing
-998,01/09/2012,ABIN,50508,0,14,99.1518,5,10000746,10009362,ANALYST,Beijing
-999,23/02/2013,FP-non GTC,139255,15,-99,74.6182,20,10000447,10005318,ANALYST,Beijing
-1000,19/01/2012,FP-non GTC,6762,0,13,0.1297,12,10000189,10004748,MODELER,Hongkong
-1001,10/11/2012,Auction,113802,15,-99,11.9583,15,10000596,10006655,ANALYST,Beijing
-1002,11/12/2012,FP-non GTC,95173,0,5,67.9416,20,10000582,10003627,ADMIN,Shanghai
-1003,05/01/2012,ABIN,2635,0,11,72.3723,15,10000375,10002658,ANALYST,Beijing
-1004,04/05/2012,FP-non GTC,43479,0,13,79.1307,14,10000944,10007826,ADMIN,Shanghai
-1005,20/07/2013,ABIN,63861,0,13,86.7155,12,10000274,10005880,MODELER,Hongkong
-1006,25/08/2013,FP-GTC,80053,0,5,11.7108,8,10000059,10001987,MODELER,Hongkong
-1007,24/10/2012,ABIN,38238,0,5,79.2964,4,10000388,10000426,MODELER,Hongkong
-1008,10/01/2012,ABIN,73506,0,5,3.8878,14,10000895,10007256,ANALYST,Beijing
-1009,11/07/2013,Auction,174106,3,12,46.7087,8,10000683,10000752,ANALYST,Beijing
-1010,16/06/2013,FP-GTC,61323,0,15,64.0224,2,10000121,10001535,ADMIN,Shanghai
-1011,23/06/2013,ABIN,63889,0,5,59.2766,16,10000466,10003627,MODELER,Hongkong
-1012,25/01/2013,Others,156356,0,14,79.9992,18,10000611,10002293,ANALYST,Beijing
-1013,09/10/2013,Auction,50508,0,16,70.9682,11,10000009,10006058,MODELER,Hongkong
-1014,19/05/2013,Auction,25147,0,16,8.0973,1,10000731,10007700,ADMIN,Shanghai
-1015,14/10/2012,Auction,67698,0,5,49.0497,10,10000605,10009329,ADMIN,Shanghai
-1016,08/06/2012,Auction,65,0,14,76.5149,17,10000165,10008171,MODELER,Hongkong
-1017,21/12/2012,FP-GTC,15868,0,5,45.9224,17,10000215,10004400,ADMIN,Shanghai
-1018,02/12/2012,Auction,1357,0,15,8.9337,4,10000730,10003527,ANALYST,Beijing
-1019,27/11/2013,ABIN,24760,0,11,62.4675,10,10000932,10006658,ADMIN,Shanghai
-1020,24/11/2013,Auction,95173,0,16,16.7927,8,10000795,10004974,ANALYST,Beijing
-1021,30/11/2013,ABIN,164,0,11,10.8897,16,10000914,10001759,MODELER,Hongkong
-1022,22/12/2012,Auction,15868,0,15,7.4153,7,10000540,10006963,MODELER,Hongkong
-1023,13/05/2012,FP-non GTC,26249,0,5,58.8647,5,10000898,10002705,MODELER,Hongkong
-1024,22/06/2013,ABIN,95672,3,12,71.8628,10,10000026,10009984,ANALYST,Beijing
-1025,22/05/2013,ABIN,63861,0,11,71.8836,3,10000650,10003996,ADMIN,Shanghai
-1026,17/11/2012,FP-GTC,32876,0,12,35.6019,6,10000959,10000392,ADMIN,Shanghai
-1027,13/04/2013,Auction,38238,0,5,21.7443,4,10000036,10001425,ANALYST,Beijing
-1028,09/09/2012,FP-non GTC,87118,0,14,46.2858,17,10000512,10005447,MODELER,Hongkong
-1029,23/08/2013,FP-non GTC,41940,0,11,98.4779,7,10000844,10007297,MODELER,Hongkong
-1030,02/09/2012,FP-non GTC,43398,0,12,91.8543,10,10000917,10008944,ANALYST,Beijing
-1031,23/11/2012,Others,145970,0,11,43.8631,15,10000997,10007702,ANALYST,Beijing
-1032,22/01/2012,FP-non GTC,164261,0,16,74.9772,5,10000091,10002303,MODELER,Hongkong
-1033,02/05/2012,ABIN,152801,0,13,15.2794,8,10000649,10004568,ADMIN,Shanghai
-1034,14/08/2013,Others,50677,0,13,26.5329,13,10000275,10008334,MODELER,Hongkong
-1035,04/06/2012,Others,156614,0,14,42.781,9,10000347,10003401,ANALYST,Beijing
-1036,04/08/2012,FP-non GTC,145970,0,15,92.6953,14,10000622,10002124,ADMIN,Shanghai
-1037,15/08/2012,Auction,32876,0,16,35.2015,9,10000533,10001315,MODELER,Hongkong
-1038,05/11/2012,Auction,95672,0,11,0.5233,3,10000965,10004000,MODELER,Hongkong
-1039,12/10/2013,ABIN,165888,0,11,27.3564,9,10000036,10006840,ADMIN,Shanghai
-1040,22/10/2012,ABIN,20485,0,14,28.6742,19,10000448,10004840,ANALYST,Beijing
-1041,02/07/2012,Auction,20213,0,15,81.3332,19,10000484,10007894,ADMIN,Shanghai
-1042,13/01/2013,Others,152801,0,11,3.3406,2,10000124,10006403,MODELER,Hongkong
-1043,17/05/2012,Others,132939,0,15,55.3503,19,10000086,10007815,MODELER,Hongkong
-1044,05/02/2013,Others,48904,0,15,25.234,18,10000246,10002491,MODELER,Hongkong
-1045,13/09/2013,FP-GTC,32876,0,-99,10.7672,16,10000729,10002514,ANALYST,Beijing
-1046,14/06/2012,ABIN,20865,0,14,47.9841,18,10000406,10009589,ADMIN,Shanghai
-1047,11/02/2013,Auction,152801,0,5,86.4355,4,10000050,10001255,ANALYST,Beijing
-1048,13/12/2012,Auction,50508,0,5,72.5114,11,10000715,10009705,ANALYST,Beijing
-1049,30/11/2013,Auction,3838,0,-99,6.8517,11,10000706,10009022,ADMIN,Shanghai
-1050,15/03/2013,FP-non GTC,20865,0,14,93.304,1,10000926,10002565,MODELER,Hongkong
-1051,12/08/2012,FP-GTC,57784,0,-99,18.4045,7,10000231,10006705,ADMIN,Shanghai
-1052,19/10/2013,Others,1161,3,14,79.1353,13,10000561,10005173,ADMIN,Shanghai
-1053,17/02/2013,Others,31519,0,-99,49.0409,3,10000904,10006932,ADMIN,Shanghai
-1054,30/03/2012,FP-non GTC,314,0,11,93.6413,7,10000041,10002998,ANALYST,Beijing
-1055,26/08/2012,Others,175750,0,-99,46.6557,20,10000719,10004608,ANALYST,Beijing
-1056,26/12/2012,Auction,139973,0,15,6.9809,20,10000044,10003763,MODELER,Hongkong
-1057,30/01/2013,Others,159184,0,13,45.4599,20,10000131,10001765,ANALYST,Beijing
-1058,27/04/2012,Auction,48904,0,5,16.3472,18,10000157,10007395,MODELER,Hongkong
-1059,10/11/2012,ABIN,16145,0,13,0.4693,8,10000668,10004952,ANALYST,Beijing
-1060,13/05/2012,Auction,121153,0,5,24.642,16,10000314,10005354,MODELER,Hongkong
-1061,01/02/2012,FP-GTC,6762,0,13,5.9417,3,10000851,10003578,MODELER,Hongkong
-1062,16/07/2012,Auction,95672,0,-99,11.9934,1,10000828,10008797,MODELER,Hongkong
-1063,21/09/2012,Auction,106246,0,5,39.0057,5,10000826,10000054,ANALYST,Beijing
-1064,20/10/2012,FP-GTC,314,0,16,5.5339,20,10000011,10002088,ANALYST,Beijing
-1065,29/04/2013,FP-non GTC,100847,0,13,3.3872,20,10000840,10001720,ADMIN,Shanghai
-1066,16/02/2013,FP-non GTC,43398,0,13,3.4171,12,10000106,10006976,ANALYST,Beijing
-1067,11/09/2013,FP-GTC,63889,0,12,15.4662,20,10000004,10006486,MODELER,Hongkong
-1068,18/12/2012,Others,80287,0,15,71.309,7,10000722,10009936,MODELER,Hongkong
-1069,14/09/2013,FP-non GTC,20865,0,-99,98.0987,19,10000591,10004783,ADMIN,Shanghai
-1070,27/01/2013,FP-non GTC,57990,3,14,84.9483,4,10000304,10000164,ANALYST,Beijing
-1071,07/02/2013,Others,95672,0,14,9.7034,14,10000894,10000059,ANALYST,Beijing
-1072,24/03/2013,FP-non GTC,38238,0,-99,98.5328,3,10000286,10005535,MODELER,Hongkong
-1073,05/01/2012,Others,164,0,15,21.3336,12,10000151,10003519,ADMIN,Shanghai
-1074,14/11/2012,Auction,57990,0,12,91.1773,13,10000087,10002414,ADMIN,Shanghai
-1075,24/03/2013,Auction,13987,0,13,5.9953,1,10000551,10006991,ADMIN,Shanghai
-1076,26/12/2013,FP-GTC,32876,0,11,86.1904,11,10000012,10004509,MODELER,Hongkong
-1077,23/06/2013,FP-non GTC,1504,0,12,9.4689,15,10000720,10008088,ANALYST,Beijing
-1078,11/01/2012,FP-GTC,57013,0,13,96.6705,5,10000790,10001942,ADMIN,Shanghai
-1079,30/08/2012,FP-non GTC,62179,0,5,22.4244,16,10000403,10006119,MODELER,Hongkong
-1080,01/09/2012,ABIN,61323,0,12,30.1513,15,10000283,10003870,ANALYST,Beijing
-1081,20/04/2013,Auction,16145,3,12,77.9327,8,10000043,10000188,MODELER,Hongkong
-1082,03/10/2013,Others,963,0,14,82.3371,8,10000002,10006340,MODELER,Hongkong
-1083,23/10/2012,ABIN,45333,0,16,86.7017,3,10000180,10000817,MODELER,Hongkong
-1084,09/10/2013,ABIN,63889,0,-99,33.9942,16,10000485,10002344,ADMIN,Shanghai
-1085,28/04/2013,ABIN,73506,0,5,22.6563,19,10000070,10007027,ANALYST,Beijing
-1086,07/07/2013,ABIN,57990,0,11,11.7659,15,10000834,10000644,ADMIN,Shanghai
-1087,04/01/2012,FP-non GTC,165888,0,16,78.7873,12,10000743,10008093,ANALYST,Beijing
-1088,05/05/2012,ABIN,158666,15,16,55.1036,17,10000372,10009391,ANALYST,Beijing
-1089,04/08/2012,ABIN,43479,0,13,61.3469,13,10000425,10002587,MODELER,Hongkong
-1090,18/02/2013,Others,161567,15,5,43.5109,4,10000315,10009432,MODELER,Hongkong
-1091,09/08/2013,FP-GTC,161567,15,14,0.9756,11,10000420,10002373,MODELER,Hongkong
-1092,30/04/2012,Others,65,0,12,31.5741,18,10000858,10005147,ANALYST,Beijing
-1093,16/10/2013,FP-non GTC,16509,0,15,8.4323,14,10000376,10007194,MODELER,Hongkong
-1094,06/11/2013,FP-GTC,15115,0,12,91.7204,8,10000495,10003677,ADMIN,Shanghai
-1095,13/04/2012,ABIN,37831,0,16,86.915,15,10000947,10000539,MODELER,Hongkong
-1096,21/04/2013,FP-GTC,95672,0,-99,60.6048,6,10000052,10001595,ANALYST,Beijing
-1097,04/12/2013,FP-GTC,121153,0,13,9.3639,11,10000384,10009074,ADMIN,Shanghai
-1098,03/08/2012,Others,50508,0,13,74.2474,1,10000103,10004661,ANALYST,Beijing
-1099,29/12/2013,FP-GTC,1357,0,5,46.9382,16,10000987,10000797,ANALYST,Beijing
-1100,01/11/2012,Others,46575,0,13,54.452,12,10000247,10000707,ANALYST,Beijing
-1101,30/06/2012,FP-non GTC,26249,0,5,74.2755,2,10000641,10000127,MODELER,Hongkong
-1102,20/06/2012,Auction,26262,0,11,11.9173,1,10000418,10009843,MODELER,Hongkong
-1103,24/07/2012,Others,80287,0,14,49.7794,2,10000590,10006327,MODELER,Hongkong
-1104,11/08/2012,FP-GTC,26262,0,11,16.291,16,10000698,10002353,MODELER,Hongkong
-1105,27/06/2013,ABIN,63861,0,-99,5.4086,4,10000810,10007456,ANALYST,Beijing
-1106,28/04/2012,FP-GTC,15115,0,15,91.7724,4,10000414,10005702,MODELER,Hongkong
-1107,19/04/2013,ABIN,73506,0,13,14.8212,20,10000728,10000486,ADMIN,Shanghai
-1108,25/01/2012,Others,32876,0,13,23.4295,1,10000673,10003806,MODELER,Hongkong
-1109,06/11/2013,ABIN,139973,3,12,41.9681,3,10000902,10000143,MODELER,Hongkong
-1110,09/03/2013,Auction,61323,0,15,53.8594,17,10000323,10000572,ANALYST,Beijing
-1111,01/03/2012,ABIN,60340,0,11,78.1279,10,10000947,10001604,ANALYST,Beijing
-1112,28/07/2012,FP-non GTC,279,15,16,80.8961,5,10000691,10005343,MODELER,Hongkong
-1113,04/02/2013,Others,24541,0,11,40.4673,16,10000727,10004879,MODELER,Hongkong
-1114,24/02/2012,Auction,152801,0,13,9.8921,4,10000661,10003640,ADMIN,Shanghai
-1115,13/08/2013,ABIN,156614,0,15,61.6452,20,10000734,10000741,ANALYST,Beijing
-1116,02/04/2012,ABIN,63861,3,11,18.4644,13,10000446,10006619,ANALYST,Beijing
-1117,28/09/2013,Others,95173,0,-99,51.2096,17,10000049,10008305,ANALYST,Beijing
-1118,13/08/2013,FP-non GTC,64076,0,12,85.3952,16,10000581,10005375,ANALYST,Beijing
-1119,09/09/2012,ABIN,20485,0,15,0.5979,17,10000081,10008733,ADMIN,Shanghai
-1120,26/04/2012,Auction,145970,0,11,40.5874,9,10000401,10009751,ANALYST,Beijing
-1121,10/05/2013,Others,26262,0,11,83.8561,14,10000545,10005955,MODELER,Hongkong
-1122,23/05/2013,FP-GTC,80287,0,11,27.6123,18,10000529,10001324,ADMIN,Shanghai
-1123,08/01/2012,Others,175750,0,14,45.4268,11,10000408,10002363,ANALYST,Beijing
-1124,20/07/2012,ABIN,38238,0,16,44.1376,15,10000796,10009325,ADMIN,Shanghai
-1125,15/09/2012,Auction,65,0,-99,97.3146,7,10000792,10005350,MODELER,Hongkong
-1126,17/10/2012,ABIN,65,0,12,68.5363,13,10000569,10006467,MODELER,Hongkong
-1127,03/08/2013,FP-GTC,40059,3
<TRUNCATED>
[18/18] kylin git commit: KYLIN-1528 Create a branch for v1.5 with
HBase 1.x API
Posted by li...@apache.org.
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eaf0cc58
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eaf0cc58
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eaf0cc58
Branch: refs/heads/master-hbase1.x
Commit: eaf0cc5866367eadacfb7a43ca793c2f585dfdc6
Parents: 4c39fcb
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 8 13:42:27 2016 +0000
----------------------------------------------------------------------
examples/test_case_data/sandbox/hbase-site.xml | 19 +---
.../kylin/provision/BuildCubeWithEngine.java | 12 +--
pom.xml | 12 +--
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 ++++----------------
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 +++---
.../apache/kylin/rest/service/CubeService.java | 35 +++-----
.../apache/kylin/rest/service/QueryService.java | 24 +++--
.../apache/kylin/rest/service/UserService.java | 17 ++--
.../kylin/storage/hbase/HBaseConnection.java | 44 ++++-----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++----
.../kylin/storage/hbase/HBaseStorage.java | 3 +-
.../storage/hbase/cube/SimpleHBaseStore.java | 20 ++---
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
.../storage/hbase/steps/DeprecatedGCStep.java | 23 ++---
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++---
.../storage/hbase/util/CleanHtableCLI.java | 12 +--
.../storage/hbase/util/CubeMigrationCLI.java | 36 ++++----
.../hbase/util/CubeMigrationCheckCLI.java | 17 ++--
.../hbase/util/DeployCoprocessorCLI.java | 27 +++---
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 +++----
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++--
.../hbase/util/HBaseRegionSizeCalculator.java | 35 ++++----
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 +++----
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
.../storage/hbase/util/StorageCleanupJob.java | 20 +++--
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
.../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
40 files changed, 365 insertions(+), 432 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
- <property>
- <name>hbase.client.pause</name>
- <value>100</value>
- <description>General client pause value. Used mostly as value to wait
- before running a retry of a failed get, region lookup, etc.
- See hbase.client.retries.number for description of how we backoff from
- this initial pause amount and how this pause works w/ retries.</description>
- </property>
- <property>
- <name>hbase.client.retries.number</name>
- <value>5</value>
- <description>Maximum retries. Used as maximum for all retryable
- operations such as the getting of a cell's value, starting a row update,
- etc. Retry interval is a rough function based on hbase.client.pause. At
- first we retry at this interval but then with backoff, we pretty quickly reach
- retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
- ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 67b62d5..8bd1d67 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -35,8 +35,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -58,6 +57,7 @@ import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.apache.kylin.tool.StorageCleanupJob;
@@ -431,10 +431,10 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
- String tableName = segment.getStorageLocationIdentifier();
- try (HTable table = new HTable(conf, tableName)) {
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
+ String tableName = segment.getStorageLocationIdentifier();
+
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 51479c8..6d3425e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,20 +46,20 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.6.0</hadoop2.version>
- <yarn.version>2.6.0</yarn.version>
+ <hadoop2.version>2.7.1</hadoop2.version>
+ <yarn.version>2.7.1</yarn.version>
<!-- Hive versions -->
- <hive.version>0.14.0</hive.version>
- <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
+ <hive.version>1.2.1</hive.version>
+ <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
<kafka.version>0.10.0.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.6.0</curator.version>
+ <curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index ea68855..8095bf8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
/**
*/
@@ -36,6 +36,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- HTableInterface getTable(String tableName) throws IOException;
+ Table getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index d9326f5..cc76b87 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
+ private Table mockedAclTable;
+ private Table mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index d0aa0ed..972eea9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements HTableInterface {
+public class MockHTable implements Table {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,14 +114,6 @@ public class MockHTable implements HTableInterface {
this.columnFamilies.add(columnFamily);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[] getTableName() {
- return tableName.getBytes();
- }
-
@Override
public TableName getName() {
return null;
@@ -200,8 +192,8 @@ public class MockHTable implements HTableInterface {
}
@Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- return new Boolean[0];
+ public boolean[] existsAll(List<Get> list) throws IOException {
+ return new boolean[0];
}
/**
@@ -306,15 +298,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- // FIXME: implement
- return null;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -446,7 +429,7 @@ public class MockHTable implements HTableInterface {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow(nkvs);
+ filter.filterRow();
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -535,6 +518,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -555,7 +543,7 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDeleteFamily()) {
+ if (kv.isDelete()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -592,6 +580,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -605,7 +598,7 @@ public class MockHTable implements HTableInterface {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, true);
+ return incrementColumnValue(row, family, qualifier, amount, null);
}
@Override
@@ -617,37 +610,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if (check(row, family, qualifier, null)) {
- Put put = new Put(row);
- put.add(family, qualifier, Bytes.toBytes(amount));
- put(put);
- return amount;
- }
- long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
- data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
- return newValue;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isAutoFlush() {
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void flushCommits() throws IOException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public void close() throws IOException {
}
@@ -673,29 +635,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public void setAutoFlush(boolean autoFlush) {
- throw new NotImplementedException();
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- throw new NotImplementedException();
-
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- throw new NotImplementedException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index 1d520c4..d1a1384 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,7 +21,8 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -58,11 +59,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index d693a67..3e3efec 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- HTableInterface htable = null;
+ Table htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,17 +226,16 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -250,7 +249,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -266,7 +265,6 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -284,7 +282,7 @@ public class AclService implements MutableAclService {
throw e;
}
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -295,17 +293,16 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 85c9284..c0f8e6f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -28,9 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.WeakHashMap;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -406,33 +404,24 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- HTable table = null;
+ Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- try {
- table = new HTable(hconf, tableName);
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
-
- regionCount = sizeMap.size();
-
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
- } finally {
- IOUtils.closeQuietly(table);
+ for (long s : sizeMap.values()) {
+ tableSize += s;
}
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 8810c85..2c031cf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -47,11 +47,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
@@ -161,14 +161,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -194,14 +193,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -213,12 +211,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- HTableInterface htable = null;
+ Table htable = null;
try {
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = conn.getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 07c7c6f..ab54882 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
- put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,14 +163,13 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -185,7 +184,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -216,7 +215,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- HTableInterface htable = null;
+ Table htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..b769391 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -64,7 +64,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
+ private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -75,7 +75,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (HConnection conn : connPool.values()) {
+ for (Connection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -144,7 +144,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -213,9 +213,9 @@ public class HBaseConnection {
// ============================================================================
- // returned HConnection can be shared by multiple threads and does not require close()
+ // returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static HConnection get(String url) {
+ public static Connection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -223,13 +223,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- HConnection connection = connPool.get(url);
+ Connection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = HConnectionManager.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
connPool.put(url, connection);
}
@@ -248,8 +248,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static boolean tableExists(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -269,18 +269,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
+ public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
+ Admin hbase = conn.getAdmin();
+ TableName tableName = TableName.valueOf(table);
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(table, newFamilyDescriptor(family));
+ hbase.addColumn(tableName, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -333,8 +333,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void deleteTable(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (!tableExists(conn, tableName)) {
@@ -344,10 +344,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(tableName)) {
- hbase.disableTable(tableName);
+ if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
+ hbase.disableTable(TableName.valueOf(tableName));
}
- hbase.deleteTable(tableName);
+ hbase.deleteTable(TableName.valueOf(tableName));
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 1d19983..714a265 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -31,14 +31,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +70,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- HConnection getConnection() throws IOException {
+ Connection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -120,7 +121,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,13 +239,12 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -252,7 +252,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -265,8 +265,6 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
- table.flushCommits();
-
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -275,7 +273,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -288,7 +286,6 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
- table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -310,7 +307,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -318,7 +315,7 @@ public class HBaseResourceStore extends ResourceStore {
}
}
- private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -337,7 +334,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
@@ -363,7 +360,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -371,8 +368,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.addColumn(B_FAMILY, B_COLUMN, content);
+ put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index 43b65cb..d36d722 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -18,7 +18,6 @@
package org.apache.kylin.storage.hbase;
-import com.google.common.base.Preconditions;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
@@ -36,6 +35,8 @@ import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition;
+import com.google.common.base.Preconditions;
+
@SuppressWarnings("unused")
//used by reflection
public class HBaseStorage implements IStorage {
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b141190..f63d9c2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,12 +26,13 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -86,14 +87,13 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final HTableInterface table;
+ final BufferedMutator table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getTable(htableName);
- table.setAutoFlush(false, true);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getBufferedMutator(htableName);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.put(put);
+ table.mutate(put);
}
@Override
public void close() throws IOException {
- table.flushCommits();
+ table.flush();
table.close();
}
}
class Reader implements IGTScanner {
- final HTableInterface table;
+ final Table table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index d99f80e..f879e2b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,8 +26,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -50,10 +51,10 @@ import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,7 +117,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -171,7 +172,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final boolean[] abnormalFinish = new boolean[1];
try {
- HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
+ Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 3cefc5f..a52af90 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -24,11 +24,12 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -154,8 +155,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index da9c932..f0949da 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -142,7 +142,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -179,7 +179,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 2814ad6..feb4842 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -26,7 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -79,7 +80,8 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin admin = conn.getAdmin();
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -92,7 +94,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(tableName)) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -101,7 +103,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
IOUtils.closeQuietly(admin);
@@ -110,8 +112,7 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -126,8 +127,7 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 46a828e..2d1c03a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -29,9 +29,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.exception.ExecuteException;
@@ -100,19 +101,21 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index d5b36df..6587d4e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -49,7 +49,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final HTableInterface hTable;
+ private final Table hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -58,7 +58,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -117,7 +117,6 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
- hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
+ hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -448,11 +450,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -472,7 +474,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -503,8 +504,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -538,13 +539,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -561,7 +561,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/eaf0cc58/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 8bd4abf..20d0f7d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -61,7 +61,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -130,9 +130,8 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
-
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -189,10 +188,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");