You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/10 13:12:50 UTC
kylin git commit: temp, more log to diag
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2497-2 7104c1f74 -> 47c67f0ab
temp, more log to diag
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/47c67f0a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/47c67f0a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/47c67f0a
Branch: refs/heads/KYLIN-2497-2
Commit: 47c67f0abdf15e3a50731d93ea4c7c3edb15d6e4
Parents: 7104c1f
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Mar 10 21:12:54 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 10 21:12:54 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/KylinReducer.java | 2 +-
.../mr/steps/FactDistinctColumnsMapper.java | 5 ++---
.../engine/mr/steps/SelfDefineSortableKey.java | 20 ++++++++------------
3 files changed, 11 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/47c67f0a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
index d428757..e47af9c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/KylinReducer.java
@@ -42,7 +42,7 @@ public class KylinReducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends Reducer<KEYI
final public void reduce(KEYIN key, Iterable<VALUEIN> values, Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
try {
if (reduceCounter++ % BatchConstants.NORMAL_RECORD_LOG_THRESHOLD == 0) {
- logger.info("Accepting Mapper Key with ordinal: " + reduceCounter);
+ logger.info("Accepting Reducer Key with ordinal: " + reduceCounter);
}
doReduce(key, values, context);
http://git-wip-us.apache.org/repos/asf/kylin/blob/47c67f0a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 9f65163..8ec7c50 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -41,7 +41,6 @@ import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
-
/**
*/
public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperBase<KEYIN, Object> {
@@ -52,7 +51,6 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
BYTES
}
-
protected boolean collectStatistics = false;
protected CuboidScheduler cuboidScheduler = null;
protected int nRowKey;
@@ -166,6 +164,8 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
tmpbuf.put(valueBytes);
outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
DataType type = factDictCols.get(i).getType();
+ logger.info("col: {}, pos: {} ,value {}, type {} ", factDictCols.get(i), dictionaryColumnIndex[i], fieldValue, type);
+
sortableKey.init(outputKey, type);
//judge type
context.write(sortableKey, EMPTY_TEXT);
@@ -256,7 +256,6 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
}
}
-
private int countNewSize(int oldSize, int dataSize) {
int newSize = oldSize * 2;
while (newSize < dataSize) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/47c67f0a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
index 85b554d..07ad850 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
@@ -24,13 +24,15 @@ import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.kylin.metadata.datatype.DataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class SelfDefineSortableKey implements WritableComparable<SelfDefineSortableKey> {
+ private static final Logger logger = LoggerFactory.getLogger(SelfDefineSortableKey.class);
+
public enum TypeFlag {
- NONE_NUMERIC_TYPE,
- INTEGER_FAMILY_TYPE,
- DOUBLE_FAMILY_TYPE
+ NONE_NUMERIC_TYPE, INTEGER_FAMILY_TYPE, DOUBLE_FAMILY_TYPE
}
private byte typeId; //non-numeric(0000 0000) int(0000 0001) other numberic(0000 0010)
@@ -42,10 +44,6 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
public SelfDefineSortableKey() {
}
- public SelfDefineSortableKey(Text key, DataType type) {
- init(key, getTypeIdByDatatype(type));
- }
-
public void init(Text key, byte typeId) {
this.typeId = typeId;
this.rawKey = key;
@@ -62,9 +60,10 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
}
}
-
public void init(Text key, DataType type) {
- init(key, getTypeIdByDatatype(type));
+ byte typeIdByDatatype = getTypeIdByDatatype(type);
+ logger.info("type is {}, byte is {} ", type, typeIdByDatatype);
+ init(key, typeIdByDatatype);
}
@Override
@@ -114,7 +113,6 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
return (typeId == TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
}
-
public byte getTypeIdByDatatype(DataType type) {
if (!type.isNumberFamily()) {
return (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal();
@@ -130,5 +128,3 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
}
}
-
-