You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/08 16:15:45 UTC
[07/18] kylin git commit: KYLIN-2167 FactDistinctColumnsReducer may
get wrong max/min partition col value
KYLIN-2167 FactDistinctColumnsReducer may get wrong max/min partition col value
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7a20a06
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7a20a06
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7a20a06
Branch: refs/heads/KYLIN-2006
Commit: e7a20a063a3f007aa2a0ed5f39c616880ba46118
Parents: 4d9a923
Author: shaofengshi <sh...@apache.org>
Authored: Tue Nov 8 14:23:11 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Nov 8 21:29:22 2016 +0800
----------------------------------------------------------------------
.../mr/steps/FactDistinctColumnsReducer.java | 8 ----
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 40 +++++++++-----------
2 files changed, 18 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7a20a06/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 5b00381..b09e614 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -152,14 +152,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWri
cuboidHLLMap.put(cuboidId, hll);
}
}
- } else if (isPartitionCol == true) {
- // for partition col min/max value
- ByteArray value = new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1));
- if (colValues.size() > 1) {
- colValues.set(1, value);
- } else {
- colValues.add(value);
- }
} else {
colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1)));
if (colValues.size() == 1000000) { //spill every 1 million
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7a20a06/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index d285799..4d71f5d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -19,6 +19,7 @@
package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
+import java.text.ParseException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
@@ -82,23 +83,8 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
private void updateTimeRange(CubeSegment segment) throws IOException {
final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
- final String factDistinctPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
- final ReadableTable readableTable = new DFSFileTable(factDistinctPath + "/" + partitionCol.getName(), -1);
- final ReadableTable.TableReader tableReader = readableTable.getReader();
- String minValue = null, maxValue = null;
- try {
- while (tableReader.next()) {
- if (minValue == null) {
- minValue = tableReader.getRow()[0];
- }
- maxValue = tableReader.getRow()[0];
- }
- } finally {
- IOUtils.closeQuietly(tableReader);
- }
-
final DataType partitionColType = partitionCol.getType();
- FastDateFormat dateFormat;
+ final FastDateFormat dateFormat;
if (partitionColType.isDate()) {
dateFormat = DateFormat.getDateFormat(DateFormat.DEFAULT_DATE_PATTERN);
} else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) {
@@ -113,14 +99,24 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
throw new IllegalStateException("Type " + partitionColType + " is not valid partition column type");
}
+ final String factDistinctPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
+ final ReadableTable readableTable = new DFSFileTable(factDistinctPath + "/" + partitionCol.getName(), -1);
+ final ReadableTable.TableReader tableReader = readableTable.getReader();
+ long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
try {
- long startTime = dateFormat.parse(minValue).getTime();
- long endTime = dateFormat.parse(maxValue).getTime();
- segment.setDateRangeStart(startTime);
- segment.setDateRangeEnd(endTime);
- } catch (Exception e) {
- throw new IllegalStateException(e);
+ while (tableReader.next()) {
+ long time = dateFormat.parse(tableReader.getRow()[0]).getTime();
+ minValue = Math.min(minValue, time);
+ maxValue = Math.max(maxValue, time);
+ }
+ } catch (ParseException e) {
+ throw new IOException(e);
+ } finally {
+ IOUtils.closeQuietly(tableReader);
}
+
+ segment.setDateRangeStart(minValue);
+ segment.setDateRangeEnd(maxValue);
}
}