You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by qh...@apache.org on 2015/07/13 08:57:23 UTC
[1/2] incubator-kylin git commit: KYLIN-882 minor,
fix invalid file location
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 988f13691 -> 2a29f153a
KYLIN-882 minor, fix invalid file location
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e2e48e22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e2e48e22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e2e48e22
Branch: refs/heads/0.8
Commit: e2e48e229d2cea12295a2af2a25e4ad63253e4ab
Parents: 988f136
Author: qianhao.zhou <qi...@ebay.com>
Authored: Mon Jul 13 14:47:34 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Mon Jul 13 14:54:39 2015 +0800
----------------------------------------------------------------------
bin/check-env.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e2e48e22/bin/check-env.sh
----------------------------------------------------------------------
diff --git a/bin/check-env.sh b/bin/check-env.sh
index dd0d27b..d0f6248 100644
--- a/bin/check-env.sh
+++ b/bin/check-env.sh
@@ -27,7 +27,7 @@ then
exit 1
fi
-WORKING_DIR=`sh bin/get-properties.sh kylin.hdfs.working.dir`
+WORKING_DIR=`sh $KYLIN_HOME/bin/get-properties.sh kylin.hdfs.working.dir`
hadoop fs -mkdir -p $WORKING_DIR
if [ $? != 0 ]
then
[2/2] incubator-kylin git commit: refactor
Posted by qh...@apache.org.
refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2a29f153
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2a29f153
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2a29f153
Branch: refs/heads/0.8
Commit: 2a29f153a71347ff14d3d61768a790c8bf0dd77f
Parents: e2e48e2
Author: qianhao.zhou <qi...@ebay.com>
Authored: Thu Jul 9 16:20:10 2015 +0800
Committer: qianhao.zhou <qi...@ebay.com>
Committed: Mon Jul 13 14:54:40 2015 +0800
----------------------------------------------------------------------
.../common/util/MemoryBudgetController.java | 10 ++---
.../job/inmemcubing/DoggedCubeBuilder.java | 3 +-
.../kylin/job/inmemcubing/InMemCubeBuilder.java | 47 +++++++-------------
3 files changed, 23 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2a29f153/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java b/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
index b4394ae..19302bf 100644
--- a/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
+++ b/common/src/main/java/org/apache/kylin/common/util/MemoryBudgetController.java
@@ -17,6 +17,7 @@
package org.apache.kylin.common.util;
+import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -27,7 +28,7 @@ public class MemoryBudgetController {
private static final boolean debug = true;
- public static interface MemoryConsumer {
+ public interface MemoryConsumer {
// return number MB released
int freeUp(int mb);
}
@@ -65,11 +66,8 @@ public class MemoryBudgetController {
private final ReentrantLock lock = new ReentrantLock();
public MemoryBudgetController(int totalBudgetMB) {
- if (totalBudgetMB < 0)
- throw new IllegalArgumentException();
- if (totalBudgetMB > getSystemAvailMB())
- throw new IllegalStateException();
-
+ Preconditions.checkArgument(totalBudgetMB >= 0);
+ Preconditions.checkState(totalBudgetMB <= getSystemAvailMB());
this.totalBudgetMB = totalBudgetMB;
this.totalReservedMB = 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2a29f153/job/src/main/java/org/apache/kylin/job/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/inmemcubing/DoggedCubeBuilder.java b/job/src/main/java/org/apache/kylin/job/inmemcubing/DoggedCubeBuilder.java
index f635af2..ce6541f 100644
--- a/job/src/main/java/org/apache/kylin/job/inmemcubing/DoggedCubeBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/inmemcubing/DoggedCubeBuilder.java
@@ -37,6 +37,7 @@ import java.io.IOException;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.TimeUnit;
/**
@@ -246,7 +247,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
final BlockingQueue<List<String>> inputQueue = new ArrayBlockingQueue<List<String>>(16);
final InMemCubeBuilder builder;
- TreeMap<Long, CuboidResult> buildResult;
+ ConcurrentNavigableMap<Long, CuboidResult> buildResult;
long inputRowCount = 0;
RuntimeException exception;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2a29f153/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java b/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
index 761199c..b0e4402 100644
--- a/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/inmemcubing/InMemCubeBuilder.java
@@ -42,7 +42,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
-import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
@@ -67,7 +67,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
private MemoryBudgetController memBudget;
private Thread[] taskThreads;
private Throwable[] taskThreadExceptions;
- private TreeSet<CuboidTask> taskPending;
+ private LinkedBlockingQueue<CuboidTask> taskPending;
private AtomicInteger taskCuboidCompleted = new AtomicInteger(0);
private CuboidResult baseResult;
@@ -136,29 +136,26 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
@Override
public void build(BlockingQueue<List<String>> input, ICuboidWriter output) throws IOException {
- TreeMap<Long, CuboidResult> result = build(input);
+ ConcurrentNavigableMap<Long, CuboidResult> result = build(input);
for (CuboidResult cuboidResult : result.values()) {
outputCuboid(cuboidResult.cuboidId, cuboidResult.table, output);
cuboidResult.table.close();
}
}
- TreeMap<Long, CuboidResult> build(BlockingQueue<List<String>> input) throws IOException {
- final TreeMap<Long, CuboidResult> result = new TreeMap<Long, CuboidResult>();
- ICuboidCollector collector = new ICuboidCollector() {
+ ConcurrentNavigableMap<Long, CuboidResult> build(BlockingQueue<List<String>> input) throws IOException {
+ final ConcurrentNavigableMap<Long, CuboidResult> result = new ConcurrentSkipListMap<Long, CuboidResult>();
+ build(input, new ICuboidCollector() {
@Override
public void collect(CuboidResult cuboidResult) {
- synchronized (result) {
- result.put(cuboidResult.cuboidId, cuboidResult);
- }
+ result.put(cuboidResult.cuboidId, cuboidResult);
}
- };
- build(input, collector);
+ });
return result;
}
- static interface ICuboidCollector {
- public void collect(CuboidResult result);
+ interface ICuboidCollector {
+ void collect(CuboidResult result);
}
static class CuboidResult {
@@ -182,7 +179,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
logger.info("In Mem Cube Build start, " + cubeDesc.getName());
// multiple threads to compute cuboid in parallel
- taskPending = new TreeSet<CuboidTask>();
+ taskPending = new LinkedBlockingQueue<>();
taskCuboidCompleted.set(0);
taskThreads = prepareTaskThreads();
taskThreadExceptions = new Throwable[taskThreadCount];
@@ -280,14 +277,9 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
try {
while (!isAllCuboidDone()) {
CuboidTask task = null;
- synchronized (taskPending) {
- while (task == null && taskHasNoException()) {
- task = taskPending.pollFirst();
- if (task == null)
- taskPending.wait(60000);
- }
+ while (task == null && taskHasNoException()) {
+ task = taskPending.poll(15, TimeUnit.SECONDS);
}
-
// if task error occurs
if (task == null)
break;
@@ -320,13 +312,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
private void addChildTasks(CuboidResult parent) {
List<Long> children = cuboidScheduler.getSpanningCuboid(parent.cuboidId);
- if (!children.isEmpty()) {
- synchronized (taskPending) {
- for (Long child : children) {
- taskPending.add(new CuboidTask(parent, child));
- }
- taskPending.notifyAll();
- }
+ for (Long child : children) {
+ taskPending.add(new CuboidTask(parent, child));
}
}
@@ -511,8 +498,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
// ===========================================================================
private static class CuboidTask implements Comparable<CuboidTask> {
- CuboidResult parent;
- long childCuboidId;
+ final CuboidResult parent;
+ final long childCuboidId;
CuboidTask(CuboidResult parent, long childCuboidId) {
this.parent = parent;