You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2016/06/03 12:31:25 UTC
[1/3] kylin git commit: eBay-JIRA-524: automatical expansion when
buffer overflow
Repository: kylin
Updated Branches:
refs/heads/1.4-rc 0456db93d -> 871154447
eBay-JIRA-524: automatical expansion when buffer overflow
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d02a5f5d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d02a5f5d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d02a5f5d
Branch: refs/heads/1.4-rc
Commit: d02a5f5d97078094dbbe7a0a809a7bcc81a3676c
Parents: 0456db9
Author: kyotoYaho <nj...@apache.org>
Authored: Tue May 17 21:10:58 2016 +0800
Committer: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Committed: Fri May 27 10:15:18 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/dict/DictCodeSystem.java | 7 +++++
.../metadata/filter/ConstantTupleFilter.java | 30 ++++++++++++++++----
.../metadata/filter/TupleFilterSerializer.java | 24 ++++++++++++++--
3 files changed, 54 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d02a5f5d/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
index 83ad20c..5a2044e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictCodeSystem.java
@@ -22,6 +22,8 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.filter.IFilterCodeSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A simple code system where all values are dictionary IDs (fixed length bytes) encoded as ISO-8859-1 strings.
@@ -30,6 +32,8 @@ import org.apache.kylin.metadata.filter.IFilterCodeSystem;
*/
public class DictCodeSystem implements IFilterCodeSystem<String> {
+ private static final Logger logger = LoggerFactory.getLogger(DictCodeSystem.class);
+
public static final DictCodeSystem INSTANCE = new DictCodeSystem();
private DictCodeSystem() {
@@ -57,6 +61,9 @@ public class DictCodeSystem implements IFilterCodeSystem<String> {
//TODO: should use ISO-8859-1 rather than UTF8
@Override
public void serialize(String value, ByteBuffer buffer) {
+ if(value.length() > 1000 ){
+ logger.info("value length exceeds 1000: {}", value.substring(0,1000));
+ }
BytesUtil.writeUTFString(value, buffer);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d02a5f5d/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
index 074f163..a92c699 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ConstantTupleFilter.java
@@ -18,6 +18,7 @@
package org.apache.kylin.metadata.filter;
+import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Collections;
@@ -25,6 +26,8 @@ import java.util.HashSet;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
@@ -33,6 +36,8 @@ import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
*/
public class ConstantTupleFilter extends TupleFilter {
+ private static final Logger logger = LoggerFactory.getLogger(ConstantTupleFilter.class);
+
public static final ConstantTupleFilter FALSE = new ConstantTupleFilter();
public static final ConstantTupleFilter TRUE = new ConstantTupleFilter((Object) null); // not sure of underlying code system, null is the only value that applies to all types
@@ -89,11 +94,26 @@ public class ConstantTupleFilter extends TupleFilter {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public byte[] serialize(IFilterCodeSystem cs) {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
- int size = this.constantValues.size();
- BytesUtil.writeVInt(size, buffer);
- for (Object val : this.constantValues) {
- cs.serialize(val, buffer);
+ ByteBuffer buffer;
+ int bufferSize = BUFFER_SIZE;
+ while(true){
+ try{
+ buffer = ByteBuffer.allocate(bufferSize);
+ int size = this.constantValues.size();
+ BytesUtil.writeVInt(size, buffer);
+ logger.info("constantValues.size: {}", size);
+ for (Object val : this.constantValues) {
+ if(! (val instanceof String)){
+ logger.error("Problematic constant value:" + val);
+ }
+ cs.serialize(val, buffer);
+ logger.info("Buffer Position: {}", buffer.position());
+ }
+ break;
+ }catch (BufferOverflowException e) {
+ logger.info("Buffer size {} cannot hold the filter, resizing to 4 times", bufferSize);
+ bufferSize *= 4;
+ }
}
byte[] result = new byte[buffer.position()];
System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
http://git-wip-us.apache.org/repos/asf/kylin/blob/d02a5f5d/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index a394a51..628a97c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -18,12 +18,15 @@
package org.apache.kylin.metadata.filter;
+import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
import java.util.Stack;
import org.apache.kylin.common.util.BytesUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* http://eli.thegreenplace.net/2011/09/29/an-interesting-tree-serialization-algorithm-from-dwarf
@@ -33,11 +36,14 @@ import org.apache.kylin.common.util.BytesUtil;
*/
public class TupleFilterSerializer {
+ private static final Logger logger = LoggerFactory.getLogger(TupleFilterSerializer.class);
+
public interface Decorator {
TupleFilter onSerialize(TupleFilter filter);
}
private static final int BUFFER_SIZE = 65536;
+ private static final int BUFFER_SIZE_MAX = 268435456;
private static final Map<Integer, TupleFilter.FilterOperatorEnum> ID_OP_MAP = new HashMap<Integer, TupleFilter.FilterOperatorEnum>();
static {
@@ -51,8 +57,22 @@ public class TupleFilterSerializer {
}
public static byte[] serialize(TupleFilter rootFilter, Decorator decorator, IFilterCodeSystem<?> cs) {
- ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
- internalSerialize(rootFilter, decorator, buffer, cs);
+ ByteBuffer buffer;
+ int bufferSize = BUFFER_SIZE;
+ while (true) {
+ try {
+ buffer = ByteBuffer.allocate(bufferSize);
+ internalSerialize(rootFilter, decorator, buffer, cs);
+ break;
+ } catch (BufferOverflowException e) {
+ logger.info("Buffer size {} cannot hold the filter, resizing to 4 times", bufferSize);
+ bufferSize *= 4;
+ if(bufferSize > BUFFER_SIZE_MAX){
+ logger.error("Buffer size {} exceeds the limitation {}", bufferSize, BUFFER_SIZE_MAX);
+ throw e;
+ }
+ }
+ }
byte[] result = new byte[buffer.position()];
System.arraycopy(buffer.array(), 0, result, 0, buffer.position());
return result;
[3/3] kylin git commit: KYLIN-1761: Fetch part of metadata to local
Posted by nj...@apache.org.
KYLIN-1761: Fetch part of metadata to local
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87115444
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87115444
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87115444
Branch: refs/heads/1.4-rc
Commit: 87115444731814f3fa1db1f70caa04e54b28b352
Parents: b220886
Author: kyotoYaho <nj...@apache.org>
Authored: Fri Jun 3 19:41:10 2016 +0800
Committer: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Committed: Fri Jun 3 19:59:48 2016 +0800
----------------------------------------------------------------------
build/bin/metastore.sh | 14 ++++++++++++++
.../apache/kylin/common/persistence/ResourceTool.java | 10 ++++++++++
.../storage/hbase/steps/SandboxMetastoreCLI.java | 2 ++
3 files changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/87115444/build/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 8908351..8d85661 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -40,6 +40,19 @@ then
${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool download ${_file}
echo "metadata store backed up to ${_file}"
+elif [ "$1" == "fetch" ]
+then
+
+ _file=$2
+
+ _now=$(date +"%Y_%m_%d_%H_%M_%S")
+ _fileDst="${KYLIN_HOME}/meta_backups/meta_${_now}"
+ echo "Starting restoring $_fileDst"
+ mkdir -p $_fileDst
+
+ ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.common.persistence.ResourceTool fetch $_fileDst $_file
+ echo "metadata store backed up to $_fileDst"
+
elif [ "$1" == "restore" ]
then
@@ -80,6 +93,7 @@ then
else
echo "usage: metastore.sh backup"
+ echo " metastore.sh fetch DATA"
echo " metastore.sh reset"
echo " metastore.sh restore PATH_TO_LOCAL_META"
echo " metastore.sh list RESOURCE_PATH"
http://git-wip-us.apache.org/repos/asf/kylin/blob/87115444/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 0975d57..187601a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -65,6 +65,9 @@ public class ResourceTool {
case "download":
copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]));
break;
+ case "fetch":
+ copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]), args[2]);
+ break;
case "upload":
copy(KylinConfig.createInstanceFromUri(args[1]), KylinConfig.getInstanceFromEnv());
break;
@@ -82,6 +85,13 @@ public class ResourceTool {
System.out.println("" + result);
}
+ public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, String path) throws IOException {
+ ResourceStore src = ResourceStore.getStore(srcConfig);
+ ResourceStore dst = ResourceStore.getStore(dstConfig);
+
+ copyR(src, dst, path);
+ }
+
public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws IOException {
ResourceStore src = ResourceStore.getStore(srcConfig);
ResourceStore dst = ResourceStore.getStore(dstConfig);
http://git-wip-us.apache.org/repos/asf/kylin/blob/87115444/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
index 7b3b698..dbdb730 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
@@ -54,6 +54,8 @@ public class SandboxMetastoreCLI {
if ("download".equalsIgnoreCase(args[0])) {
ResourceTool.main(new String[] { "download", args[1] });
+ } else if ("fetch".equalsIgnoreCase(args[0])) {
+ ResourceTool.main(new String[] { "fetch", args[1], args[2]});
} else if ("upload".equalsIgnoreCase(args[0])) {
ResourceTool.main(new String[] { "upload", args[1] });
} else {
[2/3] kylin git commit: KYLIN-1189: Automatically resume running jobs
when job engine failover
Posted by nj...@apache.org.
KYLIN-1189: Automatically resume running jobs when job engine failover
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b2208860
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b2208860
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b2208860
Branch: refs/heads/1.4-rc
Commit: b2208860fc4855018931fb686233dd7ca4388a54
Parents: d02a5f5
Author: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Authored: Mon May 30 11:27:44 2016 +0800
Committer: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Committed: Mon May 30 11:27:44 2016 +0800
----------------------------------------------------------------------
.../kylin/job/impl/threadpool/DefaultScheduler.java | 2 +-
.../apache/kylin/job/manager/ExecutableManager.java | 15 +++++++++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2208860/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 417e279..4abaa23 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -175,7 +175,7 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
executableManager.updateJobOutput(executable.getId(), ExecutableState.ERROR, null, "scheduler initializing work to reset job to ERROR status");
}
}
- executableManager.updateAllRunningJobsToError();
+ executableManager.resumeAllRunningJobs();
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2208860/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
index ff92a50..244c295 100644
--- a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
@@ -190,6 +190,21 @@ public class ExecutableManager {
}
}
+ public void resumeAllRunningJobs() {
+ try {
+ final List<ExecutableOutputPO> jobOutputs = executableDao.getJobOutputs();
+ for (ExecutableOutputPO executableOutputPO : jobOutputs) {
+ if (executableOutputPO.getStatus().equalsIgnoreCase(ExecutableState.RUNNING.toString())) {
+ executableOutputPO.setStatus(ExecutableState.READY.toString());
+ executableDao.updateJobOutput(executableOutputPO);
+ }
+ }
+ } catch (PersistentException e) {
+ logger.error("error reset job status from RUNNING to READY", e);
+ throw new RuntimeException(e);
+ }
+ }
+
public void resumeJob(String jobId) {
AbstractExecutable job = getJob(jobId);
if (job == null) {