You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/01/19 08:11:32 UTC
[1/3] kylin git commit: enable filter in ResourceTool
Repository: kylin
Updated Branches:
refs/heads/2.x-staging 573c84e61 -> 000d883b1
enable filter in ResourceTool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a7923b96
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a7923b96
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a7923b96
Branch: refs/heads/2.x-staging
Commit: a7923b9698c1ca0e9677759744ef8a64996b1aaa
Parents: 573c84e
Author: honma <ho...@ebay.com>
Authored: Tue Jan 19 15:05:38 2016 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jan 19 15:05:38 2016 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/ResourceTool.java | 31 ++++++++++++++------
1 file changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7923b96/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index c3038dd..4a23ba3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -26,6 +26,7 @@ import org.apache.kylin.common.util.StringUtil;
public class ResourceTool {
+ private static String[] includes = null;
private static String[] excludes = null;
public static void main(String[] args) throws IOException {
@@ -38,6 +39,10 @@ public class ResourceTool {
return;
}
+ String include = System.getProperty("include");
+ if (include != null) {
+ includes = include.split("\\s*,\\s*");
+ }
String exclude = System.getProperty("exclude");
if (exclude != null) {
excludes = exclude.split("\\s*,\\s*");
@@ -83,7 +88,7 @@ public class ResourceTool {
// case of resource (not a folder)
if (children == null) {
- if (matchExclude(path) == false) {
+ if (matchFilter(path)) {
RawResource res = src.getResource(path);
if (res != null) {
dst.putResource(path, res.inputStream, res.timestamp);
@@ -100,14 +105,22 @@ public class ResourceTool {
}
}
- private static boolean matchExclude(String path) {
- if (excludes == null)
- return false;
- for (String exclude : excludes) {
- if (path.startsWith(exclude))
- return true;
+ private static boolean matchFilter(String path) {
+ if (includes != null) {
+ boolean in = false;
+ for (String include : includes) {
+ in = in || path.startsWith(include);
+ }
+ if (!in)
+ return false;
+ }
+ if (excludes != null) {
+ for (String exclude : excludes) {
+ if (path.startsWith(exclude))
+ return false;
+ }
}
- return false;
+ return true;
}
public static void reset(KylinConfig config) throws IOException {
@@ -118,7 +131,7 @@ public class ResourceTool {
private static void resetR(ResourceStore store, String path) throws IOException {
ArrayList<String> children = store.listResources(path);
if (children == null) { // path is a resource (not a folder)
- if (matchExclude(path) == false) {
+ if (matchFilter(path)) {
store.deleteResource(path);
}
} else {
[3/3] kylin git commit: fix issues in upgrading 2.0
Posted by ma...@apache.org.
fix issues in upgrading 2.0
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/000d883b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/000d883b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/000d883b
Branch: refs/heads/2.x-staging
Commit: 000d883b189fb01522e78ee2017ca6f3783cff1f
Parents: bd5fc00
Author: honma <ho...@ebay.com>
Authored: Tue Jan 19 15:10:41 2016 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jan 19 15:10:41 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/query/routing/RoutingRule.java | 3 +++
.../org/apache/kylin/storage/hbase/HBaseStorage.java | 6 +++++-
.../storage/hbase/cube/v2/CubeHBaseEndpointRPC.java | 14 +++++++-------
3 files changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/000d883b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
index 3d15a51..11498d6 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
@@ -56,6 +56,9 @@ public abstract class RoutingRule {
for (Candidate candidate : candidates) {
IRealization r = candidate.realization;
sb.append(r.getName());
+ sb.append("(");
+ sb.append(r.getType());
+ sb.append(")");
sb.append(",");
}
if (sb.charAt(sb.length() - 1) != '[')
http://git-wip-us.apache.org/repos/asf/kylin/blob/000d883b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index c61212c..e71434f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -26,6 +26,7 @@ import org.apache.kylin.engine.mr.IMROutput2;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
@@ -71,8 +72,11 @@ public class HBaseStorage implements IStorage {
}
} else if (realization.getType() == RealizationType.CUBE) {
+ CubeInstance cubeInstance = (CubeInstance) realization;
String cubeStorageQuery;
- if (overwriteStorageQuery != null) {
+ if (cubeInstance.getStorageType() == IStorageAware.ID_HBASE) {//v2 query engine cannot go with v1 storage now
+ cubeStorageQuery = v1CubeStorageQuery;
+ } else if (overwriteStorageQuery != null) {
cubeStorageQuery = overwriteStorageQuery;
} else if ("v1".equalsIgnoreCase(BackdoorToggles.getHbaseCubeQueryVersion())) {
cubeStorageQuery = v1CubeStorageQuery;
http://git-wip-us.apache.org/repos/asf/kylin/blob/000d883b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 5cca195..d6ef16c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -255,11 +255,11 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(scanRequests.size() * shardNum);
for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
- for (int i = 0; i < scanRequests.size(); ++i) {
- final int scanIndex = i;
- executorService.submit(new Runnable() {
- @Override
- public void run() {
+ executorService.submit(new Runnable() {
+ @Override
+ public void run() {
+ for (int i = 0; i < scanRequests.size(); ++i) {
+ int scanIndex = i;
CubeVisitProtos.CubeVisitRequest.Builder builder = CubeVisitProtos.CubeVisitRequest.newBuilder();
builder.setGtScanRequest(scanRequestByteStrings.get(scanIndex)).setHbaseRawScan(rawScanByteStrings.get(scanIndex));
for (IntList intList : hbaseColumnsToGTIntList) {
@@ -285,8 +285,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
}
}
}
- });
- }
+ }
+ });
}
return new EndpointResultsAsGTScanner(fullGTInfo, epResultItr, scanRequests.get(0).getColumns(), totalScannedCount.get());
[2/3] kylin git commit: print error stack when gtinfo construction
for a segment fails
Posted by ma...@apache.org.
print error stack when gtinfo construction for a segment fails
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bd5fc005
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bd5fc005
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bd5fc005
Branch: refs/heads/2.x-staging
Commit: bd5fc005a86f2f875fd8a7c7a67db66c54bbd1cd
Parents: a7923b9
Author: honma <ho...@ebay.com>
Authored: Mon Jan 18 14:07:55 2016 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jan 19 15:05:53 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bd5fc005/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 9f0ff2b..a61e5c3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -101,6 +101,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
} catch (NotEnoughGTInfoException e) {
//deal with empty cube segment
logger.info("Cannot construct Segment {}'s GTInfo, this may due to empty segment or broken metadata", cubeSeg);
+ logger.info("error stack", e);
continue;
}
scanners.add(scanner);