You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/09/03 12:45:55 UTC
[01/50] [abbrv] kylin git commit: KYLIN-1908 refactor,
extract QueryMetricsFacade [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/1.5.x-CDH5.7 b46029fba -> c7d021661 (forced update)
KYLIN-1908 refactor, extract QueryMetricsFacade
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b34c38b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b34c38b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b34c38b
Branch: refs/heads/1.5.x-CDH5.7
Commit: 1b34c38b2a9ff65950dbac8229af5749ad8403ed
Parents: 0188a26
Author: Yang Li <li...@apache.org>
Authored: Sun Aug 21 19:38:11 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Aug 21 19:38:11 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +
.../common/KylinConfigCannotInitException.java | 1 +
.../kylin/rest/controller/QueryController.java | 9 +-
.../kylin/rest/init/InitialTaskManager.java | 8 +-
.../kylin/rest/metrics/QueryMetricsFacade.java | 108 +++++++++++++++++++
.../kylin/rest/util/QueryMetricsUtil.java | 94 ----------------
.../kylin/rest/metrics/QueryMetricsTest.java | 20 ++--
7 files changed, 128 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 50205fb..de6b977 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -570,6 +570,10 @@ abstract public class KylinConfigBase implements Serializable {
return Long.valueOf(this.getOptional("kylin.query.sequence.expire.time", "86400000"));//default a day
}
+ public boolean getQueryMetricsEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.metrics.enabled", "false"));
+ }
+
public int[] getQueryMetricsPercentilesIntervals() {
String[] dft = { "60", "300", "3600" };
return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java
index 7113872..a4bbeae 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java
@@ -18,6 +18,7 @@
package org.apache.kylin.common;
+@SuppressWarnings("serial")
public class KylinConfigCannotInitException extends RuntimeException {
public KylinConfigCannotInitException(String message) {
super(message);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 6f8a7e1..e7847c7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletResponse;
@@ -33,7 +32,7 @@ import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.metrics.QueryMetrics;
+import org.apache.kylin.rest.metrics.QueryMetricsFacade;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.model.SelectedColumnMeta;
import org.apache.kylin.rest.model.TableMeta;
@@ -43,7 +42,6 @@ import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.request.SaveSqlRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.QueryService;
-import org.apache.kylin.rest.util.QueryMetricsUtil;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.storage.exception.ScanOutOfLimitException;
import org.slf4j.Logger;
@@ -77,9 +75,6 @@ public class QueryController extends BasicController {
private static final Logger logger = LoggerFactory.getLogger(QueryController.class);
- private ConcurrentHashMap<String, QueryMetrics> metricsMap =
- new ConcurrentHashMap<String, QueryMetrics>();
-
public static final String SUCCESS_QUERY_CACHE = "StorageCache";
public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";
@@ -225,7 +220,7 @@ public class QueryController extends BasicController {
queryService.logQuery(sqlRequest, sqlResponse);
- QueryMetricsUtil.updateMetrics(sqlRequest, sqlResponse, metricsMap);
+ QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
if (sqlResponse.getIsException())
throw new InternalErrorException(sqlResponse.getExceptionMessage());
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
index 430d8a4..5cf48cb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
@@ -19,8 +19,8 @@
package org.apache.kylin.rest.init;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.rest.metrics.QueryMetricsFacade;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
@@ -37,12 +37,12 @@ public class InitialTaskManager implements InitializingBean {
logger.info("Kylin service is starting.....");
runInitialTasks();
-
- //init mrtrics system for kylin
- DefaultMetricsSystem.initialize("Kylin");
}
private void runInitialTasks() {
+ // init metrics system for kylin
+ QueryMetricsFacade.init();
+
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
String initTasks = kylinConfig.getInitTasks();
if (!StringUtils.isEmpty(initTasks)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
new file mode 100644
index 0000000..91aff8b
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.metrics;
+
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.SQLResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.ThreadSafe;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * The entrance of metrics features.
+ */
+@ThreadSafe
+public class QueryMetricsFacade {
+
+ private static final Logger logger = LoggerFactory.getLogger(QueryMetricsFacade.class);
+
+ private static boolean enabled = false;
+ private static ConcurrentHashMap<String, QueryMetrics> metricsMap = new ConcurrentHashMap<String, QueryMetrics>();
+
+ public static void init() {
+ enabled = KylinConfig.getInstanceFromEnv().getQueryMetricsEnabled();
+ if (!enabled)
+ return;
+
+ DefaultMetricsSystem.initialize("Kylin");
+ }
+
+ public static void updateMetrics(SQLRequest sqlRequest, SQLResponse sqlResponse) {
+ if (!enabled)
+ return;
+
+ String projectName = sqlRequest.getProject();
+ String cubeName = sqlResponse.getCube();
+
+ update(getQueryMetrics("Server_Total", metricsMap), sqlResponse);
+
+ update(getQueryMetrics(projectName, metricsMap), sqlResponse);
+
+ String cubeMetricName = projectName + ",sub=" + cubeName;
+ update(getQueryMetrics(cubeMetricName, metricsMap), sqlResponse);
+ }
+
+ private static void update(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
+ try {
+ incrQueryCount(queryMetrics, sqlResponse);
+ incrCacheHitCount(queryMetrics, sqlResponse);
+
+ if (!sqlResponse.getIsException()) {
+ queryMetrics.addQueryLatency(sqlResponse.getDuration());
+ queryMetrics.addScanRowCount(sqlResponse.getTotalScanCount());
+ queryMetrics.addResultRowCount(sqlResponse.getResults().size());
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage());
+ }
+
+ }
+
+ private static void incrQueryCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
+ if (!sqlResponse.isHitExceptionCache() && !sqlResponse.getIsException()) {
+ queryMetrics.incrQuerySuccessCount();
+ } else {
+ queryMetrics.incrQueryFailCount();
+ }
+ queryMetrics.incrQueryCount();
+ }
+
+ private static void incrCacheHitCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
+ if (sqlResponse.isStorageCacheUsed()) {
+ queryMetrics.addCacheHitCount(1);
+ }
+ }
+
+ private static QueryMetrics getQueryMetrics(String name, ConcurrentHashMap<String, QueryMetrics> metricsMap) {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ int[] intervals = config.getQueryMetricsPercentilesIntervals();
+
+ if (metricsMap.containsKey(name)) {
+ return metricsMap.get(name);
+ } else {
+ QueryMetrics queryMetrics = new QueryMetrics(intervals).registerWith(name);
+ metricsMap.put(name, queryMetrics);
+ return queryMetrics;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java
deleted file mode 100644
index 0c1c780..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.util;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.rest.metrics.QueryMetrics;
-import org.apache.kylin.rest.request.SQLRequest;
-import org.apache.kylin.rest.response.SQLResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.concurrent.ThreadSafe;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * update metrics about query
- */
-@ThreadSafe
-public class QueryMetricsUtil {
-
- private static final Logger logger = LoggerFactory.getLogger(QueryMetricsUtil.class);
-
- public static void updateMetrics(SQLRequest sqlRequest, SQLResponse sqlResponse, ConcurrentHashMap<String, QueryMetrics> metricsMap) {
- String projectName = sqlRequest.getProject();
- String cubeName = sqlResponse.getCube();
-
- update(getQueryMetrics("Server_Total", metricsMap), sqlResponse);
-
- update(getQueryMetrics(projectName, metricsMap), sqlResponse);
-
- String cubeMetricName = projectName + ",sub=" + cubeName;
- update(getQueryMetrics(cubeMetricName, metricsMap), sqlResponse);
- }
-
- private static void update(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
- try {
- incrQueryCount(queryMetrics, sqlResponse);
- incrCacheHitCount(queryMetrics, sqlResponse);
-
- if (!sqlResponse.getIsException()) {
- queryMetrics.addQueryLatency(sqlResponse.getDuration());
- queryMetrics.addScanRowCount(sqlResponse.getTotalScanCount());
- queryMetrics.addResultRowCount(sqlResponse.getResults().size());
- }
- } catch (Exception e) {
- logger.error(e.getMessage());
- }
-
- }
-
- private static void incrQueryCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
- if (!sqlResponse.isHitExceptionCache() && !sqlResponse.getIsException()) {
- queryMetrics.incrQuerySuccessCount();
- } else {
- queryMetrics.incrQueryFailCount();
- }
- queryMetrics.incrQueryCount();
- }
-
- private static void incrCacheHitCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
- if (sqlResponse.isStorageCacheUsed()) {
- queryMetrics.addCacheHitCount(1);
- }
- }
-
- private static QueryMetrics getQueryMetrics(String name, ConcurrentHashMap<String, QueryMetrics> metricsMap) {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- int[] intervals = config.getQueryMetricsPercentilesIntervals();
-
- if (metricsMap.containsKey(name)) {
- return metricsMap.get(name);
- } else {
- QueryMetrics queryMetrics = new QueryMetrics(intervals).registerWith(name);
- metricsMap.put(name, queryMetrics);
- return queryMetrics;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
index cd7ff7a..d413f40 100644
--- a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
@@ -18,26 +18,24 @@
package org.apache.kylin.rest.metrics;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.ServiceTestBase;
-import org.apache.kylin.rest.util.QueryMetricsUtil;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
-import java.lang.management.ManagementFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
public class QueryMetricsTest extends ServiceTestBase {
private static MBeanServer mBeanServer;
private static ObjectName objectName;
- private ConcurrentHashMap<String, QueryMetrics> metricsMap = new ConcurrentHashMap<String, QueryMetrics>();
@Before
public void setup() throws Exception {
@@ -71,7 +69,7 @@ public class QueryMetricsTest extends ServiceTestBase {
sqlResponse.setResults(results);
sqlResponse.setStorageCacheUsed(true);
- QueryMetricsUtil.updateMetrics(sqlRequest, sqlResponse, metricsMap);
+ QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse);
Thread.sleep(2000);
@@ -100,7 +98,7 @@ public class QueryMetricsTest extends ServiceTestBase {
sqlResponse2.setCube("test_cube");
sqlResponse2.setIsException(true);
- QueryMetricsUtil.updateMetrics(sqlRequest, sqlResponse2, metricsMap);
+ QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse2);
Thread.sleep(2000);
[07/50] [abbrv] kylin git commit: KYLIN-1904 Fix DictionaryDesc
checking rule
Posted by li...@apache.org.
KYLIN-1904 Fix DictionaryDesc checking rule
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03c6cf3d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03c6cf3d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03c6cf3d
Branch: refs/heads/1.5.x-CDH5.7
Commit: 03c6cf3dab0e3a40f663e083348cb3e4cdc59a49
Parents: 4c897dd
Author: Li Yang <li...@apache.org>
Authored: Wed Aug 24 16:59:47 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Aug 24 16:59:47 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeDesc.java | 5 +--
.../model/validation/rule/DictionaryRule.java | 43 ++++++--------------
2 files changed, 15 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/03c6cf3d/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 30290cd..2d9945a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -1104,9 +1104,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
for (DictionaryDesc desc : dictionaries) {
if (desc.getBuilderClass() != null) {
- if (col.equals(desc.getResuseColumnRef())) {
- return desc.getBuilderClass();
- } else if (col.equals(desc.getColumnRef())) {
+ // column that reuses other's dict need not be built, thus should not reach here
+ if (col.equals(desc.getColumnRef())) {
return desc.getBuilderClass();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/03c6cf3d/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
index 2368105..9e6b600 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
@@ -23,11 +23,9 @@ import java.util.List;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.DictionaryDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.dict.GlobalDictionaryBuilder;
import org.apache.kylin.metadata.model.TblColRef;
/**
@@ -45,43 +43,28 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
HashMap<TblColRef, String> colToBuilderMap = new HashMap<>();
HashMap<TblColRef, TblColRef> colToReuseColMap = new HashMap<>();
for (DictionaryDesc dictDesc : dictDescs) {
- // Make sure the same column associate with same builder class, check the reuse column by default
- TblColRef dictCol = dictDesc.getResuseColumnRef();
- if (dictCol == null) {
- dictCol = dictDesc.getColumnRef();
- }
+ TblColRef dictCol = dictDesc.getColumnRef();
if (dictCol == null) {
context.addResult(ResultLevel.ERROR, "Some column in dictionaries not found");
return;
}
- String builder = dictDesc.getBuilderClass();
- String oldBuilder = colToBuilderMap.put(dictCol, builder);
- if (oldBuilder != null && !oldBuilder.equals(builder)) {
- context.addResult(ResultLevel.ERROR, "Column " + dictCol + " has inconsistent builders " + builder + " and " + oldBuilder);
- return;
- }
-
- // Make sure one column only reuse another one column
- if (dictDesc.getResuseColumnRef() != null) {
- TblColRef oldReuseCol = colToReuseColMap.put(dictDesc.getColumnRef(), dictDesc.getResuseColumnRef());
- if (oldReuseCol != null && !dictDesc.getResuseColumnRef().equals(oldReuseCol)) {
- context.addResult(ResultLevel.ERROR, "Column " + dictDesc.getColumnRef() + " reuse inconsistent column " + dictDesc.getResuseColumnRef() + " and " + oldReuseCol);
+ TblColRef reuseCol = dictDesc.getResuseColumnRef();
+ if (reuseCol == null) {
+ // Make sure the same column associate with same builder class
+ String builder = dictDesc.getBuilderClass();
+ String oldBuilder = colToBuilderMap.put(dictCol, builder);
+ if (oldBuilder != null && !oldBuilder.equals(builder)) {
+ context.addResult(ResultLevel.ERROR, "Column " + dictCol + " has inconsistent builders " + builder + " and " + oldBuilder);
return;
}
- }
- }
-
- // Make sure one column do not use GlobalDictBuilder and DimensionDictEncoding together
- RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns();
- for (RowKeyColDesc desc : rowKeyColDescs) {
- if (desc.isUsingDictionary()) {
- String builder = colToBuilderMap.get(desc.getColRef());
- if (builder != null && builder.equals(GlobalDictionaryBuilder.class.getName())) {
- context.addResult(ResultLevel.ERROR, "Column " + desc.getColRef() + " used as dimension and conflict with GlobalDictBuilder");
+ } else {
+ // Make sure one column only reuse another one column
+ TblColRef oldReuseCol = colToReuseColMap.put(dictCol, reuseCol);
+ if (oldReuseCol != null && !reuseCol.equals(oldReuseCol)) {
+ context.addResult(ResultLevel.ERROR, "Column " + dictCol + " reuse inconsistent column " + reuseCol + " and " + oldReuseCol);
return;
}
}
}
-
}
}
[39/50] [abbrv] kylin git commit: KYLIN-1985 SnapshotTable should
only keep the columns described in tableDesc
Posted by li...@apache.org.
KYLIN-1985 SnapshotTable should only keep the columns described in tableDesc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/96da2b6c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/96da2b6c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/96da2b6c
Branch: refs/heads/1.5.x-CDH5.7
Commit: 96da2b6cd4aa38eca664ebea69f0adc992c58ccd
Parents: bcd086c
Author: Zhengdong <zh...@outlook.com>
Authored: Thu Sep 1 18:11:00 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Sep 1 18:11:31 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/dict/lookup/SnapshotTable.java | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/96da2b6c/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index 1aea124..db1a170 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -36,6 +36,7 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionary;
import org.apache.kylin.dict.TrieDictionaryBuilder;
+import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.ReadableTable;
@@ -84,8 +85,8 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
if (row.length <= maxIndex) {
throw new IllegalStateException("Bad hive table row, " + tableDesc + " expect " + (maxIndex + 1) + " columns, but got " + Arrays.toString(row));
}
-
- for (String cell : row) {
+ for (ColumnDesc column : tableDesc.getColumns()) {
+ String cell = row[column.getZeroBasedIndex()];
if (cell != null)
b.addValue(cell);
}
@@ -101,9 +102,9 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
try {
while (reader.next()) {
String[] row = reader.getRow();
- int[] rowIndex = new int[row.length];
- for (int i = 0; i < row.length; i++) {
- rowIndex[i] = dict.getIdFromValue(row[i]);
+ int[] rowIndex = new int[tableDesc.getColumnCount()];
+ for (ColumnDesc column : tableDesc.getColumns()) {
+ rowIndex[column.getZeroBasedIndex()] = dict.getIdFromValue(row[column.getZeroBasedIndex()]);
}
allRowIndices.add(rowIndex);
}
[30/50] [abbrv] kylin git commit: KYLIN-1986 CubeMigrationCLI: make
global dictionary unique
Posted by li...@apache.org.
KYLIN-1986 CubeMigrationCLI: make global dictionary unique
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a77fbc85
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a77fbc85
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a77fbc85
Branch: refs/heads/1.5.x-CDH5.7
Commit: a77fbc850178f9a730f2367add24ee1d71dc9046
Parents: a9ba1da
Author: kangkaisen <ka...@live.com>
Authored: Tue Aug 30 19:28:33 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Aug 31 13:25:28 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a77fbc85/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 0ecb6fb..3066fb5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -20,8 +20,10 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
@@ -213,7 +215,7 @@ public class CubeMigrationCLI {
private static void copyFilesInMetaStore(CubeInstance cube, String overwriteIfExists) throws IOException {
List<String> metaItems = new ArrayList<String>();
- List<String> dictAndSnapshot = new ArrayList<String>();
+ Set<String> dictAndSnapshot = new HashSet<String>();
listCubeRelatedResources(cube, metaItems, dictAndSnapshot);
if (dstStore.exists(cube.getResourcePath()) && !overwriteIfExists.equalsIgnoreCase("true"))
@@ -239,7 +241,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.PURGE_AND_DISABLE, new Object[] { cubeName }));
}
- private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, List<String> dictAndSnapshot) throws IOException {
+ private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot) throws IOException {
CubeDesc cubeDesc = cube.getDescriptor();
metaResource.add(cube.getResourcePath());
[43/50] [abbrv] kylin git commit: KYLIN-1698 backend done, pending GUI
Posted by li...@apache.org.
KYLIN-1698 backend done, pending GUI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a20d709d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a20d709d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a20d709d
Branch: refs/heads/1.5.x-CDH5.7
Commit: a20d709d27da9060d63ccb1c14902235a3aedec2
Parents: 16de8ba
Author: Li Yang <li...@apache.org>
Authored: Fri Sep 2 15:49:34 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Sep 2 15:49:48 2016 +0800
----------------------------------------------------------------------
.../kylin/metadata/datatype/DataType.java | 2 +-
.../kylin/metadata/model/PartitionDesc.java | 20 +++++++++++++++++++-
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a20d709d/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index 032fba2..c981f46 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -63,7 +63,7 @@ public class DataType implements Serializable {
register("any", "char", "varchar", "string", //
"boolean", "byte", "binary", //
"int", "short", "long", "integer", "tinyint", "smallint", "bigint", //
- "int4", "long8", //
+ "int4", "long8", // for test only
"float", "real", "double", "decimal", "numeric", //
"date", "time", "datetime", "timestamp", //
InnerDataTypeEnum.LITERAL.getDataType(), InnerDataTypeEnum.DERIVED.getDataType());
http://git-wip-us.apache.org/repos/asf/kylin/blob/a20d709d/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 3f265e2..8d3979e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.metadata.datatype.DataType;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
@@ -90,6 +91,11 @@ public class PartitionDesc {
partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
}
+ public boolean partitionColumnIsTimeMillis() {
+ DataType type = partitionDateColumnRef.getType();
+ return type.isBigInt();
+ }
+
public boolean isPartitioned() {
return partitionDateColumnRef != null;
}
@@ -166,7 +172,9 @@ public class PartitionDesc {
String partitionDateColumnName = partDesc.getPartitionDateColumn();
String partitionTimeColumnName = partDesc.getPartitionTimeColumn();
- if (partitionDateColumnName != null && partitionTimeColumnName == null) {
+ if (partDesc.partitionColumnIsTimeMillis()) {
+ buildSingleColumnRangeCondition(builder, partitionDateColumnName, startInclusive, endExclusive, tableAlias);
+ } else if (partitionDateColumnName != null && partitionTimeColumnName == null) {
buildSingleColumnRangeCondition(builder, partitionDateColumnName, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), tableAlias);
} else if (partitionDateColumnName == null && partitionTimeColumnName != null) {
buildSingleColumnRangeCondition(builder, partitionTimeColumnName, startInclusive, endExclusive, partDesc.getPartitionTimeFormat(), tableAlias);
@@ -190,6 +198,15 @@ public class PartitionDesc {
return columnName;
}
+ private static void buildSingleColumnRangeCondition(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, Map<String, String> tableAlias) {
+ partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias);
+ if (startInclusive > 0) {
+ builder.append(partitionColumnName + " >= " + startInclusive);
+ builder.append(" AND ");
+ }
+ builder.append(partitionColumnName + " < " + endExclusive);
+ }
+
private static void buildSingleColumnRangeCondition(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, String partitionColumnDateFormat, Map<String, String> tableAlias) {
partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias);
if (startInclusive > 0) {
@@ -272,4 +289,5 @@ public class PartitionDesc {
newPartDesc.setPartitionDateStart(partitionDesc.getPartitionDateStart());
return newPartDesc;
}
+
}
[04/50] [abbrv] kylin git commit: KYLIN-1908 Fix UT
Posted by li...@apache.org.
KYLIN-1908 Fix UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a863ae57
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a863ae57
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a863ae57
Branch: refs/heads/1.5.x-CDH5.7
Commit: a863ae57f6e9f0762848d837df69c1913dfecc38
Parents: 2ce1cde
Author: lidongsjtu <li...@apache.org>
Authored: Mon Aug 22 11:21:03 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Aug 22 11:21:11 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/metrics/QueryMetricsTest.java | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a863ae57/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
index d413f40..47d3f7b 100644
--- a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java
@@ -39,7 +39,6 @@ public class QueryMetricsTest extends ServiceTestBase {
@Before
public void setup() throws Exception {
-
super.setup();
mBeanServer = ManagementFactory.getPlatformMBeanServer();
@@ -48,6 +47,9 @@ public class QueryMetricsTest extends ServiceTestBase {
@Test
public void testQueryMetrics() throws Exception {
+ System.setProperty("kylin.query.metrics.enabled", "true");
+ QueryMetricsFacade.init();
+
SQLRequest sqlRequest = new SQLRequest();
sqlRequest.setSql("select * from TEST_KYLIN_FACT");
sqlRequest.setProject("default");
@@ -106,10 +108,7 @@ public class QueryMetricsTest extends ServiceTestBase {
Assert.assertEquals(1L, mBeanServer.getAttribute(objectName, "QuerySuccessCount"));
Assert.assertEquals(1L, mBeanServer.getAttribute(objectName, "QueryFailCount"));
+ System.clearProperty("kylin.query.metrics.enabled");
}
- @Test
- public void test() throws Exception {
-
- }
}
[35/50] [abbrv] kylin git commit: minor,
remove useless columns in cal_dt in examples
Posted by li...@apache.org.
minor, remove useless columns in cal_dt in examples
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/eb991059
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/eb991059
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/eb991059
Branch: refs/heads/1.5.x-CDH5.7
Commit: eb991059fc48dee03278e729a4eeacebf92e6409
Parents: 2ed2b38
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Aug 31 19:11:16 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Aug 31 19:11:31 2016 +0800
----------------------------------------------------------------------
.../localmeta/table/EDW.TEST_CAL_DT.json | 892 ++++++++++---------
.../localmeta/table/EDW.V_TEST_CAL_DT.json | 889 +++++++++---------
2 files changed, 968 insertions(+), 813 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb991059/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
index 9296109..d64eb80 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
@@ -1,408 +1,488 @@
{
-
- "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- "name" : "TEST_CAL_DT",
- "columns" : [ {
- "id" : "1",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "2",
- "name" : "YEAR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "QTR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "4",
- "name" : "MONTH_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "5",
- "name" : "WEEK_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "6",
- "name" : "AGE_FOR_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "AGE_FOR_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "8",
- "name" : "AGE_FOR_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "9",
- "name" : "AGE_FOR_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "10",
- "name" : "AGE_FOR_DT_ID",
- "datatype" : "smallint"
- }, {
- "id" : "11",
- "name" : "AGE_FOR_RTL_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "12",
- "name" : "AGE_FOR_RTL_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "13",
- "name" : "AGE_FOR_RTL_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "14",
- "name" : "AGE_FOR_RTL_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "15",
- "name" : "AGE_FOR_CS_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "16",
- "name" : "DAY_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "17",
- "name" : "DAY_OF_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "18",
- "name" : "DAY_OF_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "19",
- "name" : "DAY_OF_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "20",
- "name" : "DAY_OF_WEEK_ID",
- "datatype" : "int"
- }, {
- "id" : "21",
- "name" : "WEEK_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "22",
- "name" : "WEEK_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "23",
- "name" : "MONTH_OF_QTR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "24",
- "name" : "MONTH_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "25",
- "name" : "MONTH_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "26",
- "name" : "QTR_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "27",
- "name" : "QTR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "28",
- "name" : "YEAR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "29",
- "name" : "YEAR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "30",
- "name" : "QTR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "MONTH_END_DT",
- "datatype" : "string"
- }, {
- "id" : "32",
- "name" : "WEEK_END_DT",
- "datatype" : "string"
- }, {
- "id" : "33",
- "name" : "CAL_DT_NAME",
- "datatype" : "string"
- }, {
- "id" : "34",
- "name" : "CAL_DT_DESC",
- "datatype" : "string"
- }, {
- "id" : "35",
- "name" : "CAL_DT_SHORT_NAME",
- "datatype" : "string"
- }, {
- "id" : "36",
- "name" : "YTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "37",
- "name" : "QTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "38",
- "name" : "MTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "39",
- "name" : "WTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "40",
- "name" : "SEASON_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "41",
- "name" : "DAY_IN_YEAR_COUNT",
- "datatype" : "smallint"
- }, {
- "id" : "42",
- "name" : "DAY_IN_QTR_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "43",
- "name" : "DAY_IN_MONTH_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "44",
- "name" : "DAY_IN_WEEK_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "45",
- "name" : "RTL_YEAR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "46",
- "name" : "RTL_QTR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "47",
- "name" : "RTL_MONTH_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "48",
- "name" : "RTL_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "49",
- "name" : "CS_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "50",
- "name" : "CAL_DATE",
- "datatype" : "string"
- }, {
- "id" : "51",
- "name" : "DAY_OF_WEEK",
- "datatype" : "string"
- }, {
- "id" : "52",
- "name" : "MONTH_ID",
- "datatype" : "string"
- }, {
- "id" : "53",
- "name" : "PRD_DESC",
- "datatype" : "string"
- }, {
- "id" : "54",
- "name" : "PRD_FLAG",
- "datatype" : "string"
- }, {
- "id" : "55",
- "name" : "PRD_ID",
- "datatype" : "string"
- }, {
- "id" : "56",
- "name" : "PRD_IND",
- "datatype" : "string"
- }, {
- "id" : "57",
- "name" : "QTR_DESC",
- "datatype" : "string"
- }, {
- "id" : "58",
- "name" : "QTR_ID",
- "datatype" : "string"
- }, {
- "id" : "59",
- "name" : "QTR_IND",
- "datatype" : "string"
- }, {
- "id" : "60",
- "name" : "RETAIL_WEEK",
- "datatype" : "string"
- }, {
- "id" : "61",
- "name" : "RETAIL_YEAR",
- "datatype" : "string"
- }, {
- "id" : "62",
- "name" : "RETAIL_START_DATE",
- "datatype" : "string"
- }, {
- "id" : "63",
- "name" : "RETAIL_WK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "64",
- "name" : "WEEK_IND",
- "datatype" : "string"
- }, {
- "id" : "65",
- "name" : "WEEK_NUM_DESC",
- "datatype" : "string"
- }, {
- "id" : "66",
- "name" : "WEEK_BEG_DATE",
- "datatype" : "string"
- }, {
- "id" : "67",
- "name" : "WEEK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "68",
- "name" : "WEEK_IN_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "69",
- "name" : "WEEK_ID",
- "datatype" : "string"
- }, {
- "id" : "70",
- "name" : "WEEK_BEG_END_DESC_MDY",
- "datatype" : "string"
- }, {
- "id" : "71",
- "name" : "WEEK_BEG_END_DESC_MD",
- "datatype" : "string"
- }, {
- "id" : "72",
- "name" : "YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "73",
- "name" : "YEAR_IND",
- "datatype" : "string"
- }, {
- "id" : "74",
- "name" : "CAL_DT_MNS_1YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "75",
- "name" : "CAL_DT_MNS_2YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "76",
- "name" : "CAL_DT_MNS_1QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "77",
- "name" : "CAL_DT_MNS_2QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "78",
- "name" : "CAL_DT_MNS_1MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "79",
- "name" : "CAL_DT_MNS_2MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "80",
- "name" : "CAL_DT_MNS_1WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "81",
- "name" : "CAL_DT_MNS_2WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "82",
- "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "83",
- "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "84",
- "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "85",
- "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "86",
- "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "87",
- "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "88",
- "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "89",
- "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "90",
- "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "91",
- "name" : "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "92",
- "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "93",
- "name" : "SEASON_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "94",
- "name" : "YTM_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "95",
- "name" : "YTQ_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "96",
- "name" : "YTW_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "97",
- "name" : "CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "98",
- "name" : "CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "99",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "100",
- "name" : "UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "edw",
- "last_modified" : 0
+ "uuid": "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+ "name": "TEST_CAL_DT",
+ "columns": [
+ {
+ "id": "1",
+ "name": "CAL_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "2",
+ "name": "YEAR_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "3",
+ "name": "QTR_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "4",
+ "name": "MONTH_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "5",
+ "name": "WEEK_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "6",
+ "name": "AGE_FOR_YEAR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "7",
+ "name": "AGE_FOR_QTR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "8",
+ "name": "AGE_FOR_MONTH_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "9",
+ "name": "AGE_FOR_WEEK_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "10",
+ "name": "AGE_FOR_DT_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "11",
+ "name": "AGE_FOR_RTL_YEAR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "12",
+ "name": "AGE_FOR_RTL_QTR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "13",
+ "name": "AGE_FOR_RTL_MONTH_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "14",
+ "name": "AGE_FOR_RTL_WEEK_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "15",
+ "name": "AGE_FOR_CS_WEEK_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "16",
+ "name": "DAY_OF_CAL_ID",
+ "datatype": "int"
+ },
+ {
+ "id": "17",
+ "name": "DAY_OF_YEAR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "18",
+ "name": "DAY_OF_QTR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "19",
+ "name": "DAY_OF_MONTH_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "20",
+ "name": "DAY_OF_WEEK_ID",
+ "datatype": "int"
+ },
+ {
+ "id": "21",
+ "name": "WEEK_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "22",
+ "name": "WEEK_OF_CAL_ID",
+ "datatype": "int"
+ },
+ {
+ "id": "23",
+ "name": "MONTH_OF_QTR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "24",
+ "name": "MONTH_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "25",
+ "name": "MONTH_OF_CAL_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "26",
+ "name": "QTR_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "27",
+ "name": "QTR_OF_CAL_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "28",
+ "name": "YEAR_OF_CAL_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "29",
+ "name": "YEAR_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "30",
+ "name": "QTR_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "31",
+ "name": "MONTH_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "32",
+ "name": "WEEK_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "33",
+ "name": "CAL_DT_NAME",
+ "datatype": "string"
+ },
+ {
+ "id": "34",
+ "name": "CAL_DT_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "35",
+ "name": "CAL_DT_SHORT_NAME",
+ "datatype": "string"
+ },
+ {
+ "id": "36",
+ "name": "YTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "37",
+ "name": "QTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "38",
+ "name": "MTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "39",
+ "name": "WTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "40",
+ "name": "SEASON_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "41",
+ "name": "DAY_IN_YEAR_COUNT",
+ "datatype": "smallint"
+ },
+ {
+ "id": "42",
+ "name": "DAY_IN_QTR_COUNT",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "43",
+ "name": "DAY_IN_MONTH_COUNT",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "44",
+ "name": "DAY_IN_WEEK_COUNT",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "45",
+ "name": "RTL_YEAR_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "46",
+ "name": "RTL_QTR_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "47",
+ "name": "RTL_MONTH_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "48",
+ "name": "RTL_WEEK_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "49",
+ "name": "CS_WEEK_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "50",
+ "name": "CAL_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "51",
+ "name": "DAY_OF_WEEK",
+ "datatype": "string"
+ },
+ {
+ "id": "52",
+ "name": "MONTH_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "53",
+ "name": "PRD_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "54",
+ "name": "PRD_FLAG",
+ "datatype": "string"
+ },
+ {
+ "id": "55",
+ "name": "PRD_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "56",
+ "name": "PRD_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "57",
+ "name": "QTR_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "58",
+ "name": "QTR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "59",
+ "name": "QTR_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "60",
+ "name": "RETAIL_WEEK",
+ "datatype": "string"
+ },
+ {
+ "id": "61",
+ "name": "RETAIL_YEAR",
+ "datatype": "string"
+ },
+ {
+ "id": "62",
+ "name": "RETAIL_START_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "63",
+ "name": "RETAIL_WK_END_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "64",
+ "name": "WEEK_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "65",
+ "name": "WEEK_NUM_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "66",
+ "name": "WEEK_BEG_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "67",
+ "name": "WEEK_END_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "68",
+ "name": "WEEK_IN_YEAR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "69",
+ "name": "WEEK_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "70",
+ "name": "WEEK_BEG_END_DESC_MDY",
+ "datatype": "string"
+ },
+ {
+ "id": "71",
+ "name": "WEEK_BEG_END_DESC_MD",
+ "datatype": "string"
+ },
+ {
+ "id": "72",
+ "name": "YEAR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "73",
+ "name": "YEAR_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "74",
+ "name": "CAL_DT_MNS_1YEAR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "75",
+ "name": "CAL_DT_MNS_2YEAR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "76",
+ "name": "CAL_DT_MNS_1QTR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "77",
+ "name": "CAL_DT_MNS_2QTR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "78",
+ "name": "CAL_DT_MNS_1MONTH_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "79",
+ "name": "CAL_DT_MNS_2MONTH_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "80",
+ "name": "CAL_DT_MNS_1WEEK_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "81",
+ "name": "CAL_DT_MNS_2WEEK_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "82",
+ "name": "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "83",
+ "name": "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "84",
+ "name": "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "85",
+ "name": "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "86",
+ "name": "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "87",
+ "name": "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "88",
+ "name": "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "89",
+ "name": "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "90",
+ "name": "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "91",
+ "name": "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "92",
+ "name": "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "93",
+ "name": "SEASON_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "94",
+ "name": "YTM_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "95",
+ "name": "YTQ_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "96",
+ "name": "YTW_YN_ID",
+ "datatype": "tinyint"
+ }
+ ],
+ "database": "edw",
+ "last_modified": 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/eb991059/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
index 0015301..c48b9e2 100644
--- a/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
+++ b/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
@@ -1,409 +1,484 @@
{
-
- "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- "name" : "V_TEST_CAL_DT",
- "columns" : [ {
- "id" : "1",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "2",
- "name" : "YEAR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "QTR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "4",
- "name" : "MONTH_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "5",
- "name" : "WEEK_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "6",
- "name" : "AGE_FOR_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "AGE_FOR_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "8",
- "name" : "AGE_FOR_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "9",
- "name" : "AGE_FOR_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "10",
- "name" : "AGE_FOR_DT_ID",
- "datatype" : "smallint"
- }, {
- "id" : "11",
- "name" : "AGE_FOR_RTL_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "12",
- "name" : "AGE_FOR_RTL_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "13",
- "name" : "AGE_FOR_RTL_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "14",
- "name" : "AGE_FOR_RTL_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "15",
- "name" : "AGE_FOR_CS_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "16",
- "name" : "DAY_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "17",
- "name" : "DAY_OF_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "18",
- "name" : "DAY_OF_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "19",
- "name" : "DAY_OF_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "20",
- "name" : "DAY_OF_WEEK_ID",
- "datatype" : "int"
- }, {
- "id" : "21",
- "name" : "WEEK_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "22",
- "name" : "WEEK_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "23",
- "name" : "MONTH_OF_QTR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "24",
- "name" : "MONTH_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "25",
- "name" : "MONTH_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "26",
- "name" : "QTR_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "27",
- "name" : "QTR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "28",
- "name" : "YEAR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "29",
- "name" : "YEAR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "30",
- "name" : "QTR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "MONTH_END_DT",
- "datatype" : "string"
- }, {
- "id" : "32",
- "name" : "WEEK_END_DT",
- "datatype" : "string"
- }, {
- "id" : "33",
- "name" : "CAL_DT_NAME",
- "datatype" : "string"
- }, {
- "id" : "34",
- "name" : "CAL_DT_DESC",
- "datatype" : "string"
- }, {
- "id" : "35",
- "name" : "CAL_DT_SHORT_NAME",
- "datatype" : "string"
- }, {
- "id" : "36",
- "name" : "YTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "37",
- "name" : "QTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "38",
- "name" : "MTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "39",
- "name" : "WTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "40",
- "name" : "SEASON_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "41",
- "name" : "DAY_IN_YEAR_COUNT",
- "datatype" : "smallint"
- }, {
- "id" : "42",
- "name" : "DAY_IN_QTR_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "43",
- "name" : "DAY_IN_MONTH_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "44",
- "name" : "DAY_IN_WEEK_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "45",
- "name" : "RTL_YEAR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "46",
- "name" : "RTL_QTR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "47",
- "name" : "RTL_MONTH_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "48",
- "name" : "RTL_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "49",
- "name" : "CS_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "50",
- "name" : "CAL_DATE",
- "datatype" : "string"
- }, {
- "id" : "51",
- "name" : "DAY_OF_WEEK",
- "datatype" : "string"
- }, {
- "id" : "52",
- "name" : "MONTH_ID",
- "datatype" : "string"
- }, {
- "id" : "53",
- "name" : "PRD_DESC",
- "datatype" : "string"
- }, {
- "id" : "54",
- "name" : "PRD_FLAG",
- "datatype" : "string"
- }, {
- "id" : "55",
- "name" : "PRD_ID",
- "datatype" : "string"
- }, {
- "id" : "56",
- "name" : "PRD_IND",
- "datatype" : "string"
- }, {
- "id" : "57",
- "name" : "QTR_DESC",
- "datatype" : "string"
- }, {
- "id" : "58",
- "name" : "QTR_ID",
- "datatype" : "string"
- }, {
- "id" : "59",
- "name" : "QTR_IND",
- "datatype" : "string"
- }, {
- "id" : "60",
- "name" : "RETAIL_WEEK",
- "datatype" : "string"
- }, {
- "id" : "61",
- "name" : "RETAIL_YEAR",
- "datatype" : "string"
- }, {
- "id" : "62",
- "name" : "RETAIL_START_DATE",
- "datatype" : "string"
- }, {
- "id" : "63",
- "name" : "RETAIL_WK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "64",
- "name" : "WEEK_IND",
- "datatype" : "string"
- }, {
- "id" : "65",
- "name" : "WEEK_NUM_DESC",
- "datatype" : "string"
- }, {
- "id" : "66",
- "name" : "WEEK_BEG_DATE",
- "datatype" : "string"
- }, {
- "id" : "67",
- "name" : "WEEK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "68",
- "name" : "WEEK_IN_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "69",
- "name" : "WEEK_ID",
- "datatype" : "string"
- }, {
- "id" : "70",
- "name" : "WEEK_BEG_END_DESC_MDY",
- "datatype" : "string"
- }, {
- "id" : "71",
- "name" : "WEEK_BEG_END_DESC_MD",
- "datatype" : "string"
- }, {
- "id" : "72",
- "name" : "YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "73",
- "name" : "YEAR_IND",
- "datatype" : "string"
- }, {
- "id" : "74",
- "name" : "CAL_DT_MNS_1YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "75",
- "name" : "CAL_DT_MNS_2YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "76",
- "name" : "CAL_DT_MNS_1QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "77",
- "name" : "CAL_DT_MNS_2QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "78",
- "name" : "CAL_DT_MNS_1MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "79",
- "name" : "CAL_DT_MNS_2MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "80",
- "name" : "CAL_DT_MNS_1WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "81",
- "name" : "CAL_DT_MNS_2WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "82",
- "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "83",
- "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "84",
- "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "85",
- "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "86",
- "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "87",
- "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "88",
- "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "89",
- "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "90",
- "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "91",
- "name" : "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "92",
- "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "93",
- "name" : "SEASON_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "94",
- "name" : "YTM_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "95",
- "name" : "YTQ_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "96",
- "name" : "YTW_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "97",
- "name" : "CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "98",
- "name" : "CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "99",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "100",
- "name" : "UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "edw",
- "table_type" : "VIRTUAL_VIEW",
- "last_modified" : 0
+ "uuid": "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+ "name": "V_TEST_CAL_DT",
+ "columns": [
+ {
+ "id": "1",
+ "name": "CAL_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "2",
+ "name": "YEAR_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "3",
+ "name": "QTR_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "4",
+ "name": "MONTH_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "5",
+ "name": "WEEK_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "6",
+ "name": "AGE_FOR_YEAR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "7",
+ "name": "AGE_FOR_QTR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "8",
+ "name": "AGE_FOR_MONTH_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "9",
+ "name": "AGE_FOR_WEEK_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "10",
+ "name": "AGE_FOR_DT_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "11",
+ "name": "AGE_FOR_RTL_YEAR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "12",
+ "name": "AGE_FOR_RTL_QTR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "13",
+ "name": "AGE_FOR_RTL_MONTH_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "14",
+ "name": "AGE_FOR_RTL_WEEK_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "15",
+ "name": "AGE_FOR_CS_WEEK_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "16",
+ "name": "DAY_OF_CAL_ID",
+ "datatype": "int"
+ },
+ {
+ "id": "17",
+ "name": "DAY_OF_YEAR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "18",
+ "name": "DAY_OF_QTR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "19",
+ "name": "DAY_OF_MONTH_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "20",
+ "name": "DAY_OF_WEEK_ID",
+ "datatype": "int"
+ },
+ {
+ "id": "21",
+ "name": "WEEK_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "22",
+ "name": "WEEK_OF_CAL_ID",
+ "datatype": "int"
+ },
+ {
+ "id": "23",
+ "name": "MONTH_OF_QTR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "24",
+ "name": "MONTH_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "25",
+ "name": "MONTH_OF_CAL_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "26",
+ "name": "QTR_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "27",
+ "name": "QTR_OF_CAL_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "28",
+ "name": "YEAR_OF_CAL_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "29",
+ "name": "YEAR_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "30",
+ "name": "QTR_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "31",
+ "name": "MONTH_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "32",
+ "name": "WEEK_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "33",
+ "name": "CAL_DT_NAME",
+ "datatype": "string"
+ },
+ {
+ "id": "34",
+ "name": "CAL_DT_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "35",
+ "name": "CAL_DT_SHORT_NAME",
+ "datatype": "string"
+ },
+ {
+ "id": "36",
+ "name": "YTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "37",
+ "name": "QTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "38",
+ "name": "MTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "39",
+ "name": "WTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "40",
+ "name": "SEASON_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "41",
+ "name": "DAY_IN_YEAR_COUNT",
+ "datatype": "smallint"
+ },
+ {
+ "id": "42",
+ "name": "DAY_IN_QTR_COUNT",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "43",
+ "name": "DAY_IN_MONTH_COUNT",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "44",
+ "name": "DAY_IN_WEEK_COUNT",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "45",
+ "name": "RTL_YEAR_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "46",
+ "name": "RTL_QTR_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "47",
+ "name": "RTL_MONTH_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "48",
+ "name": "RTL_WEEK_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "49",
+ "name": "CS_WEEK_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "50",
+ "name": "CAL_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "51",
+ "name": "DAY_OF_WEEK",
+ "datatype": "string"
+ },
+ {
+ "id": "52",
+ "name": "MONTH_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "53",
+ "name": "PRD_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "54",
+ "name": "PRD_FLAG",
+ "datatype": "string"
+ },
+ {
+ "id": "55",
+ "name": "PRD_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "56",
+ "name": "PRD_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "57",
+ "name": "QTR_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "58",
+ "name": "QTR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "59",
+ "name": "QTR_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "60",
+ "name": "RETAIL_WEEK",
+ "datatype": "string"
+ },
+ {
+ "id": "61",
+ "name": "RETAIL_YEAR",
+ "datatype": "string"
+ },
+ {
+ "id": "62",
+ "name": "RETAIL_START_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "63",
+ "name": "RETAIL_WK_END_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "64",
+ "name": "WEEK_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "65",
+ "name": "WEEK_NUM_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "66",
+ "name": "WEEK_BEG_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "67",
+ "name": "WEEK_END_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "68",
+ "name": "WEEK_IN_YEAR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "69",
+ "name": "WEEK_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "70",
+ "name": "WEEK_BEG_END_DESC_MDY",
+ "datatype": "string"
+ },
+ {
+ "id": "71",
+ "name": "WEEK_BEG_END_DESC_MD",
+ "datatype": "string"
+ },
+ {
+ "id": "72",
+ "name": "YEAR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "73",
+ "name": "YEAR_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "74",
+ "name": "CAL_DT_MNS_1YEAR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "75",
+ "name": "CAL_DT_MNS_2YEAR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "76",
+ "name": "CAL_DT_MNS_1QTR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "77",
+ "name": "CAL_DT_MNS_2QTR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "78",
+ "name": "CAL_DT_MNS_1MONTH_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "79",
+ "name": "CAL_DT_MNS_2MONTH_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "80",
+ "name": "CAL_DT_MNS_1WEEK_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "81",
+ "name": "CAL_DT_MNS_2WEEK_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "82",
+ "name": "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "83",
+ "name": "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "84",
+ "name": "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "85",
+ "name": "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "86",
+ "name": "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "87",
+ "name": "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "88",
+ "name": "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "89",
+ "name": "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "90",
+ "name": "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "91",
+ "name": "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "92",
+ "name": "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "93",
+ "name": "SEASON_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "94",
+ "name": "YTM_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "95",
+ "name": "YTQ_YN_ID",
+ "datatype": "tinyint"
+ }
+ ],
+ "database": "edw",
+ "table_type": "VIRTUAL_VIEW",
+ "last_modified": 0
}
\ No newline at end of file
[10/50] [abbrv] kylin git commit: KYLIN-1904 fix DictionaryRuleTest
Posted by li...@apache.org.
KYLIN-1904 fix DictionaryRuleTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/57a259a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/57a259a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/57a259a9
Branch: refs/heads/1.5.x-CDH5.7
Commit: 57a259a9d28484b3494fc62385a8d284c1052209
Parents: 579dfcf
Author: Li Yang <li...@apache.org>
Authored: Thu Aug 25 12:01:46 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Aug 25 12:01:46 2016 +0800
----------------------------------------------------------------------
.../model/validation/rule/DictionaryRule.java | 12 ++++++++++-
.../validation/rule/DictionaryRuleTest.java | 22 ++++++++++++++------
2 files changed, 27 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/57a259a9/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
index 9e6b600..d06c816 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
@@ -48,16 +48,26 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
context.addResult(ResultLevel.ERROR, "Some column in dictionaries not found");
return;
}
+ String builder = dictDesc.getBuilderClass();
TblColRef reuseCol = dictDesc.getResuseColumnRef();
if (reuseCol == null) {
+ if (builder == null || builder.isEmpty()) {
+ context.addResult(ResultLevel.ERROR, "Column " + dictCol + " cannot have builder and reuse column both empty");
+ return;
+ }
+
// Make sure the same column associate with same builder class
- String builder = dictDesc.getBuilderClass();
String oldBuilder = colToBuilderMap.put(dictCol, builder);
if (oldBuilder != null && !oldBuilder.equals(builder)) {
context.addResult(ResultLevel.ERROR, "Column " + dictCol + " has inconsistent builders " + builder + " and " + oldBuilder);
return;
}
} else {
+ if (builder != null && !builder.isEmpty()) {
+ context.addResult(ResultLevel.ERROR, "Column " + dictCol + " cannot have builder and reuse column both");
+ return;
+ }
+
// Make sure one column only reuse another one column
TblColRef oldReuseCol = colToReuseColMap.put(dictCol, reuseCol);
if (oldReuseCol != null && !reuseCol.equals(oldReuseCol)) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/57a259a9/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index e5afe48..fdbbb2c 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -72,20 +72,25 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase {
@Test
public void testBadDesc() throws IOException {
- testBadDictionaryDesc("Column EDW.TEST_SITES.SITE_NAME has inconsistent builders " + "FakeBuilderClass and org.apache.kylin.dict.GlobalDictionaryBuilder", DictionaryDesc.create("SITE_NAME", null, "FakeBuilderClass"));
+ testDictionaryDesc("Column EDW.TEST_SITES.SITE_NAME has inconsistent builders " + "FakeBuilderClass and org.apache.kylin.dict.GlobalDictionaryBuilder", DictionaryDesc.create("SITE_NAME", null, "FakeBuilderClass"));
}
@Test
public void testBadDesc2() throws IOException {
- testBadDictionaryDesc("Column EDW.TEST_SITES.SITE_NAME has inconsistent builders " + "FakeBuilderClass and org.apache.kylin.dict.GlobalDictionaryBuilder", DictionaryDesc.create("lstg_site_id", "SITE_NAME", "FakeBuilderClass"));
+ testDictionaryDesc("Column DEFAULT.TEST_KYLIN_FACT.LSTG_SITE_ID cannot have builder and reuse column both", DictionaryDesc.create("lstg_site_id", "SITE_NAME", "FakeBuilderClass"));
}
@Test
public void testBadDesc3() throws IOException {
- testBadDictionaryDesc("Column DEFAULT.TEST_KYLIN_FACT.LSTG_SITE_ID used as dimension and conflict with GlobalDictBuilder", DictionaryDesc.create("lstg_site_id", null, GlobalDictionaryBuilder.class.getName()));
+ testDictionaryDesc("Column DEFAULT.TEST_KYLIN_FACT.LSTG_SITE_ID cannot have builder and reuse column both empty", DictionaryDesc.create("lstg_site_id", null, null));
+ }
+
+ @Test
+ public void testGoodDesc2() throws IOException {
+ testDictionaryDesc(null, DictionaryDesc.create("lstg_site_id", null, GlobalDictionaryBuilder.class.getName()));
}
- private void testBadDictionaryDesc(String expectMessage, DictionaryDesc... descs) throws IOException {
+ private void testDictionaryDesc(String expectMessage, DictionaryDesc... descs) throws IOException {
DictionaryRule rule = new DictionaryRule();
File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/test_kylin_cube_without_slr_left_join_desc.json");
CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
@@ -98,7 +103,12 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase {
ValidateContext vContext = new ValidateContext();
rule.validate(desc, vContext);
vContext.print(System.out);
- assertTrue(vContext.getResults().length >= 1);
- assertEquals(expectMessage, vContext.getResults()[0].getMessage());
+
+ if (expectMessage == null) {
+ assertTrue(vContext.getResults().length == 0);
+ } else {
+ assertTrue(vContext.getResults().length >= 1);
+ assertEquals(expectMessage, vContext.getResults()[0].getMessage());
+ }
}
}
\ No newline at end of file
[34/50] [abbrv] kylin git commit: KYLIN-1849 remove comment code
Posted by li...@apache.org.
KYLIN-1849 remove comment code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ed2b381
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ed2b381
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ed2b381
Branch: refs/heads/1.5.x-CDH5.7
Commit: 2ed2b381ba0ea67f01b3b33a0720770fa16223ee
Parents: 64005c6
Author: Jason <ji...@163.com>
Authored: Wed Aug 31 14:42:02 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Aug 31 14:42:02 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubes.js | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2ed2b381/webapp/app/js/controllers/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
index 3f125a6..f46953b 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -119,13 +119,11 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
})
$scope.loading = false;
- //$scope.listParams.cubeName = '';
defer.resolve(resp);
return defer.promise;
},function(resp){
$scope.loading = false;
- //$scope.listParams.cubeName = '';
defer.resolve([]);
SweetAlert.swal('Oops...', resp, 'error');
return defer.promise;
[20/50] [abbrv] kylin git commit: KYLIN-1904-WEB-Global-Dictionary
Posted by li...@apache.org.
KYLIN-1904-WEB-Global-Dictionary
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1f4675fc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1f4675fc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1f4675fc
Branch: refs/heads/1.5.x-CDH5.7
Commit: 1f4675fc81c8e8bcad847b4cb405b9f302e3b6d4
Parents: ab6b73b
Author: zx chen <34...@qq.com>
Authored: Thu Aug 25 13:37:24 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Mon Aug 29 11:31:44 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 114 +++++++++++++-
webapp/app/js/controllers/cubeEdit.js | 25 +++-
webapp/app/js/model/cubeDescModel.js | 9 ++
.../cubeDesigner/advanced_settings.html | 150 +++++++++++++++----
4 files changed, 264 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f4675fc/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 09fb3d4..be90f65 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -18,14 +18,14 @@
'use strict';
-KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfig,MetaModel,cubesManager,CubeDescModel) {
+KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfig,MetaModel,cubesManager,CubeDescModel,SweetAlert) {
$scope.cubesManager = cubesManager;
//rowkey
$scope.convertedRowkeys = [];
angular.forEach($scope.cubeMetaFrame.rowkey.rowkey_columns,function(item){
- //var _isDictionary = item.encoding === "dict"?"true":"false";
+ //var _isDictionaries = item.encoding === "dict"?"true":"false";
var _isFixedLength = item.encoding.substring(0,12) === "fixed_length"?"true":"false";//fixed_length:12
var _isIntLength = item.encoding.substring(0,3) === "int"?"true":"false";//fixed_length:12
var _encoding = item.encoding;
@@ -195,4 +195,114 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
}
+ $scope.isReuse=false;
+ $scope.addNew=false;
+ $scope.newDictionaries = {
+ "column":null,
+ "builder": null,
+ "reuse": null
+ }
+
+ $scope.initUpdateDictionariesStatus = function(){
+ $scope.updateDictionariesStatus = {
+ isEdit:false,
+ editIndex:-1
+ }
+ };
+ $scope.initUpdateDictionariesStatus();
+
+
+ $scope.addNewDictionaries = function (dictionaries, index) {
+ if(dictionaries&&index>=0){
+ $scope.updateDictionariesStatus.isEdit = true;
+ $scope.addNew=true;
+ $scope.updateDictionariesStatus.editIndex = index;
+ if(dictionaries.builder==null){
+ $scope.isReuse=true;
+ }
+ else{
+ $scope.isReuse=false;
+ }
+ }
+ else{
+ $scope.addNew=!$scope.addNew;
+ }
+ $scope.newDictionaries = (!!dictionaries)? jQuery.extend(true, {},dictionaries):CubeDescModel.createDictionaries();
+ };
+
+ $scope.saveNewDictionaries = function (){
+ if(!$scope.cubeMetaFrame.dictionaries){
+ $scope.cubeMetaFrame.dictionaries=[];
+ }
+
+ if($scope.updateDictionariesStatus.isEdit == true) {
+ if ($scope.cubeMetaFrame.dictionaries[$scope.updateDictionariesStatus.editIndex].column != $scope.newDictionaries.column) {
+ if(!$scope.checkColumn()){
+ return false;
+ }
+ }
+ else {
+ $scope.cubeMetaFrame.dictionaries[$scope.updateDictionariesStatus.editIndex] = $scope.newDictionaries;
+ }
+ }
+ else
+ {
+ if(!$scope.checkColumn()){
+ return false;
+ }
+ $scope.cubeMetaFrame.dictionaries.push($scope.newDictionaries);
+ }
+ $scope.newDictionaries = null;
+ $scope.initUpdateDictionariesStatus();
+ $scope.nextDictionariesInit();
+ $scope.addNew = !$scope.addNew;
+ $scope.isReuse = false;
+ return true;
+
+ };
+
+ $scope.nextDictionariesInit = function(){
+ $scope.nextDic = {
+ "coiumn":null,
+ "builder":null,
+ "reuse":null
+ }
+ }
+
+ $scope.checkColumn = function (){
+ var isColumnExit=false;
+ angular.forEach($scope.cubeMetaFrame.dictionaries,function(dictionaries){
+ if(!isColumnExit){
+ if(dictionaries.column==$scope.newDictionaries.column)
+ isColumnExit=true;
+ }
+ })
+ if(isColumnExit){
+ SweetAlert.swal('Oops...', "The column named [" + $scope.newDictionaries.column + "] already exists", 'warning');
+ return false;
+ }
+ return true;
+ }
+
+ $scope.clearNewDictionaries = function (){
+ $scope.newDictionaries = null;
+ $scope.isReuse=false;
+ $scope.initUpdateDictionariesStatus();
+ $scope.nextDictionariesInit();
+ $scope.addNew=!$scope.addNew;
+ }
+
+ $scope.change = function (){
+ $scope.newDictionaries.builder=null;
+ $scope.newDictionaries.reuse=null;
+ $scope.isReuse=!$scope.isReuse;
+ }
+
+ $scope.removeDictionaries = function(arr,element){
+ var index = arr.indexOf(element);
+ if (index > -1) {
+ arr.splice(index, 1);
+ }
+ };
+
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f4675fc/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index 86c039a..b620d7f 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -116,16 +116,33 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
$scope.getFactColumns = function () {
var me_columns = [];
angular.forEach($scope.cubeMetaFrame.dimensions,function(dimension,index){
- if($scope.metaModel.model.fact_table !== dimension.table){
- return;
- }
if(dimension.column && dimension.derived == null){
me_columns.push(dimension.column);
+
+ }
+ else{
+ angular.forEach(dimension.derived,function(derived){
+ me_columns.push(derived);
+ });
+
}
});
+ angular.forEach($scope.cubeMetaFrame.measure,function(measure){
+ if(measure.function.parameter.type==column){
+ me_columns.push(measure.function.parameter.value);
+ }
+ });
- return me_columns;
+ var unique = []
+
+ angular.forEach(me_columns, function (column) {
+ if (unique.indexOf(column) === -1) {
+ unique.push(column);
+ }
+ });
+
+ return unique;
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f4675fc/webapp/app/js/model/cubeDescModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeDescModel.js b/webapp/app/js/model/cubeDescModel.js
index a6e02ac..57d7e6e 100644
--- a/webapp/app/js/model/cubeDescModel.js
+++ b/webapp/app/js/model/cubeDescModel.js
@@ -41,6 +41,7 @@ KylinApp.service('CubeDescModel', function () {
}
}
],
+ "dictionaries" :[],
"rowkey": {
"rowkey_columns": []
},
@@ -79,6 +80,14 @@ KylinApp.service('CubeDescModel', function () {
return measure;
}
+ this.createDictionaries = function () {
+ var dictionaries = {
+ "column": null,
+ "builder": null,
+ "reuse":null
+ }
+ return dictionaries;
+ }
this.createAggGroup = function () {
var group = {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f4675fc/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index ab2b5dc..eac6944 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -16,11 +16,9 @@
* limitations under the License.
-->
-<div ng-controller="CubeAdvanceSettingCtrl">
-
+ <div ng-controller="CubeAdvanceSettingCtrl">
<ng-form name="forms.cube_setting_form">
<div>
-
<div class="form-group">
<h3 style="margin-left:42px;">Aggregation Groups </h3>
<div style="margin-left:22px;">
@@ -30,7 +28,6 @@
</div>
</div>
</div>
-
<table style="margin-left:42px; width:92%"
ng-if="cubeMetaFrame.aggregation_groups.length > 0"
class="table table-hover list">
@@ -97,8 +94,6 @@
<td class="col-xs-1">
</td>
</tr>
-
-
<tr class="row">
<td class="col-xs-3">
Hierarchy Dimensions
@@ -121,7 +116,6 @@
</ui-select>
<p ng-if="state.mode=='view'">
{{hierarchyDims}}</p>
-
</td>
<td class="col-xs-2">
<button class="btn btn-sm btn-info"
@@ -130,7 +124,6 @@
</button>
</td>
</tr>
-
<tr class="row">
<td class="col-xs-12">
<button class="btn btn-sm btn-info"
@@ -139,11 +132,8 @@
</td>
</tr>
</table>
-
</td>
</tr>
-
-
<tr class="row">
<td class="col-xs-3">
Joint Dimensions
@@ -175,7 +165,6 @@
</button>
</td>
</tr>
-
<tr class="row">
<td class="col-xs-12">
<button class="btn btn-sm btn-info"
@@ -184,11 +173,9 @@
</td>
</tr>
</table>
-
</td>
</tr>
</table>
-
</td>
<td ng-if="state.mode=='edit'" class="col-xs-1">
<button class="btn btn-xs btn-info"
@@ -198,15 +185,10 @@
</td>
</tr>
</table>
-
<button class="btn btn-sm btn-info" style="margin-left:42px" ng-click="addNewAggregationGroup()"
ng-show="state.mode=='edit'">New Aggregation Group<i class="fa fa-plus"></i>
</button>
-
-
</div>
-
-
<div class="form-group large-popover" style="overflow:auto">
<h3 style="margin-left:42px">Rowkeys <i kylinpopover placement="right" title="Rowkey" template="rowkeyTip.html" class="fa fa-info-circle"></i></h3>
@@ -291,21 +273,132 @@
</tr>
</tbody>
</table>
-
-
</div>
</div>
-
<button class="btn btn-sm btn-info" style="margin-left:42px"
ng-click="addNewRowkeyColumn()" ng-show="state.mode=='edit'">New Rowkey Column<i class="fa fa-plus"></i>
</button>
</div>
+ <div class="form-group large-popover" style="overflow:auto">
+ <h3 style="margin-left:42px">Advanced Dictionaries <i kylinpopover placement="right" title="Advanced Dictionaries" template="AdvancedDictionariesTip.html" class="fa fa-info-circle"></i></h3>
+ <div style="margin-left:42px">
+ <div class="box-body">
+ <br/>
+ <table class="table table-striped table-hover">
+ <thead>
+ <tr>
+ <th>Column</th>
+ <th>Builder Class</th>
+ <th>Reuse</th>
+ <th ng-if="state.mode=='edit'">Actions</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr ng-repeat="dictionaries in cubeMetaFrame.dictionaries | filter: state.measureFilter track by $index">
+ <td>
+ <!--Column -->
+ <span>{{dictionaries.column}}</span>
+ </td>
+ <td>
+ <!--Builder-->
+ <span>{{dictionaries.builder}}</span>
+ </td>
+ <td>
+ <!--Reuse-->
+ <span>{{dictionaries.reuse}}</span>
+ </td>
+ <td ng-if="state.mode=='edit'">
+ <!--Edit Button -->
+ <button class="btn btn-xs btn-info" ng-click="addNewDictionaries(dictionaries, $index)" >
+ <i class="fa fa-pencil"></i>
+ </button>
+ <!--Remove Button -->
+ <button class="btn btn-xs btn-danger" ng-click="removeDictionaries(cubeMetaFrame.dictionaries, dictionaries)">
+ <i class="fa fa-trash-o"></i>
+ </button>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+ <!--Add Dictionaries Button-->
+ <div class="form-group" style="margin-left:42px;">
+ <button class="btn btn-sm btn-info" ng-click="addNewDictionaries()" ng-show="state.mode=='edit' && !addNew">
+ <i class="fa fa-plus"></i> Dictionaries
+ </button>
+ </div>
+ <!--Edit Dictionsry-->
+ <ng-form name="edit_mes_form" style="margin-left:42px;">
+ <div class="box box-solid" ng-if="addNew" style="margin-left:42px;">
+ <div class="box-header">
+ <h4 class="box-title text-info">Edit Dictionaries</h4>
+ </div>
+ <div class="box-body">
+ <div class="row">
+ <div class="col-xs-8">
+ <!--Column-->
+ <div class="form-group middle-popover">
+ <div class="row">
+ <label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Column</b></label>
+ <div class="col-xs-12 col-sm-6">
+ <select class="form-control" chosen
+ ng-model="newDictionaries.column"
+ ng-change="measureReturnTypeUpdate();"
+ ng-options="column as column for column in getFactColumns()" required>
+ <option value="">-- Select a Column --</option>
+ </select>
+ </div>
+ </div>
+ </div>
+ <div class="form-group">
+ <div class="row" style="margin-left:5px;">
+ <label class="radio-inline ng-binding ng-scope">
+ <input type="radio" name="reuse" ng-value=false ng-model="isReuse" ng-change="change()"/><b>Builder Class</b>
+ </label>
+ <label class="radio-inline ng-binding ng-scope">
+ <input type="radio" name="reuse" ng-value=true ng-model="isReuse" ng-change="change()"/><b>Reuse</b>
+ </label>
+ </div>
+ </div>
+ <!--Builder-->
+ <div class="form-group" ng-if="!isReuse" >
+ <div class="row">
+ <label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Builder Class</b></label>
+ <div class="col-xs-12 col-sm-6">
+ <input type="text" name="builder" ng-model="newDictionaries.builder" required style="width:100% " />
+ </div>
+ </div>
+ </div>
+ <!--Reuse-->
+ <div class="form-group middle-popover" ng-if="isReuse" >
+ <div class="row">
+ <label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default"><b>Reuse</b></label>
+ <div class="col-xs-12 col-sm-6">
+ <select class="form-control" chosen
+ ng-model="newDictionaries.reuse"
+ ng-change="measureReturnTypeUpdate();"
+ ng-options="column as column for column in getFactColumns()" required>
+ <option value="">-- Select a Column --</option>
+ </select>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div class="box-footer">
+ <button class="btn btn-sm btn-info" ng-disabled="edit_mes_form.$invalid"
+ ng-click="saveNewDictionaries()" ng-show="state.mode=='edit'">OK</button>
+ <button class="btn btn-link" ng-click="clearNewDictionaries()">Cancel</button>
+ </div>
+ </div>
+ </ng-form>
+ </div>
+ </ng-form>
+ </div>
- </div>
-
- </ng-form>
-
-</div>
<script type="text/ng-template" id="rowkeyTip.html">
<div>
<h4>Shard By this column?</h4>
@@ -316,7 +409,7 @@
<h4>RowKey Encoding</h4>
<ol>
<li>
- "dict" encoding will try to build a dictionary for the dimension
+ "dict" encoding will try to build a dictionaries for the dimension
</li>
<li>
"fixed_length" encoding will encode the dimension vlaues by fixed length bytes with padding
@@ -327,3 +420,4 @@
</ol>
</div>
</script>
+<script type="text/ng-template" id="AdvancedDictionariesTip.html"
[09/50] [abbrv] kylin git commit: minor,
ignore exceptions in diagnosis tool
Posted by li...@apache.org.
minor, ignore exceptions in diagnosis tool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5a77f5e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5a77f5e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5a77f5e7
Branch: refs/heads/1.5.x-CDH5.7
Commit: 5a77f5e72aeda51c2e2b2c2eea77e591eb998c19
Parents: 03c6cf3
Author: lidongsjtu <li...@apache.org>
Authored: Wed Aug 24 16:43:16 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Aug 24 17:09:32 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/tool/CubeMetaExtractor.java | 17 +++---
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 56 +++++++++++++-------
2 files changed, 48 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5a77f5e7/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 5505c0a..f4fb308 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -97,14 +97,14 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
private RealizationRegistry realizationRegistry;
private BadQueryHistoryManager badQueryHistoryManager;
- boolean includeSegments;
- boolean includeJobs;
- boolean includeSegmentDetails;
- boolean onlyJobOutput;
+ private boolean includeSegments;
+ private boolean includeJobs;
+ private boolean includeSegmentDetails;
+ private boolean onlyJobOutput;
- Set<String> requiredResources = Sets.newLinkedHashSet();
- Set<String> optionalResources = Sets.newLinkedHashSet();
- Set<CubeInstance> cubesToTrimAndSave = Sets.newLinkedHashSet();//these cubes needs to be saved skipping segments
+ private Set<String> requiredResources = Sets.newLinkedHashSet();
+ private Set<String> optionalResources = Sets.newLinkedHashSet();
+ private Set<CubeInstance> cubesToTrimAndSave = Sets.newLinkedHashSet();//these cubes needs to be saved skipping segments
public CubeMetaExtractor() {
super();
@@ -239,6 +239,9 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
}
private void retrieveResourcePath(IRealization realization) {
+ if (realization == null) {
+ return;
+ }
logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
http://git-wip-us.apache.org/repos/asf/kylin/blob/5a77f5e7/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index 7b5eeaf..e77ac3b 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -123,10 +123,15 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
executorService.execute(new Runnable() {
@Override
public void run() {
- String[] cubeMetaArgs = { "-destDir", new File(exportDir, "metadata").getAbsolutePath(), "-project", projectNames, "-compress", "false", "-includeJobs", "false", "-submodule", "true" };
- CubeMetaExtractor cubeMetaExtractor = new CubeMetaExtractor();
- logger.info("CubeMetaExtractor args: " + Arrays.toString(cubeMetaArgs));
- cubeMetaExtractor.execute(cubeMetaArgs);
+ logger.info("Start to extract metadata.");
+ try {
+ String[] cubeMetaArgs = { "-destDir", new File(exportDir, "metadata").getAbsolutePath(), "-project", projectNames, "-compress", "false", "-includeJobs", "false", "-submodule", "true" };
+ CubeMetaExtractor cubeMetaExtractor = new CubeMetaExtractor();
+ logger.info("CubeMetaExtractor args: " + Arrays.toString(cubeMetaArgs));
+ cubeMetaExtractor.execute(cubeMetaArgs);
+ } catch (Exception e) {
+ logger.error("Error in export metadata.", e);
+ }
}
});
@@ -135,9 +140,14 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
executorService.execute(new Runnable() {
@Override
public void run() {
- String[] jobArgs = { "-destDir", new File(exportDir, "jobs").getAbsolutePath(), "-period", Integer.toString(period), "-compress", "false", "-submodule", "true" };
- JobInstanceExtractor jobInstanceExtractor = new JobInstanceExtractor();
- jobInstanceExtractor.execute(jobArgs);
+ logger.info("Start to extract jobs.");
+ try {
+ String[] jobArgs = { "-destDir", new File(exportDir, "jobs").getAbsolutePath(), "-period", Integer.toString(period), "-compress", "false", "-submodule", "true" };
+ JobInstanceExtractor jobInstanceExtractor = new JobInstanceExtractor();
+ jobInstanceExtractor.execute(jobArgs);
+ } catch (Exception e) {
+ logger.error("Error in export jobs.", e);
+ }
}
});
}
@@ -147,10 +157,15 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
executorService.execute(new Runnable() {
@Override
public void run() {
- String[] hbaseArgs = { "-destDir", new File(exportDir, "hbase").getAbsolutePath(), "-project", projectNames, "-compress", "false", "-submodule", "true" };
- HBaseUsageExtractor hBaseUsageExtractor = new HBaseUsageExtractor();
- logger.info("HBaseUsageExtractor args: " + Arrays.toString(hbaseArgs));
- hBaseUsageExtractor.execute(hbaseArgs);
+ logger.info("Start to extract HBase usage.");
+ try {
+ String[] hbaseArgs = { "-destDir", new File(exportDir, "hbase").getAbsolutePath(), "-project", projectNames, "-compress", "false", "-submodule", "true" };
+ HBaseUsageExtractor hBaseUsageExtractor = new HBaseUsageExtractor();
+ logger.info("HBaseUsageExtractor args: " + Arrays.toString(hbaseArgs));
+ hBaseUsageExtractor.execute(hbaseArgs);
+ } catch (Exception e) {
+ logger.error("Error in export HBase usage.", e);
+ }
}
});
}
@@ -175,7 +190,7 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
}
}
} catch (Exception e) {
- logger.warn("Error in export conf.", e);
+ logger.error("Error in export conf.", e);
}
}
});
@@ -191,8 +206,8 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
ClientEnvExtractor clientEnvExtractor = new ClientEnvExtractor();
logger.info("ClientEnvExtractor args: " + Arrays.toString(clientArgs));
clientEnvExtractor.execute(clientArgs);
- } catch (IOException e) {
- logger.warn("Error in export client info.", e);
+ } catch (Exception e) {
+ logger.error("Error in export client info.", e);
}
}
});
@@ -202,10 +217,15 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
executorService.execute(new Runnable() {
@Override
public void run() {
- String[] logsArgs = { "-destDir", new File(exportDir, "logs").getAbsolutePath(), "-logPeriod", Integer.toString(period), "-compress", "false", "-submodule", "true" };
- KylinLogExtractor logExtractor = new KylinLogExtractor();
- logger.info("KylinLogExtractor args: " + Arrays.toString(logsArgs));
- logExtractor.execute(logsArgs);
+ logger.info("Start to extract logs.");
+ try {
+ String[] logsArgs = { "-destDir", new File(exportDir, "logs").getAbsolutePath(), "-logPeriod", Integer.toString(period), "-compress", "false", "-submodule", "true" };
+ KylinLogExtractor logExtractor = new KylinLogExtractor();
+ logger.info("KylinLogExtractor args: " + Arrays.toString(logsArgs));
+ logExtractor.execute(logsArgs);
+ } catch (Exception e) {
+ logger.error("Error in export logs.", e);
+ }
}
});
[27/50] [abbrv] kylin git commit: KYLIN-1982 CubeMigrationCLI:
associate model with project
Posted by li...@apache.org.
KYLIN-1982 CubeMigrationCLI: associate model with project
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c5fdacde
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c5fdacde
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c5fdacde
Branch: refs/heads/1.5.x-CDH5.7
Commit: c5fdacde4f8f8748259afcb6800b53f9838fda75
Parents: 42f2868
Author: kangkaisen <ka...@live.com>
Authored: Sun Aug 28 20:54:26 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Aug 30 23:25:51 2016 +0800
----------------------------------------------------------------------
.../kylin/storage/hbase/util/CubeMigrationCLI.java | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c5fdacde/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index c4cf579..0ecb6fb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -138,7 +138,7 @@ public class CubeMigrationCLI {
copyFilesInMetaStore(cube, overwriteIfExists);
renameFoldersInHdfs(cube);
changeHtableHost(cube);
- addCubeIntoProject(cubeName, projectName);
+ addCubeAndModelIntoProject(cube, cubeName, projectName);
if (Boolean.parseBoolean(copyAcl) == true) {
copyACL(cube, projectName);
}
@@ -227,13 +227,12 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
-
- private static void addCubeIntoProject(String cubeName, String projectName) throws IOException {
+ private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
throw new IllegalStateException("The target project " + projectName + "does not exist");
- operations.add(new Opt(OptType.ADD_INTO_PROJECT, new Object[] { cubeName, projectName }));
+ operations.add(new Opt(OptType.ADD_INTO_PROJECT, new Object[] { srcCube, cubeName, projectName }));
}
private static void purgeAndDisable(String cubeName) throws IOException {
@@ -419,13 +418,19 @@ public class CubeMigrationCLI {
break;
}
case ADD_INTO_PROJECT: {
- String cubeName = (String) opt.params[0];
- String projectName = (String) opt.params[1];
+ CubeInstance srcCube = (CubeInstance) opt.params[0];
+ String cubeName = (String) opt.params[1];
+ String projectName = (String) opt.params[2];
+ String modelName = srcCube.getDescriptor().getModelName();
+
String projectResPath = ProjectInstance.concatResourcePath(projectName);
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
+
+ project.addModel(modelName);
project.removeRealization(RealizationType.CUBE, cubeName);
project.addRealizationEntry(RealizationType.CUBE, cubeName);
+
dstStore.putResource(projectResPath, project, projectSerializer);
logger.info("Project instance for " + projectName + " is corrected");
break;
[38/50] [abbrv] kylin git commit: ignore CubeMetaIngesterTest
temporally as the zips are outdated
Posted by li...@apache.org.
ignore CubeMetaIngesterTest temporally as the zips are outdated
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bcd086c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bcd086c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bcd086c1
Branch: refs/heads/1.5.x-CDH5.7
Commit: bcd086c11d761a9c3a0cb54770b728689fc7e371
Parents: 894c16f
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Sep 1 10:36:22 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Sep 1 10:36:22 2016 +0800
----------------------------------------------------------------------
tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bcd086c1/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
----------------------------------------------------------------------
diff --git a/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java b/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
index 581221c..ff12084 100644
--- a/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
+++ b/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
@@ -36,10 +36,12 @@ import org.hamcrest.Description;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+@Ignore
public class CubeMetaIngesterTest extends LocalFileMetadataTestCase {
@Rule
public ExpectedException thrown = ExpectedException.none();
[25/50] [abbrv] kylin git commit: minor refactor
Posted by li...@apache.org.
minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6c23313e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6c23313e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6c23313e
Branch: refs/heads/1.5.x-CDH5.7
Commit: 6c23313e9d14f8eabf165566ae0a8156b25a47e8
Parents: 27426ae
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Aug 30 16:15:28 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Aug 30 19:08:32 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ByteArray.java | 4 +
.../kylin/dict/BuildInFunctionTransformer.java | 178 -------------------
.../kylin/dict/BuiltInFunctionTransformer.java | 178 +++++++++++++++++++
.../filter/BuildInFunctionTupleFilter.java | 165 -----------------
.../filter/BuiltInFunctionTupleFilter.java | 157 ++++++++++++++++
.../kylin/metadata/filter/TupleFilter.java | 1 -
.../metadata/filter/TupleFilterSerializer.java | 2 +-
.../metadata/filter/function/Functions.java | 4 +-
.../storage/gtrecord/CubeSegmentScanner.java | 4 +-
.../test/resources/query/sql_like/query03.sql | 33 ++++
.../common/coprocessor/FilterDecorator.java | 4 +-
11 files changed, 379 insertions(+), 351 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
index a72425b..db02f34 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ByteArray.java
@@ -136,6 +136,10 @@ public class ByteArray implements Comparable<ByteArray>, Serializable {
return ByteBuffer.wrap(data, offset, length).slice();
}
+ public byte[] toBytes() {
+ return Bytes.copy(this.array(), this.offset(), this.length());
+ }
+
@Override
public int hashCode() {
if (data == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
deleted file mode 100644
index 665a7e6..0000000
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dict;
-
-import java.util.Collection;
-import java.util.ListIterator;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dimension.IDimensionEncodingMap;
-import org.apache.kylin.metadata.filter.BuildInFunctionTupleFilter;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Primitives;
-
-/**
- * only take effect when the compare filter has function
- */
-public class BuildInFunctionTransformer implements ITupleFilterTransformer {
- public static final Logger logger = LoggerFactory.getLogger(BuildInFunctionTransformer.class);
-
- private IDimensionEncodingMap dimEncMap;
-
- public BuildInFunctionTransformer(IDimensionEncodingMap dimEncMap) {
- this.dimEncMap = dimEncMap;
- }
-
- @Override
- public TupleFilter transform(TupleFilter tupleFilter) {
- TupleFilter translated = null;
- if (tupleFilter instanceof CompareTupleFilter) {
- //normal case
- translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
- if (translated != null) {
- logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
- }
- } else if (tupleFilter instanceof BuildInFunctionTupleFilter) {
- //like case
- translated = translateFunctionTupleFilter((BuildInFunctionTupleFilter) tupleFilter);
- if (translated != null) {
- logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
- }
- } else if (tupleFilter instanceof LogicalTupleFilter) {
- @SuppressWarnings("unchecked")
- ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
- while (childIterator.hasNext()) {
- TupleFilter transformed = transform(childIterator.next());
- if (transformed != null) {
- childIterator.set(transformed);
- } else {
- throw new IllegalStateException("Should not be null");
- }
- }
- }
- return translated == null ? tupleFilter : translated;
- }
-
- private TupleFilter translateFunctionTupleFilter(BuildInFunctionTupleFilter buildInFunctionTupleFilter) {
- if (!buildInFunctionTupleFilter.isValid())
- return null;
-
- TblColRef columnRef = buildInFunctionTupleFilter.getColumn();
- Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
- if (dict == null)
- return null;
-
- CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
- translated.addChild(new ColumnTupleFilter(columnRef));
-
- try {
- for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
- Object dictVal = dict.getValueFromId(i);
- if ((Boolean) buildInFunctionTupleFilter.invokeFunction(dictVal)) {
- translated.addChild(new ConstantTupleFilter(dictVal));
- }
- }
- } catch (Exception e) {
- logger.debug(e.getMessage());
- return null;
- }
- return translated;
- }
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
- if (compTupleFilter.getFunction() == null || (!(compTupleFilter.getFunction() instanceof BuildInFunctionTupleFilter)))
- return null;
-
- BuildInFunctionTupleFilter buildInFunctionTupleFilter = (BuildInFunctionTupleFilter) compTupleFilter.getFunction();
-
- if (!buildInFunctionTupleFilter.isValid())
- return null;
-
- TblColRef columnRef = buildInFunctionTupleFilter.getColumn();
- Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
- if (dict == null)
- return null;
-
- CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
- translated.addChild(new ColumnTupleFilter(columnRef));
-
- try {
- Collection<Object> inValues = Lists.newArrayList();
- for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
- Object dictVal = dict.getValueFromId(i);
- Object computedVal = buildInFunctionTupleFilter.invokeFunction(dictVal);
- Class clazz = Primitives.wrap(computedVal.getClass());
- Object targetVal = compTupleFilter.getFirstValue();
- if (Primitives.isWrapperType(clazz))
- targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
-
- int comp = ((Comparable) computedVal).compareTo(targetVal);
- boolean compResult = false;
- switch (compTupleFilter.getOperator()) {
- case EQ:
- compResult = comp == 0;
- break;
- case NEQ:
- compResult = comp != 0;
- break;
- case LT:
- compResult = comp < 0;
- break;
- case LTE:
- compResult = comp <= 0;
- break;
- case GT:
- compResult = comp > 0;
- break;
- case GTE:
- compResult = comp >= 0;
- break;
- case IN:
- compResult = compTupleFilter.getValues().contains(computedVal.toString());
- break;
- case NOTIN:
- compResult = !compTupleFilter.getValues().contains(computedVal.toString());
- break;
- default:
- break;
- }
- if (compResult) {
- inValues.add(dictVal);
- }
- }
- translated.addChild(new ConstantTupleFilter(inValues));
- } catch (Exception e) {
- logger.debug(e.getMessage());
- return null;
- }
- return translated;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
new file mode 100644
index 0000000..9707f8f
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dict;
+
+import java.util.Collection;
+import java.util.ListIterator;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.IDimensionEncodingMap;
+import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Primitives;
+
+/**
+ * only take effect when the compare filter has function
+ */
+public class BuiltInFunctionTransformer implements ITupleFilterTransformer {
+ public static final Logger logger = LoggerFactory.getLogger(BuiltInFunctionTransformer.class);
+
+ private IDimensionEncodingMap dimEncMap;
+
+ public BuiltInFunctionTransformer(IDimensionEncodingMap dimEncMap) {
+ this.dimEncMap = dimEncMap;
+ }
+
+ @Override
+ public TupleFilter transform(TupleFilter tupleFilter) {
+ TupleFilter translated = null;
+ if (tupleFilter instanceof CompareTupleFilter) {
+ //normal case
+ translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
+ if (translated != null) {
+ logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+ }
+ } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
+ //like case
+ translated = translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
+ if (translated != null) {
+ logger.info("Translated {" + tupleFilter + "} to IN clause: {" + translated + "}");
+ }
+ } else if (tupleFilter instanceof LogicalTupleFilter) {
+ @SuppressWarnings("unchecked")
+ ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
+ while (childIterator.hasNext()) {
+ TupleFilter transformed = transform(childIterator.next());
+ if (transformed != null) {
+ childIterator.set(transformed);
+ } else {
+ throw new IllegalStateException("Should not be null");
+ }
+ }
+ }
+ return translated == null ? tupleFilter : translated;
+ }
+
+ private TupleFilter translateFunctionTupleFilter(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) {
+ if (!builtInFunctionTupleFilter.isValid())
+ return null;
+
+ TblColRef columnRef = builtInFunctionTupleFilter.getColumn();
+ Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
+ if (dict == null)
+ return null;
+
+ CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+ translated.addChild(new ColumnTupleFilter(columnRef));
+
+ try {
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ Object dictVal = dict.getValueFromId(i);
+ if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) {
+ translated.addChild(new ConstantTupleFilter(dictVal));
+ }
+ }
+ } catch (Exception e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return translated;
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
+ if (compTupleFilter.getFunction() == null || (!(compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter)))
+ return null;
+
+ BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) compTupleFilter.getFunction();
+
+ if (!builtInFunctionTupleFilter.isValid())
+ return null;
+
+ TblColRef columnRef = builtInFunctionTupleFilter.getColumn();
+ Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
+ if (dict == null)
+ return null;
+
+ CompareTupleFilter translated = new CompareTupleFilter(FilterOperatorEnum.IN);
+ translated.addChild(new ColumnTupleFilter(columnRef));
+
+ try {
+ Collection<Object> inValues = Lists.newArrayList();
+ for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
+ Object dictVal = dict.getValueFromId(i);
+ Object computedVal = builtInFunctionTupleFilter.invokeFunction(dictVal);
+ Class clazz = Primitives.wrap(computedVal.getClass());
+ Object targetVal = compTupleFilter.getFirstValue();
+ if (Primitives.isWrapperType(clazz))
+ targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, compTupleFilter.getFirstValue()));
+
+ int comp = ((Comparable) computedVal).compareTo(targetVal);
+ boolean compResult = false;
+ switch (compTupleFilter.getOperator()) {
+ case EQ:
+ compResult = comp == 0;
+ break;
+ case NEQ:
+ compResult = comp != 0;
+ break;
+ case LT:
+ compResult = comp < 0;
+ break;
+ case LTE:
+ compResult = comp <= 0;
+ break;
+ case GT:
+ compResult = comp > 0;
+ break;
+ case GTE:
+ compResult = comp >= 0;
+ break;
+ case IN:
+ compResult = compTupleFilter.getValues().contains(computedVal.toString());
+ break;
+ case NOTIN:
+ compResult = !compTupleFilter.getValues().contains(computedVal.toString());
+ break;
+ default:
+ break;
+ }
+ if (compResult) {
+ inValues.add(dictVal);
+ }
+ }
+ translated.addChild(new ConstantTupleFilter(inValues));
+ } catch (Exception e) {
+ logger.debug(e.getMessage());
+ return null;
+ }
+ return translated;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java
deleted file mode 100644
index c15b083..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuildInFunctionTupleFilter.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-import java.io.Serializable;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.filter.function.BuiltInMethod;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Primitives;
-
-public class BuildInFunctionTupleFilter extends FunctionTupleFilter {
- public static final Logger logger = LoggerFactory.getLogger(BuildInFunctionTupleFilter.class);
-
- private String name;
- // FIXME Only supports single parameter functions currently
- private TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower()))
- private int colPosition;
- private Method method;
- private List<Serializable> methodParams;
- private boolean isValid = false;
-
- public BuildInFunctionTupleFilter(String name) {
- super(Lists.<TupleFilter> newArrayList(), FilterOperatorEnum.FUNCTION);
- this.methodParams = Lists.newArrayList();
-
- if (name != null) {
- this.name = name.toUpperCase();
- initMethod();
- }
- }
-
- public String getName() {
- return name;
- }
-
- public TblColRef getColumn() {
- if (columnContainerFilter == null)
- return null;
-
- if (columnContainerFilter instanceof ColumnTupleFilter)
- return ((ColumnTupleFilter) columnContainerFilter).getColumn();
- else if (columnContainerFilter instanceof BuildInFunctionTupleFilter)
- return ((BuildInFunctionTupleFilter) columnContainerFilter).getColumn();
-
- throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter.");
- }
-
- public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
- if (columnContainerFilter instanceof ColumnTupleFilter)
- methodParams.set(colPosition, (Serializable) input);
- else if (columnContainerFilter instanceof BuildInFunctionTupleFilter)
- methodParams.set(colPosition, (Serializable) ((BuildInFunctionTupleFilter) columnContainerFilter).invokeFunction(input));
- return method.invoke(null, (Object[]) (methodParams.toArray()));
- }
-
- public boolean isValid() {
- return isValid && method != null && methodParams.size() == children.size();
- }
-
- @Override
- public void addChild(TupleFilter child) {
- if (child instanceof ColumnTupleFilter || child instanceof BuildInFunctionTupleFilter) {
- columnContainerFilter = child;
- colPosition = methodParams.size();
- methodParams.add(null);
- } else if (child instanceof ConstantTupleFilter) {
- Serializable constVal = (Serializable) child.getValues().iterator().next();
- try {
- Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]);
- if (!Primitives.isWrapperType(clazz))
- methodParams.add(constVal);
- else
- methodParams.add((Serializable) clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, constVal)));
- } catch (Exception e) {
- logger.warn(e.getMessage());
- isValid = false;
- }
- }
- super.addChild(child);
- }
-
- @Override
- public boolean isEvaluable() {
- return false;
- }
-
- @Override
- public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
- throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
- }
-
- @Override
- public Collection<String> getValues() {
- throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
- }
-
- @Override
- public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- BytesUtil.writeUTFString(name, buffer);
- BytesUtil.writeVInt(colPosition, buffer);
- BytesUtil.writeVInt(isValid ? 1 : 0, buffer);
- }
-
- @Override
- public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
-
- this.name = BytesUtil.readUTFString(buffer);
- this.initMethod();
-
- this.colPosition = BytesUtil.readVInt(buffer);
- this.isValid = BytesUtil.readVInt(buffer) == 1;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(name);
- sb.append("(");
- for (int i = 0; i < methodParams.size(); i++) {
- if (colPosition == i) {
- sb.append(columnContainerFilter);
- } else {
- sb.append(methodParams.get(i));
- }
- if (i < methodParams.size() - 1)
- sb.append(",");
- }
- sb.append(")");
- return sb.toString();
- }
-
- private void initMethod() {
- if (BuiltInMethod.MAP.containsKey(name)) {
- this.method = BuiltInMethod.MAP.get(name).method;
- isValid = true;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
new file mode 100644
index 0000000..5a10371
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.io.Serializable;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.kylin.metadata.filter.function.BuiltInMethod;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Primitives;
+
+public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
+ public static final Logger logger = LoggerFactory.getLogger(BuiltInFunctionTupleFilter.class);
+
+ protected String name;
+ // FIXME Only supports single parameter functions currently
+ protected TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower()))
+ protected int colPosition;
+ protected Method method;
+ protected List<Serializable> methodParams;
+ protected boolean isValidFunc = false;
+
+ public BuiltInFunctionTupleFilter(String name) {
+ super(Lists.<TupleFilter> newArrayList(), FilterOperatorEnum.FUNCTION);
+ this.methodParams = Lists.newArrayList();
+
+ if (name != null) {
+ this.name = name.toUpperCase();
+ initMethod();
+ }
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public TblColRef getColumn() {
+ if (columnContainerFilter == null)
+ return null;
+
+ if (columnContainerFilter instanceof ColumnTupleFilter)
+ return ((ColumnTupleFilter) columnContainerFilter).getColumn();
+ else if (columnContainerFilter instanceof BuiltInFunctionTupleFilter)
+ return ((BuiltInFunctionTupleFilter) columnContainerFilter).getColumn();
+
+ throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter.");
+ }
+
+ public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
+ if (columnContainerFilter instanceof ColumnTupleFilter)
+ methodParams.set(colPosition, (Serializable) input);
+ else if (columnContainerFilter instanceof BuiltInFunctionTupleFilter)
+ methodParams.set(colPosition, (Serializable) ((BuiltInFunctionTupleFilter) columnContainerFilter).invokeFunction(input));
+ return method.invoke(null, (Object[]) (methodParams.toArray()));
+ }
+
+ public boolean isValid() {
+ return isValidFunc && method != null && methodParams.size() == children.size();
+ }
+
+ @Override
+ public void addChild(TupleFilter child) {
+ if (child instanceof ColumnTupleFilter || child instanceof BuiltInFunctionTupleFilter) {
+ columnContainerFilter = child;
+ colPosition = methodParams.size();
+ methodParams.add(null);
+ } else if (child instanceof ConstantTupleFilter) {
+ Serializable constVal = (Serializable) child.getValues().iterator().next();
+ try {
+ Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]);
+ if (!Primitives.isWrapperType(clazz))
+ methodParams.add(constVal);
+ else
+ methodParams.add((Serializable) clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, constVal)));
+ } catch (Exception e) {
+ logger.warn(e.getMessage());
+ isValidFunc = false;
+ }
+ }
+ super.addChild(child);
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return false;
+ }
+
+ @Override
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem<?> cs) {
+ throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
+ }
+
+ @Override
+ public Collection<String> getValues() {
+ throw new UnsupportedOperationException("Function filter cannot be evaluated immediately");
+ }
+
+ @Override
+ public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ throw new UnsupportedOperationException("Function filter cannot serialized");
+ }
+
+ @Override
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ throw new UnsupportedOperationException("Function filter cannot serialized");
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(name);
+ sb.append("(");
+ for (int i = 0; i < methodParams.size(); i++) {
+ if (colPosition == i) {
+ sb.append(columnContainerFilter);
+ } else {
+ sb.append(methodParams.get(i));
+ }
+ if (i < methodParams.size() - 1)
+ sb.append(",");
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+
+ protected void initMethod() {
+ if (BuiltInMethod.MAP.containsKey(name)) {
+ this.method = BuiltInMethod.MAP.get(name).method;
+ isValidFunc = true;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 4647c51..3250640 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -80,7 +80,6 @@ public abstract class TupleFilter {
protected final List<TupleFilter> children;
protected FilterOperatorEnum operator;
- protected boolean hasChildren;
protected TupleFilter(List<TupleFilter> filters, FilterOperatorEnum op) {
this.children = filters;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index bcb005f..f6c8bc4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -186,7 +186,7 @@ public class TupleFilterSerializer {
filter = new DynamicTupleFilter(null);
break;
case FUNCTION:
- filter = new BuildInFunctionTupleFilter(null);
+ filter = new BuiltInFunctionTupleFilter(null);
break;
case MASSIN:
filter = new MassInTupleFilter();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java
index 401c90b..861e530 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/Functions.java
@@ -21,7 +21,7 @@ package org.apache.kylin.metadata.filter.function;
import java.lang.reflect.InvocationTargetException;
import java.util.Map;
-import org.apache.kylin.metadata.filter.BuildInFunctionTupleFilter;
+import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
@@ -54,7 +54,7 @@ public class Functions {
}
}
- return new BuildInFunctionTupleFilter(name);
+ return new BuiltInFunctionTupleFilter(name);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 6ed7d3b..3b9d9c6 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.gridtable.CubeScanRangePlanner;
-import org.apache.kylin.dict.BuildInFunctionTransformer;
+import org.apache.kylin.dict.BuiltInFunctionTransformer;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
@@ -64,7 +64,7 @@ public class CubeSegmentScanner implements IGTScanner {
byte[] serialize = TupleFilterSerializer.serialize(originalfilter, StringCodeSystem.INSTANCE);
TupleFilter filter = TupleFilterSerializer.deserialize(serialize, StringCodeSystem.INSTANCE);
// translate FunctionTupleFilter to IN clause
- ITupleFilterTransformer translator = new BuildInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
+ ITupleFilterTransformer translator = new BuiltInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
filter = translator.transform(filter);
String plannerName = KylinConfig.getInstanceFromEnv().getQueryStorageVisitPlanner();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/kylin-it/src/test/resources/query/sql_like/query03.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query03.sql b/kylin-it/src/test/resources/query/sql_like/query03.sql
new file mode 100644
index 0000000..ae095fc
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query03.sql
@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- test case for KYLIN-1954
+
+select lstg_format_name as lstg_format_name, count(*) as cnt
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where lstg_format_name like '%BIN%' and lstg_format_name > 'A'
+group by lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c23313e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
index 01bd4e6..ea4b504 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/FilterDecorator.java
@@ -24,7 +24,7 @@ import java.util.Set;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.kv.RowKeyColumnIO;
-import org.apache.kylin.dict.BuildInFunctionTransformer;
+import org.apache.kylin.dict.BuiltInFunctionTransformer;
import org.apache.kylin.dict.DictCodeSystem;
import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.dimension.IDimensionEncodingMap;
@@ -151,7 +151,7 @@ public class FilterDecorator implements TupleFilterSerializer.Decorator {
if (filter == null)
return null;
- BuildInFunctionTransformer translator = new BuildInFunctionTransformer(dimEncMap);
+ BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap);
filter = translator.transform(filter);
// un-evaluatable filter is replaced with TRUE
[31/50] [abbrv] kylin git commit: minor
Posted by li...@apache.org.
minor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3cb0894c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3cb0894c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3cb0894c
Branch: refs/heads/1.5.x-CDH5.7
Commit: 3cb0894c4ff329d985513775c172ee47ed05ac67
Parents: a77fbc8
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Aug 31 14:36:26 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Aug 31 14:36:41 2016 +0800
----------------------------------------------------------------------
.../filter/EvaluatableLikeFunction.java | 26 +++++++++++++++++---
1 file changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3cb0894c/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
index 59961ac..28e544b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
@@ -27,8 +27,13 @@ import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.StringSerializer;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+/**
+ * typically like will be handled by BuiltInFunctionTupleFilter rather than this
+ */
public class EvaluatableLikeFunction extends BuiltInFunctionTupleFilter {
+ private boolean patternExtracted;
+
public EvaluatableLikeFunction(String name) {
super(name, FilterOperatorEnum.LIKE);
}
@@ -86,11 +91,24 @@ public class EvaluatableLikeFunction extends BuiltInFunctionTupleFilter {
return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
}
+ @Override
+ public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
+ if (!patternExtracted) {
+ this.getLikePattern();
+ }
+ return super.invokeFunction(input);
+ }
+
+ //will replace the ByteArray pattern to String type
public String getLikePattern() {
- ByteArray byteArray = (ByteArray) methodParams.get(1);
- StringSerializer s = new StringSerializer(DataType.getType("string"));
- String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length()));
- return pattern;
+ if (!patternExtracted) {
+ ByteArray byteArray = (ByteArray) methodParams.get(1);
+ StringSerializer s = new StringSerializer(DataType.getType("string"));
+ String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length()));
+ methodParams.set(1, pattern);
+ patternExtracted = true;
+ }
+ return (String) methodParams.get(1);
}
}
[02/50] [abbrv] kylin git commit: KYLIN-1966 futher refactor,
decouple flat table and cube
Posted by li...@apache.org.
KYLIN-1966 futher refactor, decouple flat table and cube
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/29ba46be
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/29ba46be
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/29ba46be
Branch: refs/heads/1.5.x-CDH5.7
Commit: 29ba46beee3c9330c54a23265fdc183333f750d8
Parents: 1b34c38
Author: Yang Li <li...@apache.org>
Authored: Mon Aug 22 08:37:12 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Aug 22 08:37:12 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeSegment.java | 8 +-
.../inmemcubing/AbstractInMemCubeBuilder.java | 7 +-
.../cube/inmemcubing/DoggedCubeBuilder.java | 7 +-
.../cube/inmemcubing/InMemCubeBuilder.java | 7 +-
.../InMemCubeBuilderInputConverter.java | 15 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 3 +-
.../cube/model/CubeJoinedFlatTableDesc.java | 42 +-----
.../cube/model/CubeJoinedFlatTableEnrich.java | 140 +++++++++++++++++++
.../org/apache/kylin/cube/util/CubingUtils.java | 7 +-
.../org/apache/kylin/engine/EngineFactory.java | 12 ++
.../apache/kylin/engine/IBatchCubingEngine.java | 8 ++
.../apache/kylin/job/JoinedFlatTableTest.java | 13 +-
.../metadata/model/IJoinedFlatTableDesc.java | 2 +
.../kylin/engine/mr/MRBatchCubingEngine.java | 13 ++
.../kylin/engine/mr/MRBatchCubingEngine2.java | 13 ++
.../java/org/apache/kylin/engine/mr/MRUtil.java | 5 +-
.../engine/mr/steps/BaseCuboidMapperBase.java | 7 +-
.../mr/steps/FactDistinctColumnsMapperBase.java | 7 +-
.../engine/mr/steps/InMemCuboidMapper.java | 5 +-
.../engine/spark/SparkBatchCubingEngine.java | 12 ++
.../apache/kylin/engine/spark/SparkCubing.java | 29 ++--
.../engine/spark/SparkCubingJobBuilder.java | 5 +-
engine-streaming/pom.xml | 4 +
.../streaming/cube/StreamingCubeBuilder.java | 9 +-
.../ITDoggedCubeBuilderStressTest.java | 5 +-
.../inmemcubing/ITDoggedCubeBuilderTest.java | 7 +-
.../inmemcubing/ITInMemCubeBuilderTest.java | 19 +--
.../kylin/rest/controller/CubeController.java | 5 +-
.../kylin/source/kafka/KafkaStreamingInput.java | 4 +-
29 files changed, 310 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 0797ab3..aaa88f1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -35,18 +35,16 @@ import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.IBuildable;
-import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -530,10 +528,6 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable {
return cubeInstance;
}
- public IJoinedFlatTableDesc getJoinedFlatTableDesc() {
- return new CubeJoinedFlatTableDesc(this);
- }
-
public String getIndexPath() {
return indexPath;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
index d417d11..0bfaab3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/AbstractInMemCubeBuilder.java
@@ -28,6 +28,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanRequestBuilder;
import org.apache.kylin.gridtable.GridTable;
import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,18 +40,22 @@ abstract public class AbstractInMemCubeBuilder {
private static Logger logger = LoggerFactory.getLogger(AbstractInMemCubeBuilder.class);
+ final protected IJoinedFlatTableDesc flatDesc;
final protected CubeDesc cubeDesc;
final protected Map<TblColRef, Dictionary<String>> dictionaryMap;
protected int taskThreadCount = 4;
protected int reserveMemoryMB = 100;
- public AbstractInMemCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ public AbstractInMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ if (flatDesc == null)
+ throw new NullPointerException();
if (cubeDesc == null)
throw new NullPointerException();
if (dictionaryMap == null)
throw new IllegalArgumentException("dictionary cannot be null");
+ this.flatDesc = flatDesc;
this.cubeDesc = cubeDesc;
this.dictionaryMap = dictionaryMap;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
index 15f2241..69f1f82 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/DoggedCubeBuilder.java
@@ -38,6 +38,7 @@ import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequestBuilder;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.measure.MeasureAggregators;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,8 +55,8 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
private int splitRowThreshold = Integer.MAX_VALUE;
private int unitRows = 1000;
- public DoggedCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
- super(cubeDesc, dictionaryMap);
+ public DoggedCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ super(cubeDesc, flatDesc, dictionaryMap);
// check memory more often if a single row is big
if (cubeDesc.hasMemoryHungryMeasures())
@@ -270,7 +271,7 @@ public class DoggedCubeBuilder extends AbstractInMemCubeBuilder {
RuntimeException exception;
public SplitThread() {
- this.builder = new InMemCubeBuilder(cubeDesc, dictionaryMap);
+ this.builder = new InMemCubeBuilder(cubeDesc, flatDesc, dictionaryMap);
this.builder.setConcurrentThreads(taskThreadCount);
this.builder.setReserveMemoryMB(reserveMemoryMB);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 36d1296..e4908b8 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -48,6 +48,7 @@ import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.measure.topn.Counter;
import org.apache.kylin.measure.topn.TopNCounter;
import org.apache.kylin.metadata.datatype.DoubleMutable;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -86,8 +87,8 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
private Object[] totalSumForSanityCheck;
private ICuboidCollector resultCollector;
- public InMemCubeBuilder(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
- super(cubeDesc, dictionaryMap);
+ public InMemCubeBuilder(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ super(cubeDesc, flatDesc, dictionaryMap);
this.cuboidScheduler = new CuboidScheduler(cubeDesc);
this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
this.totalCuboidCount = cuboidScheduler.getCuboidCount();
@@ -514,7 +515,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
this.info = info;
this.input = input;
this.record = new GTRecord(info);
- this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc, dictionaryMap, info);
+ this.inMemCubeBuilderInputConverter = new InMemCubeBuilderInputConverter(cubeDesc, flatDesc, dictionaryMap, info);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
index 607f6bb..ab44f63 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilderInputConverter.java
@@ -23,11 +23,12 @@ import java.util.Map;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -40,7 +41,7 @@ public class InMemCubeBuilderInputConverter {
public static final byte[] HIVE_NULL = Bytes.toBytes("\\N");
- private final CubeJoinedFlatTableDesc intermediateTableDesc;
+ private final CubeJoinedFlatTableEnrich flatDesc;
private final MeasureDesc[] measureDescs;
private final MeasureIngester<?>[] measureIngesters;
private final int measureCount;
@@ -48,9 +49,9 @@ public class InMemCubeBuilderInputConverter {
private final GTInfo gtInfo;
protected List<byte[]> nullBytes;
- public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, Map<TblColRef, Dictionary<String>> dictionaryMap, GTInfo gtInfo) {
+ public InMemCubeBuilderInputConverter(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDesc, Map<TblColRef, Dictionary<String>> dictionaryMap, GTInfo gtInfo) {
this.gtInfo = gtInfo;
- this.intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc);
+ this.flatDesc = new CubeJoinedFlatTableEnrich(flatDesc, cubeDesc);
this.measureCount = cubeDesc.getMeasures().size();
this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
this.measureIngesters = MeasureIngester.create(cubeDesc.getMeasures());
@@ -74,11 +75,11 @@ public class InMemCubeBuilderInputConverter {
}
private Object[] buildKey(List<String> row) {
- int keySize = intermediateTableDesc.getRowKeyColumnIndexes().length;
+ int keySize = flatDesc.getRowKeyColumnIndexes().length;
Object[] key = new Object[keySize];
for (int i = 0; i < keySize; i++) {
- key[i] = row.get(intermediateTableDesc.getRowKeyColumnIndexes()[i]);
+ key[i] = row.get(flatDesc.getRowKeyColumnIndexes()[i]);
if (key[i] != null && isNull(Bytes.toBytes((String) key[i]))) {
key[i] = null;
}
@@ -98,7 +99,7 @@ public class InMemCubeBuilderInputConverter {
private Object buildValueOf(int idxOfMeasure, List<String> row) {
MeasureDesc measure = measureDescs[idxOfMeasure];
FunctionDesc function = measure.getFunction();
- int[] colIdxOnFlatTable = intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];
+ int[] colIdxOnFlatTable = flatDesc.getMeasureColumnIndexes()[idxOfMeasure];
int paramCount = function.getParameterCount();
String[] inputToMeasure = new String[paramCount];
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 2c83972..30290cd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -78,7 +78,7 @@ import com.google.common.collect.Sets;
*/
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeDesc extends RootPersistentEntity {
+public class CubeDesc extends RootPersistentEntity implements IEngineAware {
private static final Logger logger = LoggerFactory.getLogger(CubeDesc.class);
public static class CannotFilterExtendedColumnException extends RuntimeException {
@@ -1004,6 +1004,7 @@ public class CubeDesc extends RootPersistentEntity {
this.storageType = storageType;
}
+ @Override
public int getEngineType() {
return engineType;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index 1f9d772..6aeb617 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.kylin.common.util.BytesSplitter;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
@@ -42,8 +41,6 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
private final CubeSegment cubeSegment;
private int columnCount;
- private int[] rowKeyColumnIndexes; // the column index on flat table
- private int[][] measureColumnIndexes; // [i] is the i.th measure related column index on flat table
private List<TblColRef> columnList = Lists.newArrayList();
private Map<TblColRef, Integer> columnIndexMap;
@@ -65,9 +62,6 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
// check what columns from hive tables are required, and index them
private void parseCubeDesc() {
- long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
- Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
-
if (cubeSegment == null) {
this.tableName = "kylin_intermediate_" + cubeDesc.getName();
} else {
@@ -81,34 +75,15 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
columnIndex++;
}
- // build index for rowkey columns
- List<TblColRef> cuboidColumns = baseCuboid.getColumns();
- int rowkeyColCount = cubeDesc.getRowkey().getRowKeyColumns().length;
- rowKeyColumnIndexes = new int[rowkeyColCount];
- for (int i = 0; i < rowkeyColCount; i++) {
- TblColRef col = cuboidColumns.get(i);
- Integer dimIdx = columnIndexMap.get(col);
- if (dimIdx == null) {
- throw new RuntimeException("Can't find column " + col);
- }
- rowKeyColumnIndexes[i] = dimIdx;
- }
-
List<MeasureDesc> measures = cubeDesc.getMeasures();
int measureSize = measures.size();
- measureColumnIndexes = new int[measureSize][];
for (int i = 0; i < measureSize; i++) {
FunctionDesc func = measures.get(i).getFunction();
List<TblColRef> colRefs = func.getParameter().getColRefs();
- if (colRefs == null) {
- measureColumnIndexes[i] = null;
- } else {
- measureColumnIndexes[i] = new int[colRefs.size()];
+ if (colRefs != null) {
for (int j = 0; j < colRefs.size(); j++) {
TblColRef c = colRefs.get(j);
- measureColumnIndexes[i][j] = columnList.indexOf(c);
- if (measureColumnIndexes[i][j] < 0) {
- measureColumnIndexes[i][j] = columnIndex;
+ if (columnList.indexOf(c) < 0) {
columnIndexMap.put(c, columnIndex);
columnList.add(c);
columnIndex++;
@@ -148,18 +123,6 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
// TODO: check data types here
}
- public CubeDesc getCubeDesc() {
- return cubeDesc;
- }
-
- public int[] getRowKeyColumnIndexes() {
- return rowKeyColumnIndexes;
- }
-
- public int[][] getMeasureColumnIndexes() {
- return measureColumnIndexes;
- }
-
@Override
public String getTableName() {
return tableName;
@@ -175,6 +138,7 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
return cubeDesc.getModel();
}
+ @Override
public int getColumnIndex(TblColRef colRef) {
Integer index = columnIndexMap.get(colRef);
if (index == null)
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
new file mode 100644
index 0000000..5212859
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model;
+
+import java.util.List;
+
+import org.apache.kylin.common.util.BytesSplitter;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+/**
+ * An enrich of IJoinedFlatTableDesc for cubes
+ */
+public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc {
+
+ private CubeDesc cubeDesc;
+ private IJoinedFlatTableDesc flatDesc;
+ private int[] rowKeyColumnIndexes; // the column index on flat table
+ private int[][] measureColumnIndexes; // [i] is the i.th measure related column index on flat table
+
+ public CubeJoinedFlatTableEnrich(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) {
+ // != works due to object cache
+ if (cubeDesc.getModel() != flatDesc.getDataModel())
+ throw new IllegalArgumentException();
+
+ this.cubeDesc = cubeDesc;
+ this.flatDesc = flatDesc;
+ parseCubeDesc();
+ }
+
+ // check what columns from hive tables are required, and index them
+ private void parseCubeDesc() {
+ long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
+ Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
+
+ // build index for rowkey columns
+ List<TblColRef> cuboidColumns = baseCuboid.getColumns();
+ int rowkeyColCount = cubeDesc.getRowkey().getRowKeyColumns().length;
+ rowKeyColumnIndexes = new int[rowkeyColCount];
+ for (int i = 0; i < rowkeyColCount; i++) {
+ TblColRef col = cuboidColumns.get(i);
+ rowKeyColumnIndexes[i] = flatDesc.getColumnIndex(col);
+ }
+
+ List<MeasureDesc> measures = cubeDesc.getMeasures();
+ int measureSize = measures.size();
+ measureColumnIndexes = new int[measureSize][];
+ for (int i = 0; i < measureSize; i++) {
+ FunctionDesc func = measures.get(i).getFunction();
+ List<TblColRef> colRefs = func.getParameter().getColRefs();
+ if (colRefs == null) {
+ measureColumnIndexes[i] = null;
+ } else {
+ measureColumnIndexes[i] = new int[colRefs.size()];
+ for (int j = 0; j < colRefs.size(); j++) {
+ TblColRef c = colRefs.get(j);
+ measureColumnIndexes[i][j] = flatDesc.getColumnIndex(c);
+ }
+ }
+ }
+ }
+
+ // sanity check the input record (in bytes) matches what's expected
+ public void sanityCheck(BytesSplitter bytesSplitter) {
+ int columnCount = flatDesc.getAllColumns().size();
+ if (columnCount != bytesSplitter.getBufferSize()) {
+ throw new IllegalArgumentException("Expect " + columnCount + " columns, but see " + bytesSplitter.getBufferSize() + " -- " + bytesSplitter);
+ }
+
+ // TODO: check data types here
+ }
+
+ public CubeDesc getCubeDesc() {
+ return cubeDesc;
+ }
+
+ public int[] getRowKeyColumnIndexes() {
+ return rowKeyColumnIndexes;
+ }
+
+ public int[][] getMeasureColumnIndexes() {
+ return measureColumnIndexes;
+ }
+
+ @Override
+ public String getTableName() {
+ return flatDesc.getTableName();
+ }
+
+ @Override
+ public List<TblColRef> getAllColumns() {
+ return flatDesc.getAllColumns();
+ }
+
+ @Override
+ public DataModelDesc getDataModel() {
+ return flatDesc.getDataModel();
+ }
+
+ @Override
+ public int getColumnIndex(TblColRef colRef) {
+ return flatDesc.getColumnIndex(colRef);
+ }
+
+ @Override
+ public long getSourceOffsetStart() {
+ return flatDesc.getSourceOffsetStart();
+ }
+
+ @Override
+ public long getSourceOffsetEnd() {
+ return flatDesc.getSourceOffsetEnd();
+ }
+
+ @Override
+ public TblColRef getDistributedBy() {
+ return flatDesc.getDistributedBy();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index b7f79e1..aa4610f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -48,12 +48,13 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
import org.slf4j.Logger;
@@ -74,8 +75,8 @@ public class CubingUtils {
private static Logger logger = LoggerFactory.getLogger(CubingUtils.class);
- public static Map<Long, HyperLogLogPlusCounter> sampling(CubeDesc cubeDesc, Iterable<List<String>> streams) {
- CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(cubeDesc);
+ public static Map<Long, HyperLogLogPlusCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) {
+ final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc);
final int rowkeyLength = cubeDesc.getRowkey().getRowKeyColumns().length;
final List<Long> allCuboidIds = new CuboidScheduler(cubeDesc).getAllCuboidIds();
final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
index 79d2f81..7044a3e 100644
--- a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
+++ b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
@@ -23,8 +23,10 @@ import java.util.Map;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ImplementationSwitch;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.metadata.model.IEngineAware;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
public class EngineFactory {
@@ -46,6 +48,16 @@ public class EngineFactory {
return streamingEngines.get(aware.getEngineType());
}
+ /** Mark deprecated to indicate for test purpose only */
+ @Deprecated
+ public static IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) {
+ return batchEngine(cubeDesc).getJoinedFlatTableDesc(cubeDesc);
+ }
+
+ public static IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) {
+ return batchEngine(newSegment).getJoinedFlatTableDesc(newSegment);
+ }
+
/** Build a new cube segment, typically its time range appends to the end of current cube. */
public static DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) {
return batchEngine(newSegment).createBatchCubingJob(newSegment, submitter);
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java b/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
index 556893c..754dbde 100644
--- a/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
+++ b/core-job/src/main/java/org/apache/kylin/engine/IBatchCubingEngine.java
@@ -19,9 +19,17 @@
package org.apache.kylin.engine;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
public interface IBatchCubingEngine {
+
+ /** Mark deprecated to indicate for test purpose only */
+ @Deprecated
+ public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc);
+
+ public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment);
/** Build a new cube segment, typically its time range appends to the end of current cube. */
public DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter);
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java b/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
index 17a7178..0faf22a 100644
--- a/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/JoinedFlatTableTest.java
@@ -27,8 +27,9 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
@@ -42,7 +43,7 @@ import org.junit.Test;
public class JoinedFlatTableTest extends LocalFileMetadataTestCase {
CubeInstance cube = null;
- CubeJoinedFlatTableDesc intermediateTableDesc = null;
+ IJoinedFlatTableDesc flatTableDesc = null;
String fakeJobUUID = "abc-def";
CubeSegment cubeSegment = null;
@@ -51,7 +52,7 @@ public class JoinedFlatTableTest extends LocalFileMetadataTestCase {
this.createTestMetadata();
cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready");
cubeSegment = cube.getSegments().get(0);
- intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeSegment);
+ flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
}
@After
@@ -61,7 +62,7 @@ public class JoinedFlatTableTest extends LocalFileMetadataTestCase {
@Test
public void testGenCreateTableDDL() {
- String ddl = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, "/tmp");
+ String ddl = JoinedFlatTable.generateCreateTableStatement(flatTableDesc, "/tmp");
System.out.println(ddl);
System.out.println("The length for the ddl is " + ddl.length());
@@ -69,14 +70,14 @@ public class JoinedFlatTableTest extends LocalFileMetadataTestCase {
@Test
public void testGenDropTableDDL() {
- String ddl = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc);
+ String ddl = JoinedFlatTable.generateDropTableStatement(flatTableDesc);
System.out.println(ddl);
assertEquals(101, ddl.length());
}
@Test
public void testGenerateInsertSql() throws IOException {
- String sqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
+ String sqls = JoinedFlatTable.generateInsertDataStatement(flatTableDesc, new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
System.out.println(sqls);
int length = sqls.length();
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
index 55ea71f..f3a4107 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
@@ -29,6 +29,8 @@ public interface IJoinedFlatTableDesc {
DataModelDesc getDataModel();
List<TblColRef> getAllColumns();
+
+ int getColumnIndex(TblColRef colRef);
long getSourceOffsetStart();
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
index 5198db1..681c545 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine.java
@@ -19,12 +19,25 @@
package org.apache.kylin.engine.mr;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.engine.IBatchCubingEngine;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
public class MRBatchCubingEngine implements IBatchCubingEngine {
@Override
+ public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) {
+ return new CubeJoinedFlatTableDesc(cubeDesc);
+ }
+
+ @Override
+ public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) {
+ return new CubeJoinedFlatTableDesc(newSegment);
+ }
+
+ @Override
public DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) {
return new BatchCubingJobBuilder(newSegment, submitter).build();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java
index b3af7d7..d9fdcb9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRBatchCubingEngine2.java
@@ -19,12 +19,25 @@
package org.apache.kylin.engine.mr;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.engine.IBatchCubingEngine;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
public class MRBatchCubingEngine2 implements IBatchCubingEngine {
@Override
+ public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) {
+ return new CubeJoinedFlatTableDesc(cubeDesc);
+ }
+
+ @Override
+ public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) {
+ return new CubeJoinedFlatTableDesc(newSegment);
+ }
+
+ @Override
public DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) {
return new BatchCubingJobBuilder2(newSegment, submitter).build();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
index 877358b..2c3b77f 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.IMRInput.IMRBatchCubingInputSide;
import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
import org.apache.kylin.engine.mr.IMROutput.IMRBatchCubingOutputSide;
@@ -30,6 +31,7 @@ import org.apache.kylin.engine.mr.IMROutput.IMRBatchMergeOutputSide;
import org.apache.kylin.engine.mr.IMROutput2.IMRBatchCubingOutputSide2;
import org.apache.kylin.engine.mr.IMROutput2.IMRBatchMergeOutputSide2;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.storage.StorageFactory;
@@ -37,7 +39,8 @@ import org.apache.kylin.storage.StorageFactory;
public class MRUtil {
public static IMRBatchCubingInputSide getBatchCubingInputSide(CubeSegment seg) {
- return SourceFactory.createEngineAdapter(seg, IMRInput.class).getBatchCubingInputSide(seg.getJoinedFlatTableDesc());
+ IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(seg);
+ return SourceFactory.createEngineAdapter(seg, IMRInput.class).getBatchCubingInputSide(flatDesc);
}
public static IMRTableInputFormat getTableInputFormat(String tableName) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
index 4786505..588b087 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java
@@ -37,7 +37,8 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -65,7 +66,7 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
protected CubeDesc cubeDesc;
protected CubeSegment cubeSegment;
protected List<byte[]> nullBytes;
- protected CubeJoinedFlatTableDesc intermediateTableDesc;
+ protected CubeJoinedFlatTableEnrich intermediateTableDesc;
protected String intermediateTableRowDelimiter;
protected byte byteRowDelimiter;
protected int counter;
@@ -102,7 +103,7 @@ public class BaseCuboidMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VAL
long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
- intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeSegment);
+ intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
bytesSplitter = new BytesSplitter(200, 16384);
rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index 20259cb..3fa966d 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -30,7 +30,8 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.MRUtil;
@@ -54,7 +55,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
protected Text outputValue = new Text();
protected int errorRecordCounter = 0;
- protected CubeJoinedFlatTableDesc intermediateTableDesc;
+ protected CubeJoinedFlatTableEnrich intermediateTableDesc;
protected int[] dictionaryColumnIndex;
@Override
@@ -72,7 +73,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSeg).getFlatTableInputFormat();
- intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc);
+ intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSeg), cubeDesc);
dictionaryColumnIndex = new int[factDictCols.size()];
for (int i = 0; i < factDictCols.size(); i++) {
TblColRef colRef = factDictCols.get(i);
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
index 24c37ce..1d90d01 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidMapper.java
@@ -39,12 +39,14 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.ByteArrayWritable;
import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat;
import org.apache.kylin.engine.mr.KylinMapper;
import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -79,6 +81,7 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArr
String segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);
cubeSegment = cube.getSegmentById(segmentID);
flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSegment).getFlatTableInputFormat();
+ IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
Map<TblColRef, Dictionary<String>> dictionaryMap = Maps.newHashMap();
@@ -92,7 +95,7 @@ public class InMemCuboidMapper<KEYIN> extends KylinMapper<KEYIN, Object, ByteArr
dictionaryMap.put(col, cubeSegment.getDictionary(col));
}
- DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap);
+ DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
cubeBuilder.setReserveMemoryMB(calculateReserveMB(context.getConfiguration()));
ExecutorService executorService = Executors.newSingleThreadExecutor();
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine.java
index 4eec233..08ed207 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingEngine.java
@@ -18,8 +18,10 @@
package org.apache.kylin.engine.spark;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.IBatchCubingEngine;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
/**
*/
@@ -52,4 +54,14 @@ public class SparkBatchCubingEngine implements IBatchCubingEngine {
public Class<?> getStorageInterface() {
return null;
}
+
+ @Override
+ public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) {
+ throw new UnsupportedOperationException();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 3ccbcc8..5c2def2 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -69,12 +69,13 @@ import org.apache.kylin.cube.inmemcubing.AbstractInMemCubeBuilder;
import org.apache.kylin.cube.inmemcubing.DoggedCubeBuilder;
import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.cube.model.RowKeyDesc;
import org.apache.kylin.cube.util.CubingUtils;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CubeStatsReader;
import org.apache.kylin.engine.spark.cube.BufferedCuboidWriter;
@@ -84,6 +85,7 @@ import org.apache.kylin.measure.BufferedMeasureEncoder;
import org.apache.kylin.measure.MeasureAggregators;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
@@ -175,9 +177,10 @@ public class SparkCubing extends AbstractApplication {
final CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
final String[] columns = intermediateTable.columns();
+ final CubeSegment seg = cubeInstance.getSegmentById(segmentId);
final CubeDesc cubeDesc = cubeInstance.getDescriptor();
final HashMap<Integer, TblColRef> tblColRefMap = Maps.newHashMap();
- final CubeJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(cubeDesc);
+ final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(seg), cubeDesc);
final List<TblColRef> baseCuboidColumn = Cuboid.findById(cubeDesc, Cuboid.getBaseCuboidId(cubeDesc)).getColumns();
final long start = System.currentTimeMillis();
final RowKeyDesc rowKey = cubeDesc.getRowkey();
@@ -186,7 +189,7 @@ public class SparkCubing extends AbstractApplication {
if (!rowKey.isUseDictionary(col)) {
continue;
}
- final int rowKeyColumnIndex = flatTableDesc.getRowKeyColumnIndexes()[i];
+ final int rowKeyColumnIndex = flatDesc.getRowKeyColumnIndexes()[i];
tblColRefMap.put(rowKeyColumnIndex, col);
}
@@ -228,18 +231,19 @@ public class SparkCubing extends AbstractApplication {
})));
}
final long end = System.currentTimeMillis();
- CubingUtils.writeDictionary(cubeInstance.getSegmentById(segmentId), dictionaryMap, start, end);
+ CubingUtils.writeDictionary(seg, dictionaryMap, start, end);
try {
CubeUpdate cubeBuilder = new CubeUpdate(cubeInstance);
- cubeBuilder.setToUpdateSegs(cubeInstance.getSegmentById(segmentId));
+ cubeBuilder.setToUpdateSegs(seg);
cubeManager.updateCube(cubeBuilder);
} catch (IOException e) {
throw new RuntimeException("Failed to deal with the request: " + e.getLocalizedMessage());
}
}
- private Map<Long, HyperLogLogPlusCounter> sampling(final JavaRDD<List<String>> rowJavaRDD, final String cubeName) throws Exception {
+ private Map<Long, HyperLogLogPlusCounter> sampling(final JavaRDD<List<String>> rowJavaRDD, final String cubeName, String segmentId) throws Exception {
CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).reloadCubeLocal(cubeName);
+ CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
CubeDesc cubeDesc = cubeInstance.getDescriptor();
CuboidScheduler cuboidScheduler = new CuboidScheduler(cubeDesc);
List<Long> allCuboidIds = cuboidScheduler.getAllCuboidIds();
@@ -248,9 +252,9 @@ public class SparkCubing extends AbstractApplication {
zeroValue.put(id, new HyperLogLogPlusCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
}
- CubeJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(cubeDesc);
+ CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
- final int[] rowKeyColumnIndexes = flatTableDesc.getRowKeyColumnIndexes();
+ final int[] rowKeyColumnIndexes = flatDesc.getRowKeyColumnIndexes();
final int nRowKey = cubeDesc.getRowkey().getRowKeyColumns().length;
final long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
final Map<Long, Integer[]> allCuboidsBitSet = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
@@ -318,9 +322,7 @@ public class SparkCubing extends AbstractApplication {
return samplingResult;
}
- /*
- return hfile location
- */
+ /** return hfile location */
private String build(JavaRDD<List<String>> javaRDD, final String cubeName, final String segmentId, final byte[][] splitKeys) throws Exception {
CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
CubeDesc cubeDesc = cubeInstance.getDescriptor();
@@ -365,7 +367,8 @@ public class SparkCubing extends AbstractApplication {
LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue();
System.out.println("load properties finished");
- AbstractInMemCubeBuilder inMemCubeBuilder = new DoggedCubeBuilder(cubeInstance.getDescriptor(), dictionaryMap);
+ IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
+ AbstractInMemCubeBuilder inMemCubeBuilder = new DoggedCubeBuilder(cubeInstance.getDescriptor(), flatDesc, dictionaryMap);
final SparkCuboidWriter sparkCuboidWriter = new BufferedCuboidWriter(new DefaultTupleConverter(cubeInstance.getSegmentById(segmentId), columnLengthMap));
Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, sparkCuboidWriter));
try {
@@ -611,7 +614,7 @@ public class SparkCubing extends AbstractApplication {
}
});
- final Map<Long, HyperLogLogPlusCounter> samplingResult = sampling(rowJavaRDD, cubeName);
+ final Map<Long, HyperLogLogPlusCounter> samplingResult = sampling(rowJavaRDD, cubeName, segmentId);
final byte[][] splitKeys = createHTable(cubeName, segmentId, samplingResult);
final String hfile = build(rowJavaRDD, cubeName, segmentId, splitKeys);
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
index 05246f4..edd9460 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingJobBuilder.java
@@ -18,6 +18,7 @@
package org.apache.kylin.engine.spark;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.IMRInput;
import org.apache.kylin.engine.mr.IMROutput2;
@@ -48,11 +49,11 @@ public class SparkCubingJobBuilder extends JobBuilderSupport {
}
public DefaultChainedExecutable build() {
- final CubingJob result = CubingJob.createBuildJob((CubeSegment) seg, submitter, config);
+ final CubingJob result = CubingJob.createBuildJob(seg, submitter, config);
final String jobId = result.getId();
inputSide.addStepPhase1_CreateFlatTable(result);
- final IJoinedFlatTableDesc joinedFlatTableDesc = seg.getJoinedFlatTableDesc();
+ final IJoinedFlatTableDesc joinedFlatTableDesc = EngineFactory.getJoinedFlatTableDesc(seg);
final String tableName = joinedFlatTableDesc.getTableName();
logger.info("intermediate table:" + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/engine-streaming/pom.xml b/engine-streaming/pom.xml
index e25a133..095871c 100644
--- a/engine-streaming/pom.xml
+++ b/engine-streaming/pom.xml
@@ -45,6 +45,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-storage</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-job</artifactId>
+ </dependency>
<!-- Env & Test -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
----------------------------------------------------------------------
diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
index 465a983..180f0b8 100644
--- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
+++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/cube/StreamingCubeBuilder.java
@@ -55,9 +55,11 @@ import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.cube.inmemcubing.ICuboidWriter;
import org.apache.kylin.cube.inmemcubing.InMemCubeBuilder;
import org.apache.kylin.cube.util.CubingUtils;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.streaming.StreamingBatchBuilder;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.metadata.model.IBuildable;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -84,8 +86,10 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
try {
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
+ final IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeInstance.getDescriptor());
+
LinkedBlockingQueue<List<String>> blockingQueue = new LinkedBlockingQueue<List<String>>();
- InMemCubeBuilder inMemCubeBuilder = new InMemCubeBuilder(cubeInstance.getDescriptor(), dictionaryMap);
+ InMemCubeBuilder inMemCubeBuilder = new InMemCubeBuilder(cubeInstance.getDescriptor(), flatDesc, dictionaryMap);
final Future<?> future = Executors.newCachedThreadPool().submit(inMemCubeBuilder.buildAsRunnable(blockingQueue, cuboidWriter));
processedRowCount = streamingBatch.getMessages().size();
for (StreamingMessage streamingMessage : streamingBatch.getMessages()) {
@@ -129,9 +133,10 @@ public class StreamingCubeBuilder implements StreamingBatchBuilder {
public Map<Long, HyperLogLogPlusCounter> sampling(StreamingBatch streamingBatch) {
final CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
+ final IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cubeInstance.getDescriptor());
long start = System.currentTimeMillis();
- final Map<Long, HyperLogLogPlusCounter> samplingResult = CubingUtils.sampling(cubeInstance.getDescriptor(), Lists.transform(streamingBatch.getMessages(), new Function<StreamingMessage, List<String>>() {
+ final Map<Long, HyperLogLogPlusCounter> samplingResult = CubingUtils.sampling(cubeInstance.getDescriptor(), flatDesc, Lists.transform(streamingBatch.getMessages(), new Function<StreamingMessage, List<String>>() {
@Nullable
@Override
public List<String> apply(@Nullable StreamingMessage input) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
index ef69d57..4d23979 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
@@ -31,7 +31,9 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -79,7 +81,8 @@ public class ITDoggedCubeBuilderStressTest extends LocalFileMetadataTestCase {
ExecutorService executorService = Executors.newSingleThreadExecutor();
long randSeed = System.currentTimeMillis();
- DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap);
+ IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
+ DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
doggedBuilder.setConcurrentThreads(THREADS);
{
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
index 8923744..8827dff 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
@@ -40,7 +40,9 @@ import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -87,7 +89,8 @@ public class ITDoggedCubeBuilderTest extends LocalFileMetadataTestCase {
ExecutorService executorService = Executors.newSingleThreadExecutor();
long randSeed = System.currentTimeMillis();
- DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap);
+ IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
+ DoggedCubeBuilder doggedBuilder = new DoggedCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
doggedBuilder.setConcurrentThreads(THREADS);
doggedBuilder.setSplitRowThreshold(SPLIT_ROWS);
FileRecordWriter doggedResult = new FileRecordWriter();
@@ -99,7 +102,7 @@ public class ITDoggedCubeBuilderTest extends LocalFileMetadataTestCase {
doggedResult.close();
}
- InMemCubeBuilder inmemBuilder = new InMemCubeBuilder(cube.getDescriptor(), dictionaryMap);
+ InMemCubeBuilder inmemBuilder = new InMemCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
inmemBuilder.setConcurrentThreads(THREADS);
FileRecordWriter inmemResult = new FileRecordWriter();
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
index 87b222e..6612375 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
@@ -42,11 +42,13 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich;
import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.After;
@@ -109,7 +111,8 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
private void testBuildInner() throws Exception {
- InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(cube.getDescriptor(), dictionaryMap);
+ IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
+ InMemCubeBuilder cubeBuilder = new InMemCubeBuilder(cube.getDescriptor(), flatDesc, dictionaryMap);
//DoggedCubeBuilder cubeBuilder = new DoggedCubeBuilder(cube.getDescriptor(), dictionaryMap);
cubeBuilder.setConcurrentThreads(nThreads);
@@ -149,8 +152,8 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
}
static void feedData(final CubeInstance cube, final String flatTable, ArrayBlockingQueue<List<String>> queue, int count, long randSeed) throws IOException, InterruptedException {
- CubeJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor());
- int nColumns = flatTableDesc.getAllColumns().size();
+ IJoinedFlatTableDesc flatDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor());
+ int nColumns = flatDesc.getAllColumns().size();
@SuppressWarnings("unchecked")
Set<String>[] distinctSets = new Set[nColumns];
@@ -190,15 +193,15 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
static Map<TblColRef, Dictionary<String>> getDictionaryMap(CubeInstance cube, String flatTable) throws IOException {
Map<TblColRef, Dictionary<String>> result = Maps.newHashMap();
CubeDesc desc = cube.getDescriptor();
- CubeJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(desc);
- int nColumns = flatTableDesc.getAllColumns().size();
+ CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(desc), desc);
+ int nColumns = flatDesc.getAllColumns().size();
List<TblColRef> columns = Cuboid.getBaseCuboid(desc).getColumns();
for (int c = 0; c < columns.size(); c++) {
TblColRef col = columns.get(c);
if (desc.getRowkey().isUseDictionary(col)) {
logger.info("Building dictionary for " + col);
- List<byte[]> valueList = readValueList(flatTable, nColumns, flatTableDesc.getRowKeyColumnIndexes()[c]);
+ List<byte[]> valueList = readValueList(flatTable, nColumns, flatDesc.getRowKeyColumnIndexes()[c]);
Dictionary<String> dict = DictionaryGenerator.buildDictionary(col.getType(), new IterableDictionaryValueEnumerator(valueList));
result.put(col, dict);
}
@@ -211,7 +214,7 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
if (dictCols.isEmpty())
continue;
- int[] flatTableIdx = flatTableDesc.getMeasureColumnIndexes()[measureIdx];
+ int[] flatTableIdx = flatDesc.getMeasureColumnIndexes()[measureIdx];
List<TblColRef> paramCols = func.getParameter().getColRefs();
for (int i = 0; i < paramCols.size(); i++) {
TblColRef col = paramCols.get(i);
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 4e56f74..57b0965 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -34,11 +34,12 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.dimension.DimensionEncodingFactory;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.streaming.StreamingConfig;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.JoinedFlatTable;
+import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -135,7 +136,7 @@ public class CubeController extends BasicController {
public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
CubeSegment cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.READY);
- CubeJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(cubeSegment);
+ IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment);
String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc);
GeneralResponse repsonse = new GeneralResponse();
http://git-wip-us.apache.org/repos/asf/kylin/blob/29ba46be/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
index e055d9e..c3bdb75 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaStreamingInput.java
@@ -46,7 +46,7 @@ import org.apache.kylin.common.util.StreamingBatch;
import org.apache.kylin.common.util.StreamingMessage;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.streaming.IStreamingInput;
import org.apache.kylin.engine.streaming.StreamingConfig;
import org.apache.kylin.engine.streaming.StreamingManager;
@@ -90,7 +90,7 @@ public class KafkaStreamingInput implements IStreamingInput {
try {
final KafkaConfigManager kafkaConfigManager = KafkaConfigManager.getInstance(kylinConfig);
final KafkaConfig kafkaConfig = kafkaConfigManager.getKafkaConfig(streaming);
- List<TblColRef> columns = new CubeJoinedFlatTableDesc(cube.getDescriptor()).getAllColumns();
+ List<TblColRef> columns = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor()).getAllColumns();
final StreamingParser streamingParser = StreamingParser.getStreamingParser(kafkaConfig.getParserName(), kafkaConfig.getParserProperties(), columns);
final ExecutorService executorService = Executors.newCachedThreadPool();
[14/50] [abbrv] kylin git commit: KYLIN-1972 Fix query access denied
when query hybrid
Posted by li...@apache.org.
KYLIN-1972 Fix query access denied when query hybrid
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1ea79dda
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1ea79dda
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1ea79dda
Branch: refs/heads/1.5.x-CDH5.7
Commit: 1ea79dda1a2d6202e4ca7bfc115f68090b06e9b6
Parents: 6388809
Author: liapan <li...@ebay.com>
Authored: Wed Aug 24 15:50:13 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Aug 27 22:05:56 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/QueryController.java | 42 +++++++++++++++++++-
1 file changed, 40 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1ea79dda/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index e7847c7..a45f82e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -30,6 +30,8 @@ import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.metrics.QueryMetricsFacade;
@@ -44,6 +46,7 @@ import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.storage.exception.ScanOutOfLimitException;
+import org.apache.kylin.storage.hybrid.HybridInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -256,9 +259,44 @@ public class QueryController extends BasicController {
private void checkQueryAuth(SQLResponse sqlResponse) throws AccessDeniedException {
if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled()) {
- CubeInstance cubeInstance = this.queryService.getCubeManager().getCube(sqlResponse.getCube());
- queryService.checkAuthorization(cubeInstance);
+ HybridInstance hybridInstance = this.queryService.getHybridManager().getHybridInstance(sqlResponse.getCube());
+ if (hybridInstance != null) {
+ if (checkHybridAuthorization(hybridInstance)) {
+ logger.info(hybridInstance.getName() + " ACL validation success.");
+ } else {
+ throw new AccessDeniedException("Access is denied");
+ }
+ } else {
+ CubeInstance cubeInstance = this.queryService.getCubeManager().getCube(sqlResponse.getCube());
+ queryService.checkAuthorization(cubeInstance);
+ }
+ }
+ }
+
+ private boolean checkHybridAuthorization(HybridInstance hybridInstance) {
+ boolean access = false;
+ List<RealizationEntry> realizationEntries = hybridInstance.getRealizationEntries();
+ for (RealizationEntry realizationEntry : realizationEntries) {
+ String reName = realizationEntry.getRealization();
+ logger.debug("[realizationEntry] realizationEntry name: " + reName + " realizationEntry type: " + realizationEntry.getType().name());
+ if (RealizationType.CUBE == realizationEntry.getType()) {
+ CubeInstance cubeInstance = queryService.getCubeManager().getCube(reName);
+ try {
+ queryService.checkAuthorization(cubeInstance);
+ logger.info(hybridInstance.getName() + " ACL validation cube: " + cubeInstance.getName() + " success.");
+ logger.info(hybridInstance.getName() + " ACL validation success.");
+ return true;
+ } catch (AccessDeniedException e) {
+ logger.info(hybridInstance.getName() + " ACL validation cube: " + cubeInstance.getName() + " failed.");
+ }
+ } else if (RealizationType.HYBRID == realizationEntry.getType()) {
+ HybridInstance innerHybridInstance = queryService.getHybridManager().getHybridInstance(reName);
+ if (checkHybridAuthorization(innerHybridInstance)) {
+ return true;
+ }
+ }
}
+ return access;
}
public void setQueryService(QueryService queryService) {
[19/50] [abbrv] kylin git commit: KYLIN-1702: change the layout path
of table snapshot with tablename
Posted by li...@apache.org.
KYLIN-1702: change the layout path of table snapshot with tablename
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7815224d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7815224d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7815224d
Branch: refs/heads/1.5.x-CDH5.7
Commit: 7815224da90f81a752ceb3591370ebe39d2a45d1
Parents: ae9d747
Author: Zhong <ya...@lm-shc-16501214.corp.ebay.com>
Authored: Tue Aug 16 16:11:18 2016 +0800
Committer: Zhong <nj...@apache.org>
Committed: Mon Aug 29 09:35:50 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/lookup/SnapshotManager.java | 4 ++--
.../org/apache/kylin/dict/lookup/SnapshotTable.java | 16 ++++++++++++++--
2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7815224d/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index c7b0d26..b15847b 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -115,7 +115,7 @@ public class SnapshotManager {
}
public SnapshotTable buildSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException {
- SnapshotTable snapshot = new SnapshotTable(table);
+ SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getName());
snapshot.updateRandomUuid();
String dup = checkDupByInfo(snapshot);
@@ -135,7 +135,7 @@ public class SnapshotManager {
}
public SnapshotTable rebuildSnapshot(ReadableTable table, TableDesc tableDesc, String overwriteUUID) throws IOException {
- SnapshotTable snapshot = new SnapshotTable(table);
+ SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getName());
snapshot.setUuid(overwriteUUID);
snapshot.takeSnapshot(table, tableDesc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7815224d/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index b2306b2..1aea124 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import com.google.common.base.Strings;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -49,6 +50,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class SnapshotTable extends RootPersistentEntity implements ReadableTable {
+ @JsonProperty("tableName")
+ private String tableName;
@JsonProperty("signature")
private TableSignature signature;
@JsonProperty("useDictionary")
@@ -61,7 +64,8 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
public SnapshotTable() {
}
- SnapshotTable(ReadableTable table) throws IOException {
+ SnapshotTable(ReadableTable table, String tableName) throws IOException {
+ this.tableName = tableName;
this.signature = table.getSignature();
this.useDictionary = true;
}
@@ -111,10 +115,18 @@ public class SnapshotTable extends RootPersistentEntity implements ReadableTable
}
public String getResourcePath() {
- return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new File(signature.getPath()).getName() + "/" + uuid + ".snapshot";
+ return getResourceDir() + "/" + uuid + ".snapshot";
}
public String getResourceDir() {
+ if (Strings.isNullOrEmpty(tableName)) {
+ return getOldResourceDir();
+ } else {
+ return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + tableName;
+ }
+ }
+
+ private String getOldResourceDir() {
return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new File(signature.getPath()).getName();
}
[13/50] [abbrv] kylin git commit: KYLIN-1970 Fix UI system menu ACL.
Posted by li...@apache.org.
KYLIN-1970 Fix UI system menu ACL.
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/63888096
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/63888096
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/63888096
Branch: refs/heads/1.5.x-CDH5.7
Commit: 638880961ee745ccc107232f59d8c0055b6fc66b
Parents: 5ee76e8
Author: sanjulian <ju...@hotmail.com>
Authored: Wed Aug 24 15:16:35 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Aug 27 21:54:54 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/header.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/63888096/webapp/app/partials/header.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/header.html b/webapp/app/partials/header.html
index 6249f51..0e37e63 100644
--- a/webapp/app/partials/header.html
+++ b/webapp/app/partials/header.html
@@ -50,7 +50,7 @@
<li class="{{activeTab=='jobs'?'purple':'green'}}" ng-if="userService.isAuthorized()">
<a href="jobs">Monitor</a>
</li>
- <li class="{{activeTab=='admin'?'purple':'green'}}" ng-if="userService.isAuthorized()">
+ <li class="{{activeTab=='admin'?'purple':'green'}}" ng-if="userService.hasRole('ROLE_ADMIN')">
<a href="admin">System</a>
</li>
</ul>
[37/50] [abbrv] kylin git commit: minor, fix all useless columns
Posted by li...@apache.org.
minor, fix all useless columns
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/894c16f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/894c16f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/894c16f2
Branch: refs/heads/1.5.x-CDH5.7
Commit: 894c16f2c978667e2526c6a2a9e46c861e644ff8
Parents: 9702dff
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Aug 31 20:12:45 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Aug 31 20:12:45 2016 +0800
----------------------------------------------------------------------
.../metadata/table/DEFAULT.KYLIN_CAL_DT.json | 887 +++++++++---------
.../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json | 4 +-
.../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json | 4 +-
.../localmeta/table/EDW.TEST_CAL_DT.json | 892 +++++++++----------
.../table/EDW.TEST_SELLER_TYPE_DIM.json | 8 +-
.../localmeta/table/EDW.TEST_SITES.json | 4 +-
.../localmeta/table/EDW.V_TEST_CAL_DT.json | 889 +++++++++---------
7 files changed, 1229 insertions(+), 1459 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/894c16f2/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
index de404f1..d197115 100644
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
+++ b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
@@ -1,483 +1,408 @@
{
- "uuid": "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- "name": "KYLIN_CAL_DT",
- "columns": [
- {
- "id": "1",
- "name": "CAL_DT",
- "datatype": "date"
- },
- {
- "id": "2",
- "name": "YEAR_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "3",
- "name": "QTR_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "4",
- "name": "MONTH_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "5",
- "name": "WEEK_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "6",
- "name": "AGE_FOR_YEAR_ID",
- "datatype": "smallint"
- },
- {
- "id": "7",
- "name": "AGE_FOR_QTR_ID",
- "datatype": "smallint"
- },
- {
- "id": "8",
- "name": "AGE_FOR_MONTH_ID",
- "datatype": "smallint"
- },
- {
- "id": "9",
- "name": "AGE_FOR_WEEK_ID",
- "datatype": "smallint"
- },
- {
- "id": "10",
- "name": "AGE_FOR_DT_ID",
- "datatype": "smallint"
- },
- {
- "id": "11",
- "name": "AGE_FOR_RTL_YEAR_ID",
- "datatype": "smallint"
- },
- {
- "id": "12",
- "name": "AGE_FOR_RTL_QTR_ID",
- "datatype": "smallint"
- },
- {
- "id": "13",
- "name": "AGE_FOR_RTL_MONTH_ID",
- "datatype": "smallint"
- },
- {
- "id": "14",
- "name": "AGE_FOR_RTL_WEEK_ID",
- "datatype": "smallint"
- },
- {
- "id": "15",
- "name": "AGE_FOR_CS_WEEK_ID",
- "datatype": "smallint"
- },
- {
- "id": "16",
- "name": "DAY_OF_CAL_ID",
- "datatype": "int"
- },
- {
- "id": "17",
- "name": "DAY_OF_YEAR_ID",
- "datatype": "smallint"
- },
- {
- "id": "18",
- "name": "DAY_OF_QTR_ID",
- "datatype": "smallint"
- },
- {
- "id": "19",
- "name": "DAY_OF_MONTH_ID",
- "datatype": "smallint"
- },
- {
- "id": "20",
- "name": "DAY_OF_WEEK_ID",
- "datatype": "int"
- },
- {
- "id": "21",
- "name": "WEEK_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "22",
- "name": "WEEK_OF_CAL_ID",
- "datatype": "int"
- },
- {
- "id": "23",
- "name": "MONTH_OF_QTR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "24",
- "name": "MONTH_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "25",
- "name": "MONTH_OF_CAL_ID",
- "datatype": "smallint"
- },
- {
- "id": "26",
- "name": "QTR_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "27",
- "name": "QTR_OF_CAL_ID",
- "datatype": "smallint"
- },
- {
- "id": "28",
- "name": "YEAR_OF_CAL_ID",
- "datatype": "smallint"
- },
- {
- "id": "29",
- "name": "YEAR_END_DT",
- "datatype": "string"
- },
- {
- "id": "30",
- "name": "QTR_END_DT",
- "datatype": "string"
- },
- {
- "id": "31",
- "name": "MONTH_END_DT",
- "datatype": "string"
- },
- {
- "id": "32",
- "name": "WEEK_END_DT",
- "datatype": "string"
- },
- {
- "id": "33",
- "name": "CAL_DT_NAME",
- "datatype": "string"
- },
- {
- "id": "34",
- "name": "CAL_DT_DESC",
- "datatype": "string"
- },
- {
- "id": "35",
- "name": "CAL_DT_SHORT_NAME",
- "datatype": "string"
- },
- {
- "id": "36",
- "name": "YTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "37",
- "name": "QTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "38",
- "name": "MTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "39",
- "name": "WTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "40",
- "name": "SEASON_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "41",
- "name": "DAY_IN_YEAR_COUNT",
- "datatype": "smallint"
- },
- {
- "id": "42",
- "name": "DAY_IN_QTR_COUNT",
- "datatype": "tinyint"
- },
- {
- "id": "43",
- "name": "DAY_IN_MONTH_COUNT",
- "datatype": "tinyint"
- },
- {
- "id": "44",
- "name": "DAY_IN_WEEK_COUNT",
- "datatype": "tinyint"
- },
- {
- "id": "45",
- "name": "RTL_YEAR_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "46",
- "name": "RTL_QTR_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "47",
- "name": "RTL_MONTH_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "48",
- "name": "RTL_WEEK_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "49",
- "name": "CS_WEEK_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "50",
- "name": "CAL_DATE",
- "datatype": "string"
- },
- {
- "id": "51",
- "name": "DAY_OF_WEEK",
- "datatype": "string"
- },
- {
- "id": "52",
- "name": "MONTH_ID",
- "datatype": "string"
- },
- {
- "id": "53",
- "name": "PRD_DESC",
- "datatype": "string"
- },
- {
- "id": "54",
- "name": "PRD_FLAG",
- "datatype": "string"
- },
- {
- "id": "55",
- "name": "PRD_ID",
- "datatype": "string"
- },
- {
- "id": "56",
- "name": "PRD_IND",
- "datatype": "string"
- },
- {
- "id": "57",
- "name": "QTR_DESC",
- "datatype": "string"
- },
- {
- "id": "58",
- "name": "QTR_ID",
- "datatype": "string"
- },
- {
- "id": "59",
- "name": "QTR_IND",
- "datatype": "string"
- },
- {
- "id": "60",
- "name": "RETAIL_WEEK",
- "datatype": "string"
- },
- {
- "id": "61",
- "name": "RETAIL_YEAR",
- "datatype": "string"
- },
- {
- "id": "62",
- "name": "RETAIL_START_DATE",
- "datatype": "string"
- },
- {
- "id": "63",
- "name": "RETAIL_WK_END_DATE",
- "datatype": "string"
- },
- {
- "id": "64",
- "name": "WEEK_IND",
- "datatype": "string"
- },
- {
- "id": "65",
- "name": "WEEK_NUM_DESC",
- "datatype": "string"
- },
- {
- "id": "66",
- "name": "WEEK_BEG_DATE",
- "datatype": "string"
- },
- {
- "id": "67",
- "name": "WEEK_END_DATE",
- "datatype": "string"
- },
- {
- "id": "68",
- "name": "WEEK_IN_YEAR_ID",
- "datatype": "string"
- },
- {
- "id": "69",
- "name": "WEEK_ID",
- "datatype": "string"
- },
- {
- "id": "70",
- "name": "WEEK_BEG_END_DESC_MDY",
- "datatype": "string"
- },
- {
- "id": "71",
- "name": "WEEK_BEG_END_DESC_MD",
- "datatype": "string"
- },
- {
- "id": "72",
- "name": "YEAR_ID",
- "datatype": "string"
- },
- {
- "id": "73",
- "name": "YEAR_IND",
- "datatype": "string"
- },
- {
- "id": "74",
- "name": "CAL_DT_MNS_1YEAR_DT",
- "datatype": "string"
- },
- {
- "id": "75",
- "name": "CAL_DT_MNS_2YEAR_DT",
- "datatype": "string"
- },
- {
- "id": "76",
- "name": "CAL_DT_MNS_1QTR_DT",
- "datatype": "string"
- },
- {
- "id": "77",
- "name": "CAL_DT_MNS_2QTR_DT",
- "datatype": "string"
- },
- {
- "id": "78",
- "name": "CAL_DT_MNS_1MONTH_DT",
- "datatype": "string"
- },
- {
- "id": "79",
- "name": "CAL_DT_MNS_2MONTH_DT",
- "datatype": "string"
- },
- {
- "id": "80",
- "name": "CAL_DT_MNS_1WEEK_DT",
- "datatype": "string"
- },
- {
- "id": "81",
- "name": "CAL_DT_MNS_2WEEK_DT",
- "datatype": "string"
- },
- {
- "id": "82",
- "name": "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "83",
- "name": "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "84",
- "name": "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "85",
- "name": "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "86",
- "name": "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "87",
- "name": "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "88",
- "name": "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype": "tinyint"
- },
- {
- "id": "89",
- "name": "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype": "tinyint"
- },
- {
- "id": "90",
- "name": "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype": "string"
- },
- {
- "id": "91",
- "name": "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "92",
- "name": "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "93",
- "name": "SEASON_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "94",
- "name": "YTM_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "95",
- "name": "YTQ_YN_ID",
- "datatype": "tinyint"
- }
- ],
- "database": "default",
- "last_modified": 0
+ "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+
+ "name" : "KYLIN_CAL_DT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "CAL_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "2",
+ "name" : "YEAR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "QTR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "4",
+ "name" : "MONTH_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "5",
+ "name" : "WEEK_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "6",
+ "name" : "AGE_FOR_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "AGE_FOR_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "8",
+ "name" : "AGE_FOR_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "9",
+ "name" : "AGE_FOR_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "10",
+ "name" : "AGE_FOR_DT_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "11",
+ "name" : "AGE_FOR_RTL_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "12",
+ "name" : "AGE_FOR_RTL_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "13",
+ "name" : "AGE_FOR_RTL_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "14",
+ "name" : "AGE_FOR_RTL_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "15",
+ "name" : "AGE_FOR_CS_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "16",
+ "name" : "DAY_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "17",
+ "name" : "DAY_OF_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "18",
+ "name" : "DAY_OF_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "19",
+ "name" : "DAY_OF_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "20",
+ "name" : "DAY_OF_WEEK_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "21",
+ "name" : "WEEK_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "22",
+ "name" : "WEEK_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "23",
+ "name" : "MONTH_OF_QTR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "24",
+ "name" : "MONTH_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "25",
+ "name" : "MONTH_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "26",
+ "name" : "QTR_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "27",
+ "name" : "QTR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "28",
+ "name" : "YEAR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "29",
+ "name" : "YEAR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "30",
+ "name" : "QTR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "MONTH_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "32",
+ "name" : "WEEK_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "33",
+ "name" : "CAL_DT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "34",
+ "name" : "CAL_DT_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "35",
+ "name" : "CAL_DT_SHORT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "36",
+ "name" : "YTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "37",
+ "name" : "QTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "38",
+ "name" : "MTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "39",
+ "name" : "WTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "40",
+ "name" : "SEASON_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "41",
+ "name" : "DAY_IN_YEAR_COUNT",
+ "datatype" : "smallint"
+ }, {
+ "id" : "42",
+ "name" : "DAY_IN_QTR_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "43",
+ "name" : "DAY_IN_MONTH_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "44",
+ "name" : "DAY_IN_WEEK_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "45",
+ "name" : "RTL_YEAR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "46",
+ "name" : "RTL_QTR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "47",
+ "name" : "RTL_MONTH_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "48",
+ "name" : "RTL_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "49",
+ "name" : "CS_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "50",
+ "name" : "CAL_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "51",
+ "name" : "DAY_OF_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "52",
+ "name" : "MONTH_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "53",
+ "name" : "PRD_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "54",
+ "name" : "PRD_FLAG",
+ "datatype" : "string"
+ }, {
+ "id" : "55",
+ "name" : "PRD_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "56",
+ "name" : "PRD_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "57",
+ "name" : "QTR_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "58",
+ "name" : "QTR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "59",
+ "name" : "QTR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "60",
+ "name" : "RETAIL_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "61",
+ "name" : "RETAIL_YEAR",
+ "datatype" : "string"
+ }, {
+ "id" : "62",
+ "name" : "RETAIL_START_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "63",
+ "name" : "RETAIL_WK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "64",
+ "name" : "WEEK_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "65",
+ "name" : "WEEK_NUM_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "66",
+ "name" : "WEEK_BEG_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "67",
+ "name" : "WEEK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "68",
+ "name" : "WEEK_IN_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "69",
+ "name" : "WEEK_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "70",
+ "name" : "WEEK_BEG_END_DESC_MDY",
+ "datatype" : "string"
+ }, {
+ "id" : "71",
+ "name" : "WEEK_BEG_END_DESC_MD",
+ "datatype" : "string"
+ }, {
+ "id" : "72",
+ "name" : "YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "73",
+ "name" : "YEAR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "74",
+ "name" : "CAL_DT_MNS_1YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "75",
+ "name" : "CAL_DT_MNS_2YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "76",
+ "name" : "CAL_DT_MNS_1QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "77",
+ "name" : "CAL_DT_MNS_2QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "78",
+ "name" : "CAL_DT_MNS_1MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "79",
+ "name" : "CAL_DT_MNS_2MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "80",
+ "name" : "CAL_DT_MNS_1WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "81",
+ "name" : "CAL_DT_MNS_2WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "82",
+ "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "83",
+ "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "84",
+ "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "85",
+ "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "86",
+ "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "87",
+ "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "88",
+ "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "89",
+ "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "90",
+ "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "91",
+ "name" : "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "92",
+ "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "93",
+ "name" : "SEASON_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "94",
+ "name" : "YTM_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "95",
+ "name" : "YTQ_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "96",
+ "name" : "YTW_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "97",
+ "name" : "KYLIN_CAL_DT_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "98",
+ "name" : "KYLIN_CAL_DT_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "99",
+ "name" : "KYLIN_CAL_DT_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "100",
+ "name" : "KYLIN_CAL_DT_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "default",
+ "last_modified" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/894c16f2/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
index b9891dc..8e97309 100644
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
+++ b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
@@ -36,7 +36,7 @@
"datatype" : "string"
}, {
"id" : "9",
- "name" : "CRE_DATE",
+ "name" : "KYLIN_GROUPINGS_CRE_DATE",
"datatype" : "string"
}, {
"id" : "10",
@@ -44,7 +44,7 @@
"datatype" : "string"
}, {
"id" : "11",
- "name" : "CRE_USER",
+ "name" : "KYLIN_GROUPINGS_CRE_USER",
"datatype" : "string"
}, {
"id" : "12",
http://git-wip-us.apache.org/repos/asf/kylin/blob/894c16f2/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
index 7d2964f..069f41b 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
@@ -36,7 +36,7 @@
"datatype" : "string"
}, {
"id" : "9",
- "name" : "CRE_DATE",
+ "name" : "GROUPINGS_CRE_DATE",
"datatype" : "string"
}, {
"id" : "10",
@@ -44,7 +44,7 @@
"datatype" : "string"
}, {
"id" : "11",
- "name" : "CRE_USER",
+ "name" : "GROUPINGS_CRE_USER",
"datatype" : "string"
}, {
"id" : "12",
http://git-wip-us.apache.org/repos/asf/kylin/blob/894c16f2/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
index d64eb80..dde24cf 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
@@ -1,488 +1,408 @@
{
- "uuid": "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- "name": "TEST_CAL_DT",
- "columns": [
- {
- "id": "1",
- "name": "CAL_DT",
- "datatype": "date"
- },
- {
- "id": "2",
- "name": "YEAR_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "3",
- "name": "QTR_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "4",
- "name": "MONTH_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "5",
- "name": "WEEK_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "6",
- "name": "AGE_FOR_YEAR_ID",
- "datatype": "smallint"
- },
- {
- "id": "7",
- "name": "AGE_FOR_QTR_ID",
- "datatype": "smallint"
- },
- {
- "id": "8",
- "name": "AGE_FOR_MONTH_ID",
- "datatype": "smallint"
- },
- {
- "id": "9",
- "name": "AGE_FOR_WEEK_ID",
- "datatype": "smallint"
- },
- {
- "id": "10",
- "name": "AGE_FOR_DT_ID",
- "datatype": "smallint"
- },
- {
- "id": "11",
- "name": "AGE_FOR_RTL_YEAR_ID",
- "datatype": "smallint"
- },
- {
- "id": "12",
- "name": "AGE_FOR_RTL_QTR_ID",
- "datatype": "smallint"
- },
- {
- "id": "13",
- "name": "AGE_FOR_RTL_MONTH_ID",
- "datatype": "smallint"
- },
- {
- "id": "14",
- "name": "AGE_FOR_RTL_WEEK_ID",
- "datatype": "smallint"
- },
- {
- "id": "15",
- "name": "AGE_FOR_CS_WEEK_ID",
- "datatype": "smallint"
- },
- {
- "id": "16",
- "name": "DAY_OF_CAL_ID",
- "datatype": "int"
- },
- {
- "id": "17",
- "name": "DAY_OF_YEAR_ID",
- "datatype": "smallint"
- },
- {
- "id": "18",
- "name": "DAY_OF_QTR_ID",
- "datatype": "smallint"
- },
- {
- "id": "19",
- "name": "DAY_OF_MONTH_ID",
- "datatype": "smallint"
- },
- {
- "id": "20",
- "name": "DAY_OF_WEEK_ID",
- "datatype": "int"
- },
- {
- "id": "21",
- "name": "WEEK_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "22",
- "name": "WEEK_OF_CAL_ID",
- "datatype": "int"
- },
- {
- "id": "23",
- "name": "MONTH_OF_QTR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "24",
- "name": "MONTH_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "25",
- "name": "MONTH_OF_CAL_ID",
- "datatype": "smallint"
- },
- {
- "id": "26",
- "name": "QTR_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "27",
- "name": "QTR_OF_CAL_ID",
- "datatype": "smallint"
- },
- {
- "id": "28",
- "name": "YEAR_OF_CAL_ID",
- "datatype": "smallint"
- },
- {
- "id": "29",
- "name": "YEAR_END_DT",
- "datatype": "string"
- },
- {
- "id": "30",
- "name": "QTR_END_DT",
- "datatype": "string"
- },
- {
- "id": "31",
- "name": "MONTH_END_DT",
- "datatype": "string"
- },
- {
- "id": "32",
- "name": "WEEK_END_DT",
- "datatype": "string"
- },
- {
- "id": "33",
- "name": "CAL_DT_NAME",
- "datatype": "string"
- },
- {
- "id": "34",
- "name": "CAL_DT_DESC",
- "datatype": "string"
- },
- {
- "id": "35",
- "name": "CAL_DT_SHORT_NAME",
- "datatype": "string"
- },
- {
- "id": "36",
- "name": "YTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "37",
- "name": "QTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "38",
- "name": "MTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "39",
- "name": "WTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "40",
- "name": "SEASON_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "41",
- "name": "DAY_IN_YEAR_COUNT",
- "datatype": "smallint"
- },
- {
- "id": "42",
- "name": "DAY_IN_QTR_COUNT",
- "datatype": "tinyint"
- },
- {
- "id": "43",
- "name": "DAY_IN_MONTH_COUNT",
- "datatype": "tinyint"
- },
- {
- "id": "44",
- "name": "DAY_IN_WEEK_COUNT",
- "datatype": "tinyint"
- },
- {
- "id": "45",
- "name": "RTL_YEAR_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "46",
- "name": "RTL_QTR_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "47",
- "name": "RTL_MONTH_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "48",
- "name": "RTL_WEEK_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "49",
- "name": "CS_WEEK_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "50",
- "name": "CAL_DATE",
- "datatype": "string"
- },
- {
- "id": "51",
- "name": "DAY_OF_WEEK",
- "datatype": "string"
- },
- {
- "id": "52",
- "name": "MONTH_ID",
- "datatype": "string"
- },
- {
- "id": "53",
- "name": "PRD_DESC",
- "datatype": "string"
- },
- {
- "id": "54",
- "name": "PRD_FLAG",
- "datatype": "string"
- },
- {
- "id": "55",
- "name": "PRD_ID",
- "datatype": "string"
- },
- {
- "id": "56",
- "name": "PRD_IND",
- "datatype": "string"
- },
- {
- "id": "57",
- "name": "QTR_DESC",
- "datatype": "string"
- },
- {
- "id": "58",
- "name": "QTR_ID",
- "datatype": "string"
- },
- {
- "id": "59",
- "name": "QTR_IND",
- "datatype": "string"
- },
- {
- "id": "60",
- "name": "RETAIL_WEEK",
- "datatype": "string"
- },
- {
- "id": "61",
- "name": "RETAIL_YEAR",
- "datatype": "string"
- },
- {
- "id": "62",
- "name": "RETAIL_START_DATE",
- "datatype": "string"
- },
- {
- "id": "63",
- "name": "RETAIL_WK_END_DATE",
- "datatype": "string"
- },
- {
- "id": "64",
- "name": "WEEK_IND",
- "datatype": "string"
- },
- {
- "id": "65",
- "name": "WEEK_NUM_DESC",
- "datatype": "string"
- },
- {
- "id": "66",
- "name": "WEEK_BEG_DATE",
- "datatype": "string"
- },
- {
- "id": "67",
- "name": "WEEK_END_DATE",
- "datatype": "string"
- },
- {
- "id": "68",
- "name": "WEEK_IN_YEAR_ID",
- "datatype": "string"
- },
- {
- "id": "69",
- "name": "WEEK_ID",
- "datatype": "string"
- },
- {
- "id": "70",
- "name": "WEEK_BEG_END_DESC_MDY",
- "datatype": "string"
- },
- {
- "id": "71",
- "name": "WEEK_BEG_END_DESC_MD",
- "datatype": "string"
- },
- {
- "id": "72",
- "name": "YEAR_ID",
- "datatype": "string"
- },
- {
- "id": "73",
- "name": "YEAR_IND",
- "datatype": "string"
- },
- {
- "id": "74",
- "name": "CAL_DT_MNS_1YEAR_DT",
- "datatype": "string"
- },
- {
- "id": "75",
- "name": "CAL_DT_MNS_2YEAR_DT",
- "datatype": "string"
- },
- {
- "id": "76",
- "name": "CAL_DT_MNS_1QTR_DT",
- "datatype": "string"
- },
- {
- "id": "77",
- "name": "CAL_DT_MNS_2QTR_DT",
- "datatype": "string"
- },
- {
- "id": "78",
- "name": "CAL_DT_MNS_1MONTH_DT",
- "datatype": "string"
- },
- {
- "id": "79",
- "name": "CAL_DT_MNS_2MONTH_DT",
- "datatype": "string"
- },
- {
- "id": "80",
- "name": "CAL_DT_MNS_1WEEK_DT",
- "datatype": "string"
- },
- {
- "id": "81",
- "name": "CAL_DT_MNS_2WEEK_DT",
- "datatype": "string"
- },
- {
- "id": "82",
- "name": "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "83",
- "name": "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "84",
- "name": "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "85",
- "name": "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "86",
- "name": "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "87",
- "name": "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "88",
- "name": "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype": "tinyint"
- },
- {
- "id": "89",
- "name": "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype": "tinyint"
- },
- {
- "id": "90",
- "name": "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype": "string"
- },
- {
- "id": "91",
- "name": "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "92",
- "name": "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "93",
- "name": "SEASON_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "94",
- "name": "YTM_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "95",
- "name": "YTQ_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "96",
- "name": "YTW_YN_ID",
- "datatype": "tinyint"
- }
- ],
- "database": "edw",
- "last_modified": 0
+
+ "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+ "name" : "TEST_CAL_DT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "CAL_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "2",
+ "name" : "YEAR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "QTR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "4",
+ "name" : "MONTH_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "5",
+ "name" : "WEEK_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "6",
+ "name" : "AGE_FOR_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "AGE_FOR_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "8",
+ "name" : "AGE_FOR_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "9",
+ "name" : "AGE_FOR_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "10",
+ "name" : "AGE_FOR_DT_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "11",
+ "name" : "AGE_FOR_RTL_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "12",
+ "name" : "AGE_FOR_RTL_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "13",
+ "name" : "AGE_FOR_RTL_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "14",
+ "name" : "AGE_FOR_RTL_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "15",
+ "name" : "AGE_FOR_CS_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "16",
+ "name" : "DAY_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "17",
+ "name" : "DAY_OF_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "18",
+ "name" : "DAY_OF_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "19",
+ "name" : "DAY_OF_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "20",
+ "name" : "DAY_OF_WEEK_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "21",
+ "name" : "WEEK_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "22",
+ "name" : "WEEK_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "23",
+ "name" : "MONTH_OF_QTR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "24",
+ "name" : "MONTH_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "25",
+ "name" : "MONTH_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "26",
+ "name" : "QTR_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "27",
+ "name" : "QTR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "28",
+ "name" : "YEAR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "29",
+ "name" : "YEAR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "30",
+ "name" : "QTR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "MONTH_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "32",
+ "name" : "WEEK_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "33",
+ "name" : "CAL_DT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "34",
+ "name" : "CAL_DT_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "35",
+ "name" : "CAL_DT_SHORT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "36",
+ "name" : "YTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "37",
+ "name" : "QTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "38",
+ "name" : "MTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "39",
+ "name" : "WTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "40",
+ "name" : "SEASON_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "41",
+ "name" : "DAY_IN_YEAR_COUNT",
+ "datatype" : "smallint"
+ }, {
+ "id" : "42",
+ "name" : "DAY_IN_QTR_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "43",
+ "name" : "DAY_IN_MONTH_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "44",
+ "name" : "DAY_IN_WEEK_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "45",
+ "name" : "RTL_YEAR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "46",
+ "name" : "RTL_QTR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "47",
+ "name" : "RTL_MONTH_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "48",
+ "name" : "RTL_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "49",
+ "name" : "CS_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "50",
+ "name" : "CAL_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "51",
+ "name" : "DAY_OF_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "52",
+ "name" : "MONTH_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "53",
+ "name" : "PRD_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "54",
+ "name" : "PRD_FLAG",
+ "datatype" : "string"
+ }, {
+ "id" : "55",
+ "name" : "PRD_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "56",
+ "name" : "PRD_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "57",
+ "name" : "QTR_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "58",
+ "name" : "QTR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "59",
+ "name" : "QTR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "60",
+ "name" : "RETAIL_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "61",
+ "name" : "RETAIL_YEAR",
+ "datatype" : "string"
+ }, {
+ "id" : "62",
+ "name" : "RETAIL_START_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "63",
+ "name" : "RETAIL_WK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "64",
+ "name" : "WEEK_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "65",
+ "name" : "WEEK_NUM_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "66",
+ "name" : "WEEK_BEG_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "67",
+ "name" : "WEEK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "68",
+ "name" : "WEEK_IN_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "69",
+ "name" : "WEEK_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "70",
+ "name" : "WEEK_BEG_END_DESC_MDY",
+ "datatype" : "string"
+ }, {
+ "id" : "71",
+ "name" : "WEEK_BEG_END_DESC_MD",
+ "datatype" : "string"
+ }, {
+ "id" : "72",
+ "name" : "YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "73",
+ "name" : "YEAR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "74",
+ "name" : "CAL_DT_MNS_1YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "75",
+ "name" : "CAL_DT_MNS_2YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "76",
+ "name" : "CAL_DT_MNS_1QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "77",
+ "name" : "CAL_DT_MNS_2QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "78",
+ "name" : "CAL_DT_MNS_1MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "79",
+ "name" : "CAL_DT_MNS_2MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "80",
+ "name" : "CAL_DT_MNS_1WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "81",
+ "name" : "CAL_DT_MNS_2WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "82",
+ "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "83",
+ "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "84",
+ "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "85",
+ "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "86",
+ "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "87",
+ "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "88",
+ "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "89",
+ "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "90",
+ "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "91",
+ "name" : "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "92",
+ "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "93",
+ "name" : "SEASON_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "94",
+ "name" : "YTM_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "95",
+ "name" : "YTQ_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "96",
+ "name" : "YTW_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "97",
+ "name" : "CAL_DT_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "98",
+ "name" : "CAL_DT_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "99",
+ "name" : "CAL_DT_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "100",
+ "name" : "CAL_DT_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "edw",
+ "last_modified" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/894c16f2/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
index 6bc2815..eec48ea 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
@@ -24,19 +24,19 @@
"datatype" : "string"
}, {
"id" : "6",
- "name" : "CRE_DATE",
+ "name" : "DIM_CRE_DATE",
"datatype" : "string"
}, {
"id" : "7",
- "name" : "CRE_USER",
+ "name" : "DIM_CRE_USER",
"datatype" : "string"
}, {
"id" : "8",
- "name" : "UPD_DATE",
+ "name" : "DIM_UPD_DATE",
"datatype" : "string"
}, {
"id" : "9",
- "name" : "UPD_USER",
+ "name" : "DIM_UPD_USER",
"datatype" : "string"
} ],
"database" : "edw",
http://git-wip-us.apache.org/repos/asf/kylin/blob/894c16f2/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
index d5cc116..06a0b2e 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
@@ -32,7 +32,7 @@
"datatype" : "string"
}, {
"id" : "8",
- "name" : "UPD_DATE",
+ "name" : "SITES_UPD_DATE",
"datatype" : "string"
}, {
"id" : "9",
@@ -40,7 +40,7 @@
"datatype" : "string"
}, {
"id" : "10",
- "name" : "UPD_USER",
+ "name" : "SITES_UPD_USER",
"datatype" : "string"
} ],
"database" : "edw",
http://git-wip-us.apache.org/repos/asf/kylin/blob/894c16f2/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
index c48b9e2..9298c7f 100644
--- a/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
+++ b/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
@@ -1,484 +1,409 @@
{
- "uuid": "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- "name": "V_TEST_CAL_DT",
- "columns": [
- {
- "id": "1",
- "name": "CAL_DT",
- "datatype": "date"
- },
- {
- "id": "2",
- "name": "YEAR_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "3",
- "name": "QTR_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "4",
- "name": "MONTH_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "5",
- "name": "WEEK_BEG_DT",
- "datatype": "date"
- },
- {
- "id": "6",
- "name": "AGE_FOR_YEAR_ID",
- "datatype": "smallint"
- },
- {
- "id": "7",
- "name": "AGE_FOR_QTR_ID",
- "datatype": "smallint"
- },
- {
- "id": "8",
- "name": "AGE_FOR_MONTH_ID",
- "datatype": "smallint"
- },
- {
- "id": "9",
- "name": "AGE_FOR_WEEK_ID",
- "datatype": "smallint"
- },
- {
- "id": "10",
- "name": "AGE_FOR_DT_ID",
- "datatype": "smallint"
- },
- {
- "id": "11",
- "name": "AGE_FOR_RTL_YEAR_ID",
- "datatype": "smallint"
- },
- {
- "id": "12",
- "name": "AGE_FOR_RTL_QTR_ID",
- "datatype": "smallint"
- },
- {
- "id": "13",
- "name": "AGE_FOR_RTL_MONTH_ID",
- "datatype": "smallint"
- },
- {
- "id": "14",
- "name": "AGE_FOR_RTL_WEEK_ID",
- "datatype": "smallint"
- },
- {
- "id": "15",
- "name": "AGE_FOR_CS_WEEK_ID",
- "datatype": "smallint"
- },
- {
- "id": "16",
- "name": "DAY_OF_CAL_ID",
- "datatype": "int"
- },
- {
- "id": "17",
- "name": "DAY_OF_YEAR_ID",
- "datatype": "smallint"
- },
- {
- "id": "18",
- "name": "DAY_OF_QTR_ID",
- "datatype": "smallint"
- },
- {
- "id": "19",
- "name": "DAY_OF_MONTH_ID",
- "datatype": "smallint"
- },
- {
- "id": "20",
- "name": "DAY_OF_WEEK_ID",
- "datatype": "int"
- },
- {
- "id": "21",
- "name": "WEEK_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "22",
- "name": "WEEK_OF_CAL_ID",
- "datatype": "int"
- },
- {
- "id": "23",
- "name": "MONTH_OF_QTR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "24",
- "name": "MONTH_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "25",
- "name": "MONTH_OF_CAL_ID",
- "datatype": "smallint"
- },
- {
- "id": "26",
- "name": "QTR_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "27",
- "name": "QTR_OF_CAL_ID",
- "datatype": "smallint"
- },
- {
- "id": "28",
- "name": "YEAR_OF_CAL_ID",
- "datatype": "smallint"
- },
- {
- "id": "29",
- "name": "YEAR_END_DT",
- "datatype": "string"
- },
- {
- "id": "30",
- "name": "QTR_END_DT",
- "datatype": "string"
- },
- {
- "id": "31",
- "name": "MONTH_END_DT",
- "datatype": "string"
- },
- {
- "id": "32",
- "name": "WEEK_END_DT",
- "datatype": "string"
- },
- {
- "id": "33",
- "name": "CAL_DT_NAME",
- "datatype": "string"
- },
- {
- "id": "34",
- "name": "CAL_DT_DESC",
- "datatype": "string"
- },
- {
- "id": "35",
- "name": "CAL_DT_SHORT_NAME",
- "datatype": "string"
- },
- {
- "id": "36",
- "name": "YTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "37",
- "name": "QTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "38",
- "name": "MTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "39",
- "name": "WTD_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "40",
- "name": "SEASON_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "41",
- "name": "DAY_IN_YEAR_COUNT",
- "datatype": "smallint"
- },
- {
- "id": "42",
- "name": "DAY_IN_QTR_COUNT",
- "datatype": "tinyint"
- },
- {
- "id": "43",
- "name": "DAY_IN_MONTH_COUNT",
- "datatype": "tinyint"
- },
- {
- "id": "44",
- "name": "DAY_IN_WEEK_COUNT",
- "datatype": "tinyint"
- },
- {
- "id": "45",
- "name": "RTL_YEAR_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "46",
- "name": "RTL_QTR_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "47",
- "name": "RTL_MONTH_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "48",
- "name": "RTL_WEEK_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "49",
- "name": "CS_WEEK_BEG_DT",
- "datatype": "string"
- },
- {
- "id": "50",
- "name": "CAL_DATE",
- "datatype": "string"
- },
- {
- "id": "51",
- "name": "DAY_OF_WEEK",
- "datatype": "string"
- },
- {
- "id": "52",
- "name": "MONTH_ID",
- "datatype": "string"
- },
- {
- "id": "53",
- "name": "PRD_DESC",
- "datatype": "string"
- },
- {
- "id": "54",
- "name": "PRD_FLAG",
- "datatype": "string"
- },
- {
- "id": "55",
- "name": "PRD_ID",
- "datatype": "string"
- },
- {
- "id": "56",
- "name": "PRD_IND",
- "datatype": "string"
- },
- {
- "id": "57",
- "name": "QTR_DESC",
- "datatype": "string"
- },
- {
- "id": "58",
- "name": "QTR_ID",
- "datatype": "string"
- },
- {
- "id": "59",
- "name": "QTR_IND",
- "datatype": "string"
- },
- {
- "id": "60",
- "name": "RETAIL_WEEK",
- "datatype": "string"
- },
- {
- "id": "61",
- "name": "RETAIL_YEAR",
- "datatype": "string"
- },
- {
- "id": "62",
- "name": "RETAIL_START_DATE",
- "datatype": "string"
- },
- {
- "id": "63",
- "name": "RETAIL_WK_END_DATE",
- "datatype": "string"
- },
- {
- "id": "64",
- "name": "WEEK_IND",
- "datatype": "string"
- },
- {
- "id": "65",
- "name": "WEEK_NUM_DESC",
- "datatype": "string"
- },
- {
- "id": "66",
- "name": "WEEK_BEG_DATE",
- "datatype": "string"
- },
- {
- "id": "67",
- "name": "WEEK_END_DATE",
- "datatype": "string"
- },
- {
- "id": "68",
- "name": "WEEK_IN_YEAR_ID",
- "datatype": "string"
- },
- {
- "id": "69",
- "name": "WEEK_ID",
- "datatype": "string"
- },
- {
- "id": "70",
- "name": "WEEK_BEG_END_DESC_MDY",
- "datatype": "string"
- },
- {
- "id": "71",
- "name": "WEEK_BEG_END_DESC_MD",
- "datatype": "string"
- },
- {
- "id": "72",
- "name": "YEAR_ID",
- "datatype": "string"
- },
- {
- "id": "73",
- "name": "YEAR_IND",
- "datatype": "string"
- },
- {
- "id": "74",
- "name": "CAL_DT_MNS_1YEAR_DT",
- "datatype": "string"
- },
- {
- "id": "75",
- "name": "CAL_DT_MNS_2YEAR_DT",
- "datatype": "string"
- },
- {
- "id": "76",
- "name": "CAL_DT_MNS_1QTR_DT",
- "datatype": "string"
- },
- {
- "id": "77",
- "name": "CAL_DT_MNS_2QTR_DT",
- "datatype": "string"
- },
- {
- "id": "78",
- "name": "CAL_DT_MNS_1MONTH_DT",
- "datatype": "string"
- },
- {
- "id": "79",
- "name": "CAL_DT_MNS_2MONTH_DT",
- "datatype": "string"
- },
- {
- "id": "80",
- "name": "CAL_DT_MNS_1WEEK_DT",
- "datatype": "string"
- },
- {
- "id": "81",
- "name": "CAL_DT_MNS_2WEEK_DT",
- "datatype": "string"
- },
- {
- "id": "82",
- "name": "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "83",
- "name": "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "84",
- "name": "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "85",
- "name": "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "86",
- "name": "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "87",
- "name": "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "88",
- "name": "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype": "tinyint"
- },
- {
- "id": "89",
- "name": "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype": "tinyint"
- },
- {
- "id": "90",
- "name": "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype": "string"
- },
- {
- "id": "91",
- "name": "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "92",
- "name": "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "93",
- "name": "SEASON_OF_YEAR_ID",
- "datatype": "tinyint"
- },
- {
- "id": "94",
- "name": "YTM_YN_ID",
- "datatype": "tinyint"
- },
- {
- "id": "95",
- "name": "YTQ_YN_ID",
- "datatype": "tinyint"
- }
- ],
- "database": "edw",
- "table_type": "VIRTUAL_VIEW",
- "last_modified": 0
+
+ "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+ "name" : "V_TEST_CAL_DT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "CAL_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "2",
+ "name" : "YEAR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "QTR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "4",
+ "name" : "MONTH_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "5",
+ "name" : "WEEK_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "6",
+ "name" : "AGE_FOR_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "AGE_FOR_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "8",
+ "name" : "AGE_FOR_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "9",
+ "name" : "AGE_FOR_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "10",
+ "name" : "AGE_FOR_DT_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "11",
+ "name" : "AGE_FOR_RTL_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "12",
+ "name" : "AGE_FOR_RTL_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "13",
+ "name" : "AGE_FOR_RTL_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "14",
+ "name" : "AGE_FOR_RTL_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "15",
+ "name" : "AGE_FOR_CS_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "16",
+ "name" : "DAY_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "17",
+ "name" : "DAY_OF_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "18",
+ "name" : "DAY_OF_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "19",
+ "name" : "DAY_OF_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "20",
+ "name" : "DAY_OF_WEEK_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "21",
+ "name" : "WEEK_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "22",
+ "name" : "WEEK_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "23",
+ "name" : "MONTH_OF_QTR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "24",
+ "name" : "MONTH_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "25",
+ "name" : "MONTH_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "26",
+ "name" : "QTR_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "27",
+ "name" : "QTR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "28",
+ "name" : "YEAR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "29",
+ "name" : "YEAR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "30",
+ "name" : "QTR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "MONTH_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "32",
+ "name" : "WEEK_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "33",
+ "name" : "CAL_DT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "34",
+ "name" : "CAL_DT_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "35",
+ "name" : "CAL_DT_SHORT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "36",
+ "name" : "YTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "37",
+ "name" : "QTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "38",
+ "name" : "MTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "39",
+ "name" : "WTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "40",
+ "name" : "SEASON_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "41",
+ "name" : "DAY_IN_YEAR_COUNT",
+ "datatype" : "smallint"
+ }, {
+ "id" : "42",
+ "name" : "DAY_IN_QTR_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "43",
+ "name" : "DAY_IN_MONTH_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "44",
+ "name" : "DAY_IN_WEEK_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "45",
+ "name" : "RTL_YEAR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "46",
+ "name" : "RTL_QTR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "47",
+ "name" : "RTL_MONTH_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "48",
+ "name" : "RTL_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "49",
+ "name" : "CS_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "50",
+ "name" : "CAL_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "51",
+ "name" : "DAY_OF_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "52",
+ "name" : "MONTH_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "53",
+ "name" : "PRD_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "54",
+ "name" : "PRD_FLAG",
+ "datatype" : "string"
+ }, {
+ "id" : "55",
+ "name" : "PRD_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "56",
+ "name" : "PRD_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "57",
+ "name" : "QTR_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "58",
+ "name" : "QTR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "59",
+ "name" : "QTR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "60",
+ "name" : "RETAIL_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "61",
+ "name" : "RETAIL_YEAR",
+ "datatype" : "string"
+ }, {
+ "id" : "62",
+ "name" : "RETAIL_START_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "63",
+ "name" : "RETAIL_WK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "64",
+ "name" : "WEEK_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "65",
+ "name" : "WEEK_NUM_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "66",
+ "name" : "WEEK_BEG_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "67",
+ "name" : "WEEK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "68",
+ "name" : "WEEK_IN_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "69",
+ "name" : "WEEK_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "70",
+ "name" : "WEEK_BEG_END_DESC_MDY",
+ "datatype" : "string"
+ }, {
+ "id" : "71",
+ "name" : "WEEK_BEG_END_DESC_MD",
+ "datatype" : "string"
+ }, {
+ "id" : "72",
+ "name" : "YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "73",
+ "name" : "YEAR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "74",
+ "name" : "CAL_DT_MNS_1YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "75",
+ "name" : "CAL_DT_MNS_2YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "76",
+ "name" : "CAL_DT_MNS_1QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "77",
+ "name" : "CAL_DT_MNS_2QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "78",
+ "name" : "CAL_DT_MNS_1MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "79",
+ "name" : "CAL_DT_MNS_2MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "80",
+ "name" : "CAL_DT_MNS_1WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "81",
+ "name" : "CAL_DT_MNS_2WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "82",
+ "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "83",
+ "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "84",
+ "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "85",
+ "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "86",
+ "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "87",
+ "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "88",
+ "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "89",
+ "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "90",
+ "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "91",
+ "name" : "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "92",
+ "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "93",
+ "name" : "SEASON_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "94",
+ "name" : "YTM_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "95",
+ "name" : "YTQ_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "96",
+ "name" : "YTW_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "97",
+ "name" : "V_CAL_DT_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "98",
+ "name" : "V_CAL_DT_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "99",
+ "name" : "V_CAL_DT_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "100",
+ "name" : "V_CAL_DT_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "edw",
+ "table_type" : "VIRTUAL_VIEW",
+ "last_modified" : 0
}
\ No newline at end of file
[15/50] [abbrv] kylin git commit: KYLIN-1972 code review
Posted by li...@apache.org.
KYLIN-1972 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d046421
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d046421
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d046421
Branch: refs/heads/1.5.x-CDH5.7
Commit: 1d046421feb2c4845f4a833d6a2a7f3d07a5730b
Parents: 1ea79dd
Author: Yang Li <li...@apache.org>
Authored: Sat Aug 27 22:30:33 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Aug 27 22:30:33 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/QueryController.java | 42 +-------------------
.../apache/kylin/rest/service/QueryService.java | 31 ++++++++++++++-
2 files changed, 31 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d046421/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index a45f82e..5cf6492 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -29,9 +29,6 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.metadata.project.RealizationEntry;
-import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.metrics.QueryMetricsFacade;
@@ -46,7 +43,6 @@ import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.storage.exception.ScanOutOfLimitException;
-import org.apache.kylin.storage.hybrid.HybridInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -259,46 +255,10 @@ public class QueryController extends BasicController {
private void checkQueryAuth(SQLResponse sqlResponse) throws AccessDeniedException {
if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled()) {
- HybridInstance hybridInstance = this.queryService.getHybridManager().getHybridInstance(sqlResponse.getCube());
- if (hybridInstance != null) {
- if (checkHybridAuthorization(hybridInstance)) {
- logger.info(hybridInstance.getName() + " ACL validation success.");
- } else {
- throw new AccessDeniedException("Access is denied");
- }
- } else {
- CubeInstance cubeInstance = this.queryService.getCubeManager().getCube(sqlResponse.getCube());
- queryService.checkAuthorization(cubeInstance);
- }
+ queryService.checkAuthorization(sqlResponse.getCube());
}
}
- private boolean checkHybridAuthorization(HybridInstance hybridInstance) {
- boolean access = false;
- List<RealizationEntry> realizationEntries = hybridInstance.getRealizationEntries();
- for (RealizationEntry realizationEntry : realizationEntries) {
- String reName = realizationEntry.getRealization();
- logger.debug("[realizationEntry] realizationEntry name: " + reName + " realizationEntry type: " + realizationEntry.getType().name());
- if (RealizationType.CUBE == realizationEntry.getType()) {
- CubeInstance cubeInstance = queryService.getCubeManager().getCube(reName);
- try {
- queryService.checkAuthorization(cubeInstance);
- logger.info(hybridInstance.getName() + " ACL validation cube: " + cubeInstance.getName() + " success.");
- logger.info(hybridInstance.getName() + " ACL validation success.");
- return true;
- } catch (AccessDeniedException e) {
- logger.info(hybridInstance.getName() + " ACL validation cube: " + cubeInstance.getName() + " failed.");
- }
- } else if (RealizationType.HYBRID == realizationEntry.getType()) {
- HybridInstance innerHybridInstance = queryService.getHybridManager().getHybridInstance(reName);
- if (checkHybridAuthorization(innerHybridInstance)) {
- return true;
- }
- }
- }
- return access;
- }
-
public void setQueryService(QueryService queryService) {
this.queryService = queryService;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d046421/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 6d778d0..3acaeb8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -55,6 +55,8 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.model.ColumnMeta;
@@ -67,6 +69,7 @@ import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.rest.util.Serializer;
import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hybrid.HybridInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -258,8 +261,34 @@ public class QueryService extends BasicService {
logger.info(stringBuilder.toString());
}
+ public void checkAuthorization(String cubeName) throws AccessDeniedException {
+ // special care for hybrid
+ HybridInstance hybridInstance = getHybridManager().getHybridInstance(cubeName);
+ if (hybridInstance != null) {
+ checkHybridAuthorization(hybridInstance);
+ return;
+ }
+
+ CubeInstance cubeInstance = getCubeManager().getCube(cubeName);
+ checkCubeAuthorization(cubeInstance);
+ }
+
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')" + " or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'READ')")
- public void checkAuthorization(CubeInstance cube) throws AccessDeniedException {
+ private void checkCubeAuthorization(CubeInstance cube) throws AccessDeniedException {
+ }
+
+ private void checkHybridAuthorization(HybridInstance hybridInstance) throws AccessDeniedException {
+ List<RealizationEntry> realizationEntries = hybridInstance.getRealizationEntries();
+ for (RealizationEntry realizationEntry : realizationEntries) {
+ String reName = realizationEntry.getRealization();
+ if (RealizationType.CUBE == realizationEntry.getType()) {
+ CubeInstance cubeInstance = getCubeManager().getCube(reName);
+ checkCubeAuthorization(cubeInstance);
+ } else if (RealizationType.HYBRID == realizationEntry.getType()) {
+ HybridInstance innerHybridInstance = getHybridManager().getHybridInstance(reName);
+ checkHybridAuthorization(innerHybridInstance);
+ }
+ }
}
private SQLResponse queryWithSqlMassage(SQLRequest sqlRequest) throws Exception {
[36/50] [abbrv] kylin git commit: minor, change sample cal_dt too
Posted by li...@apache.org.
minor, change sample cal_dt too
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9702dff2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9702dff2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9702dff2
Branch: refs/heads/1.5.x-CDH5.7
Commit: 9702dff253e680b46b40a35359ed7cecb5b38665
Parents: eb99105
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Aug 31 19:17:43 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Aug 31 19:17:43 2016 +0800
----------------------------------------------------------------------
.../metadata/table/DEFAULT.KYLIN_CAL_DT.json | 887 ++++++++++---------
1 file changed, 481 insertions(+), 406 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9702dff2/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
index 225d476..de404f1 100644
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
+++ b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
@@ -1,408 +1,483 @@
{
- "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
-
- "name" : "KYLIN_CAL_DT",
- "columns" : [ {
- "id" : "1",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "2",
- "name" : "YEAR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "QTR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "4",
- "name" : "MONTH_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "5",
- "name" : "WEEK_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "6",
- "name" : "AGE_FOR_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "AGE_FOR_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "8",
- "name" : "AGE_FOR_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "9",
- "name" : "AGE_FOR_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "10",
- "name" : "AGE_FOR_DT_ID",
- "datatype" : "smallint"
- }, {
- "id" : "11",
- "name" : "AGE_FOR_RTL_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "12",
- "name" : "AGE_FOR_RTL_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "13",
- "name" : "AGE_FOR_RTL_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "14",
- "name" : "AGE_FOR_RTL_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "15",
- "name" : "AGE_FOR_CS_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "16",
- "name" : "DAY_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "17",
- "name" : "DAY_OF_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "18",
- "name" : "DAY_OF_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "19",
- "name" : "DAY_OF_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "20",
- "name" : "DAY_OF_WEEK_ID",
- "datatype" : "int"
- }, {
- "id" : "21",
- "name" : "WEEK_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "22",
- "name" : "WEEK_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "23",
- "name" : "MONTH_OF_QTR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "24",
- "name" : "MONTH_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "25",
- "name" : "MONTH_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "26",
- "name" : "QTR_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "27",
- "name" : "QTR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "28",
- "name" : "YEAR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "29",
- "name" : "YEAR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "30",
- "name" : "QTR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "MONTH_END_DT",
- "datatype" : "string"
- }, {
- "id" : "32",
- "name" : "WEEK_END_DT",
- "datatype" : "string"
- }, {
- "id" : "33",
- "name" : "CAL_DT_NAME",
- "datatype" : "string"
- }, {
- "id" : "34",
- "name" : "CAL_DT_DESC",
- "datatype" : "string"
- }, {
- "id" : "35",
- "name" : "CAL_DT_SHORT_NAME",
- "datatype" : "string"
- }, {
- "id" : "36",
- "name" : "YTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "37",
- "name" : "QTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "38",
- "name" : "MTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "39",
- "name" : "WTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "40",
- "name" : "SEASON_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "41",
- "name" : "DAY_IN_YEAR_COUNT",
- "datatype" : "smallint"
- }, {
- "id" : "42",
- "name" : "DAY_IN_QTR_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "43",
- "name" : "DAY_IN_MONTH_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "44",
- "name" : "DAY_IN_WEEK_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "45",
- "name" : "RTL_YEAR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "46",
- "name" : "RTL_QTR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "47",
- "name" : "RTL_MONTH_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "48",
- "name" : "RTL_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "49",
- "name" : "CS_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "50",
- "name" : "CAL_DATE",
- "datatype" : "string"
- }, {
- "id" : "51",
- "name" : "DAY_OF_WEEK",
- "datatype" : "string"
- }, {
- "id" : "52",
- "name" : "MONTH_ID",
- "datatype" : "string"
- }, {
- "id" : "53",
- "name" : "PRD_DESC",
- "datatype" : "string"
- }, {
- "id" : "54",
- "name" : "PRD_FLAG",
- "datatype" : "string"
- }, {
- "id" : "55",
- "name" : "PRD_ID",
- "datatype" : "string"
- }, {
- "id" : "56",
- "name" : "PRD_IND",
- "datatype" : "string"
- }, {
- "id" : "57",
- "name" : "QTR_DESC",
- "datatype" : "string"
- }, {
- "id" : "58",
- "name" : "QTR_ID",
- "datatype" : "string"
- }, {
- "id" : "59",
- "name" : "QTR_IND",
- "datatype" : "string"
- }, {
- "id" : "60",
- "name" : "RETAIL_WEEK",
- "datatype" : "string"
- }, {
- "id" : "61",
- "name" : "RETAIL_YEAR",
- "datatype" : "string"
- }, {
- "id" : "62",
- "name" : "RETAIL_START_DATE",
- "datatype" : "string"
- }, {
- "id" : "63",
- "name" : "RETAIL_WK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "64",
- "name" : "WEEK_IND",
- "datatype" : "string"
- }, {
- "id" : "65",
- "name" : "WEEK_NUM_DESC",
- "datatype" : "string"
- }, {
- "id" : "66",
- "name" : "WEEK_BEG_DATE",
- "datatype" : "string"
- }, {
- "id" : "67",
- "name" : "WEEK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "68",
- "name" : "WEEK_IN_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "69",
- "name" : "WEEK_ID",
- "datatype" : "string"
- }, {
- "id" : "70",
- "name" : "WEEK_BEG_END_DESC_MDY",
- "datatype" : "string"
- }, {
- "id" : "71",
- "name" : "WEEK_BEG_END_DESC_MD",
- "datatype" : "string"
- }, {
- "id" : "72",
- "name" : "YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "73",
- "name" : "YEAR_IND",
- "datatype" : "string"
- }, {
- "id" : "74",
- "name" : "CAL_DT_MNS_1YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "75",
- "name" : "CAL_DT_MNS_2YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "76",
- "name" : "CAL_DT_MNS_1QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "77",
- "name" : "CAL_DT_MNS_2QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "78",
- "name" : "CAL_DT_MNS_1MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "79",
- "name" : "CAL_DT_MNS_2MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "80",
- "name" : "CAL_DT_MNS_1WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "81",
- "name" : "CAL_DT_MNS_2WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "82",
- "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "83",
- "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "84",
- "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "85",
- "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "86",
- "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "87",
- "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "88",
- "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "89",
- "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "90",
- "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "91",
- "name" : "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "92",
- "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "93",
- "name" : "SEASON_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "94",
- "name" : "YTM_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "95",
- "name" : "YTQ_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "96",
- "name" : "YTW_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "97",
- "name" : "CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "98",
- "name" : "CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "99",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "100",
- "name" : "UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "default",
- "last_modified" : 0
+ "uuid": "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+ "name": "KYLIN_CAL_DT",
+ "columns": [
+ {
+ "id": "1",
+ "name": "CAL_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "2",
+ "name": "YEAR_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "3",
+ "name": "QTR_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "4",
+ "name": "MONTH_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "5",
+ "name": "WEEK_BEG_DT",
+ "datatype": "date"
+ },
+ {
+ "id": "6",
+ "name": "AGE_FOR_YEAR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "7",
+ "name": "AGE_FOR_QTR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "8",
+ "name": "AGE_FOR_MONTH_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "9",
+ "name": "AGE_FOR_WEEK_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "10",
+ "name": "AGE_FOR_DT_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "11",
+ "name": "AGE_FOR_RTL_YEAR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "12",
+ "name": "AGE_FOR_RTL_QTR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "13",
+ "name": "AGE_FOR_RTL_MONTH_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "14",
+ "name": "AGE_FOR_RTL_WEEK_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "15",
+ "name": "AGE_FOR_CS_WEEK_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "16",
+ "name": "DAY_OF_CAL_ID",
+ "datatype": "int"
+ },
+ {
+ "id": "17",
+ "name": "DAY_OF_YEAR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "18",
+ "name": "DAY_OF_QTR_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "19",
+ "name": "DAY_OF_MONTH_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "20",
+ "name": "DAY_OF_WEEK_ID",
+ "datatype": "int"
+ },
+ {
+ "id": "21",
+ "name": "WEEK_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "22",
+ "name": "WEEK_OF_CAL_ID",
+ "datatype": "int"
+ },
+ {
+ "id": "23",
+ "name": "MONTH_OF_QTR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "24",
+ "name": "MONTH_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "25",
+ "name": "MONTH_OF_CAL_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "26",
+ "name": "QTR_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "27",
+ "name": "QTR_OF_CAL_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "28",
+ "name": "YEAR_OF_CAL_ID",
+ "datatype": "smallint"
+ },
+ {
+ "id": "29",
+ "name": "YEAR_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "30",
+ "name": "QTR_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "31",
+ "name": "MONTH_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "32",
+ "name": "WEEK_END_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "33",
+ "name": "CAL_DT_NAME",
+ "datatype": "string"
+ },
+ {
+ "id": "34",
+ "name": "CAL_DT_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "35",
+ "name": "CAL_DT_SHORT_NAME",
+ "datatype": "string"
+ },
+ {
+ "id": "36",
+ "name": "YTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "37",
+ "name": "QTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "38",
+ "name": "MTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "39",
+ "name": "WTD_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "40",
+ "name": "SEASON_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "41",
+ "name": "DAY_IN_YEAR_COUNT",
+ "datatype": "smallint"
+ },
+ {
+ "id": "42",
+ "name": "DAY_IN_QTR_COUNT",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "43",
+ "name": "DAY_IN_MONTH_COUNT",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "44",
+ "name": "DAY_IN_WEEK_COUNT",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "45",
+ "name": "RTL_YEAR_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "46",
+ "name": "RTL_QTR_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "47",
+ "name": "RTL_MONTH_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "48",
+ "name": "RTL_WEEK_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "49",
+ "name": "CS_WEEK_BEG_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "50",
+ "name": "CAL_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "51",
+ "name": "DAY_OF_WEEK",
+ "datatype": "string"
+ },
+ {
+ "id": "52",
+ "name": "MONTH_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "53",
+ "name": "PRD_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "54",
+ "name": "PRD_FLAG",
+ "datatype": "string"
+ },
+ {
+ "id": "55",
+ "name": "PRD_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "56",
+ "name": "PRD_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "57",
+ "name": "QTR_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "58",
+ "name": "QTR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "59",
+ "name": "QTR_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "60",
+ "name": "RETAIL_WEEK",
+ "datatype": "string"
+ },
+ {
+ "id": "61",
+ "name": "RETAIL_YEAR",
+ "datatype": "string"
+ },
+ {
+ "id": "62",
+ "name": "RETAIL_START_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "63",
+ "name": "RETAIL_WK_END_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "64",
+ "name": "WEEK_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "65",
+ "name": "WEEK_NUM_DESC",
+ "datatype": "string"
+ },
+ {
+ "id": "66",
+ "name": "WEEK_BEG_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "67",
+ "name": "WEEK_END_DATE",
+ "datatype": "string"
+ },
+ {
+ "id": "68",
+ "name": "WEEK_IN_YEAR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "69",
+ "name": "WEEK_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "70",
+ "name": "WEEK_BEG_END_DESC_MDY",
+ "datatype": "string"
+ },
+ {
+ "id": "71",
+ "name": "WEEK_BEG_END_DESC_MD",
+ "datatype": "string"
+ },
+ {
+ "id": "72",
+ "name": "YEAR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "73",
+ "name": "YEAR_IND",
+ "datatype": "string"
+ },
+ {
+ "id": "74",
+ "name": "CAL_DT_MNS_1YEAR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "75",
+ "name": "CAL_DT_MNS_2YEAR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "76",
+ "name": "CAL_DT_MNS_1QTR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "77",
+ "name": "CAL_DT_MNS_2QTR_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "78",
+ "name": "CAL_DT_MNS_1MONTH_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "79",
+ "name": "CAL_DT_MNS_2MONTH_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "80",
+ "name": "CAL_DT_MNS_1WEEK_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "81",
+ "name": "CAL_DT_MNS_2WEEK_DT",
+ "datatype": "string"
+ },
+ {
+ "id": "82",
+ "name": "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "83",
+ "name": "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "84",
+ "name": "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "85",
+ "name": "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "86",
+ "name": "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "87",
+ "name": "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "88",
+ "name": "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "89",
+ "name": "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "90",
+ "name": "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype": "string"
+ },
+ {
+ "id": "91",
+ "name": "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "92",
+ "name": "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "93",
+ "name": "SEASON_OF_YEAR_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "94",
+ "name": "YTM_YN_ID",
+ "datatype": "tinyint"
+ },
+ {
+ "id": "95",
+ "name": "YTQ_YN_ID",
+ "datatype": "tinyint"
+ }
+ ],
+ "database": "default",
+ "last_modified": 0
}
\ No newline at end of file
[05/50] [abbrv] kylin git commit: KYLIN-1909 Fix cube list ACL bug.
Posted by li...@apache.org.
KYLIN-1909 Fix cube list ACL bug.
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f41f8c55
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f41f8c55
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f41f8c55
Branch: refs/heads/1.5.x-CDH5.7
Commit: f41f8c55614172134105441f864b0c8a1f55583e
Parents: a863ae5
Author: sanjulian <ju...@hotmail.com>
Authored: Mon Aug 22 12:49:16 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Aug 22 13:31:04 2016 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/CubeController.java | 23 ++++++++++++++------
.../apache/kylin/rest/service/CubeService.java | 20 -----------------
2 files changed, 16 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f41f8c55/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 57b0965..ecdab40 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -20,12 +20,7 @@ package org.apache.kylin.rest.controller;
import java.io.IOException;
import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.JsonUtil;
@@ -98,7 +93,21 @@ public class CubeController extends BasicController {
@RequestMapping(value = "", method = { RequestMethod.GET })
@ResponseBody
public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
- return cubeService.getCubes(cubeName, projectName, modelName, limit, offset);
+ List<CubeInstance> cubes;
+ cubes = cubeService.listAllCubes(cubeName, projectName, modelName);
+
+ int climit = (null == limit) ? cubes.size() : limit;
+ int coffset = (null == offset) ? 0 : offset;
+
+ if (cubes.size() <= coffset) {
+ return Collections.emptyList();
+ }
+
+ if ((cubes.size() - coffset) < climit) {
+ return cubes.subList(coffset, cubes.size());
+ }
+
+ return cubes.subList(coffset, coffset + climit);
}
@RequestMapping(value = "validEncodings", method = { RequestMethod.GET })
http://git-wip-us.apache.org/repos/asf/kylin/blob/f41f8c55/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index cfb4cf8..4cd527c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -128,26 +128,6 @@ public class CubeService extends BasicService {
return filterCubes;
}
- @PostFilter(Constant.ACCESS_POST_FILTER_READ)
- public List<CubeInstance> getCubes(final String cubeName, final String projectName, final String modelName, final Integer limit, final Integer offset) {
-
- List<CubeInstance> cubes;
- cubes = listAllCubes(cubeName, projectName, modelName);
-
- int climit = (null == limit) ? cubes.size() : limit;
- int coffset = (null == offset) ? 0 : offset;
-
- if (cubes.size() <= coffset) {
- return Collections.emptyList();
- }
-
- if ((cubes.size() - coffset) < climit) {
- return cubes.subList(coffset, cubes.size());
- }
-
- return cubes.subList(coffset, coffset + climit);
- }
-
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public CubeInstance updateCubeCost(CubeInstance cube, int cost) throws IOException {
[06/50] [abbrv] kylin git commit: KYLIN-1909 Fix test and style
Posted by li...@apache.org.
KYLIN-1909 Fix test and style
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c897dd2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c897dd2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c897dd2
Branch: refs/heads/1.5.x-CDH5.7
Commit: 4c897dd2e8b008c3baaa970d8a0f50d88137026b
Parents: f41f8c5
Author: lidongsjtu <li...@apache.org>
Authored: Mon Aug 22 13:39:29 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Aug 22 13:39:29 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller/CubeController.java | 8 +++++++-
.../org/apache/kylin/rest/controller/CubeControllerTest.java | 6 ++++++
.../java/org/apache/kylin/rest/service/CubeServiceTest.java | 5 +----
3 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c897dd2/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index ecdab40..7081d02 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -20,7 +20,13 @@ package org.apache.kylin.rest.controller;
import java.io.IOException;
import java.net.UnknownHostException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.JsonUtil;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c897dd2/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index f6db8c1..6319899 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -163,4 +163,10 @@ public class CubeControllerTest extends ServiceTestBase {
Assert.assertTrue(segNumber == newSegNumber + 1);
}
+ @Test
+ public void testGetCubes() {
+ List<CubeInstance> cubes = cubeController.getCubes(null, null, null, 1, 0);
+ Assert.assertTrue(cubes.size() == 1);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c897dd2/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
index a3c983d..f98d6b9 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
@@ -48,12 +48,9 @@ public class CubeServiceTest extends ServiceTestBase {
Assert.assertNotNull(cubeService.getMetadataManager());
Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
- List<CubeInstance> cubes = cubeService.getCubes(null, null, null, null, null);
+ List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null);
Assert.assertNotNull(cubes);
CubeInstance cube = cubes.get(0);
cubeService.isCubeDescFreeEditable(cube.getDescriptor());
-
- cubes = cubeService.getCubes(null, null, null, 1, 0);
- Assert.assertTrue(cubes.size() == 1);
}
}
[40/50] [abbrv] kylin git commit: KYLIN-1993 Set default storage_type
andengine_type in kylin.properties
Posted by li...@apache.org.
KYLIN-1993 Set default storage_type andengine_type in kylin.properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7d32ecb0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7d32ecb0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7d32ecb0
Branch: refs/heads/1.5.x-CDH5.7
Commit: 7d32ecb0d7ff52910c8c106dbd244b79c181c428
Parents: 96da2b6
Author: lidongsjtu <li...@apache.org>
Authored: Thu Sep 1 19:15:01 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Sep 1 19:15:11 2016 +0800
----------------------------------------------------------------------
build/bin/sample.sh | 10 +
.../apache/kylin/common/KylinConfigBase.java | 8 +
.../metadata/cube/kylin_sales_cube.json | 12 -
.../cube_desc/kylin_sales_cube_desc.json | 183 ---------
.../metadata/model_desc/kylin_sales_model.json | 43 --
.../metadata/project/learn_kylin.json | 19 -
.../metadata/table/DEFAULT.KYLIN_CAL_DT.json | 408 -------------------
.../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json | 152 -------
.../metadata/table/DEFAULT.KYLIN_SALES.json | 44 --
.../template/cube/kylin_sales_cube.json | 12 +
.../cube_desc/kylin_sales_cube_desc.json | 183 +++++++++
.../template/model_desc/kylin_sales_model.json | 43 ++
.../template/project/learn_kylin.json | 19 +
.../template/table/DEFAULT.KYLIN_CAL_DT.json | 408 +++++++++++++++++++
.../table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json | 152 +++++++
.../template/table/DEFAULT.KYLIN_SALES.json | 44 ++
16 files changed, 879 insertions(+), 861 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 9939c62..06e05d1 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -41,6 +41,16 @@ fi
echo "Sample hive tables are created successfully; Going to create sample cube..."
hadoop fs -rm -r /tmp/kylin/sample_cube
+# set engine type and storage type to cube desc
+default_engine_type=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.default.cube.engine`
+default_storage_type=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.default.storage.engine`
+if [ -z "$default_engine_type" ]; then
+ default_engine_type=2
+ default_storage_type=2
+fi
+export default_engine_type
+export default_storage_type
+
cd ${KYLIN_HOME}
hbase org.apache.hadoop.util.RunJar ${job_jar} org.apache.kylin.common.persistence.ResourceTool upload ${KYLIN_HOME}/sample_cube/metadata || { exit 1; }
echo "Sample cube is created successfully in project 'learn_kylin'; Restart Kylin server or reload the metadata from web UI to see the change."
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index de6b977..1390e24 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -738,6 +738,14 @@ abstract public class KylinConfigBase implements Serializable {
return r;
}
+ public int getDefaultStorageEngine() {
+ return Integer.parseInt(getOptional("kylin.default.storage.engine", "2"));
+ }
+
+ public int getDefaultCubeEngine() {
+ return Integer.parseInt(getOptional("kylin.default.cube.engine", "2"));
+ }
+
public Map<Integer, String> getSchedulers() {
Map<Integer, String> r = convertKeyToInteger(getPropertiesByPrefix("kylin.scheduler."));
r.put(0, "org.apache.kylin.job.impl.threadpool.DefaultScheduler");
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/metadata/cube/kylin_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube/kylin_sales_cube.json b/examples/sample_cube/metadata/cube/kylin_sales_cube.json
deleted file mode 100644
index 86042ba..0000000
--- a/examples/sample_cube/metadata/cube/kylin_sales_cube.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "uuid" : "2fbca32a-a33e-4b69-83dd-0bb8b1f8c53b",
-
- "last_modified" : 0,
- "name" : "kylin_sales_cube",
- "owner" : null,
- "descriptor" : "kylin_sales_cube_desc",
- "status" : "DISABLED",
- "segments" : [ ],
- "create_time" : null,
- "auto_merge_time_ranges" : [604800000, 2419200000]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
deleted file mode 100644
index b4aa205..0000000
--- a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+++ /dev/null
@@ -1,183 +0,0 @@
-{
- "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name" : "kylin_sales_cube_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "DEFAULT.KYLIN_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.KYLIN_SALES",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "SELLER_CNT_HLL",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- },
- "returntype" : "hllc(10)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "SELLER_FORMAT_CNT",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "LSTG_FORMAT_NAME",
- "next_parameter" : null
- },
- "returntype" : "hllc(10)"
- },
- "dependent_measure_ref" : null
- }, {
- "name": "TOP_SELLER",
- "function": {
- "expression": "TOP_N",
- "parameter": {
- "type": "column",
- "value": "PRICE",
- "next_parameter": {
- "type": "column",
- "value": "SELLER_ID",
- "next_parameter": null
- }
- },
- "returntype": "topn(100)"
- },
- "dependent_measure_ref": null
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "PART_DT",
- "encoding" : "dict"
- }, {
- "column" : "LEAF_CATEG_ID",
- "encoding" : "dict"
- }, {
- "column" : "META_CATEG_NAME",
- "encoding" : "dict"
- }, {
- "column" : "CATEG_LVL2_NAME",
- "encoding" : "dict"
- }, {
- "column" : "CATEG_LVL3_NAME",
- "encoding" : "dict"
- }, {
- "column" : "LSTG_FORMAT_NAME",
- "encoding" : "fixed_length:12"
- }, {
- "column" : "LSTG_SITE_ID",
- "encoding" : "dict"
- } ]
- },
- "last_modified" : 1451468470824,
- "model_name" : "kylin_sales_model",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
- } ]
- }, {
- "name" : "f2",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT", "TOP_SELLER" ]
- } ]
- } ]
- },
- "aggregation_groups" : [ {
- "includes" : [ "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "META_CATEG_NAME", "PART_DT" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : [ ],
- "joint_dims" : [ ]
- }
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
- "override_kylin_properties" : { },
- "partition_date_start" : 1325376000000
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/metadata/model_desc/kylin_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/model_desc/kylin_sales_model.json b/examples/sample_cube/metadata/model_desc/kylin_sales_model.json
deleted file mode 100644
index 7ae47c1..0000000
--- a/examples/sample_cube/metadata/model_desc/kylin_sales_model.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
- "uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f",
- "name" : "kylin_sales_model",
- "lookups" : [ {
- "table" : "DEFAULT.KYLIN_CAL_DT",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "PART_DT" ]
- }
- }, {
- "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }],
- "dimensions" : [ {
- "table" : "DEFAULT.KYLIN_CAL_DT",
- "columns" : ["CAL_DT", "WEEK_BEG_DT"]
- }, {
- "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
- "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
- }, {
- "table" : "DEFAULT.KYLIN_SALES",
- "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT" ]
- } ],
- "metrics": [
- "PRICE",
- "ITEM_COUNT",
- "SELLER_ID",
- "LSTG_FORMAT_NAME"
- ],
- "last_modified" : 1422435345362,
- "fact_table" : "DEFAULT.KYLIN_SALES",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : "DEFAULT.KYLIN_SALES.PART_DT",
- "partition_date_start" : 1325376000000,
- "partition_type" : "APPEND"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/metadata/project/learn_kylin.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/project/learn_kylin.json b/examples/sample_cube/metadata/project/learn_kylin.json
deleted file mode 100644
index e468214..0000000
--- a/examples/sample_cube/metadata/project/learn_kylin.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "uuid": "2fbca32a-a33e-4b69-83dd-0bb8b1f8c91b",
- "name": "learn_kylin",
- "realizations": [
- {
- "name": "kylin_sales_cube",
- "type": "CUBE",
- "realization": "kylin_sales_cube"
- }
- ],
- "tables": [
- "DEFAULT.KYLIN_SALES",
- "DEFAULT.KYLIN_CAL_DT",
- "DEFAULT.KYLIN_CATEGORY_GROUPINGS"
- ],
- "models": [
- "kylin_sales_model"
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
deleted file mode 100644
index d197115..0000000
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CAL_DT.json
+++ /dev/null
@@ -1,408 +0,0 @@
-{
- "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
-
- "name" : "KYLIN_CAL_DT",
- "columns" : [ {
- "id" : "1",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "2",
- "name" : "YEAR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "QTR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "4",
- "name" : "MONTH_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "5",
- "name" : "WEEK_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "6",
- "name" : "AGE_FOR_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "AGE_FOR_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "8",
- "name" : "AGE_FOR_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "9",
- "name" : "AGE_FOR_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "10",
- "name" : "AGE_FOR_DT_ID",
- "datatype" : "smallint"
- }, {
- "id" : "11",
- "name" : "AGE_FOR_RTL_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "12",
- "name" : "AGE_FOR_RTL_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "13",
- "name" : "AGE_FOR_RTL_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "14",
- "name" : "AGE_FOR_RTL_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "15",
- "name" : "AGE_FOR_CS_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "16",
- "name" : "DAY_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "17",
- "name" : "DAY_OF_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "18",
- "name" : "DAY_OF_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "19",
- "name" : "DAY_OF_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "20",
- "name" : "DAY_OF_WEEK_ID",
- "datatype" : "int"
- }, {
- "id" : "21",
- "name" : "WEEK_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "22",
- "name" : "WEEK_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "23",
- "name" : "MONTH_OF_QTR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "24",
- "name" : "MONTH_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "25",
- "name" : "MONTH_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "26",
- "name" : "QTR_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "27",
- "name" : "QTR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "28",
- "name" : "YEAR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "29",
- "name" : "YEAR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "30",
- "name" : "QTR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "MONTH_END_DT",
- "datatype" : "string"
- }, {
- "id" : "32",
- "name" : "WEEK_END_DT",
- "datatype" : "string"
- }, {
- "id" : "33",
- "name" : "CAL_DT_NAME",
- "datatype" : "string"
- }, {
- "id" : "34",
- "name" : "CAL_DT_DESC",
- "datatype" : "string"
- }, {
- "id" : "35",
- "name" : "CAL_DT_SHORT_NAME",
- "datatype" : "string"
- }, {
- "id" : "36",
- "name" : "YTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "37",
- "name" : "QTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "38",
- "name" : "MTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "39",
- "name" : "WTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "40",
- "name" : "SEASON_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "41",
- "name" : "DAY_IN_YEAR_COUNT",
- "datatype" : "smallint"
- }, {
- "id" : "42",
- "name" : "DAY_IN_QTR_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "43",
- "name" : "DAY_IN_MONTH_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "44",
- "name" : "DAY_IN_WEEK_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "45",
- "name" : "RTL_YEAR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "46",
- "name" : "RTL_QTR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "47",
- "name" : "RTL_MONTH_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "48",
- "name" : "RTL_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "49",
- "name" : "CS_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "50",
- "name" : "CAL_DATE",
- "datatype" : "string"
- }, {
- "id" : "51",
- "name" : "DAY_OF_WEEK",
- "datatype" : "string"
- }, {
- "id" : "52",
- "name" : "MONTH_ID",
- "datatype" : "string"
- }, {
- "id" : "53",
- "name" : "PRD_DESC",
- "datatype" : "string"
- }, {
- "id" : "54",
- "name" : "PRD_FLAG",
- "datatype" : "string"
- }, {
- "id" : "55",
- "name" : "PRD_ID",
- "datatype" : "string"
- }, {
- "id" : "56",
- "name" : "PRD_IND",
- "datatype" : "string"
- }, {
- "id" : "57",
- "name" : "QTR_DESC",
- "datatype" : "string"
- }, {
- "id" : "58",
- "name" : "QTR_ID",
- "datatype" : "string"
- }, {
- "id" : "59",
- "name" : "QTR_IND",
- "datatype" : "string"
- }, {
- "id" : "60",
- "name" : "RETAIL_WEEK",
- "datatype" : "string"
- }, {
- "id" : "61",
- "name" : "RETAIL_YEAR",
- "datatype" : "string"
- }, {
- "id" : "62",
- "name" : "RETAIL_START_DATE",
- "datatype" : "string"
- }, {
- "id" : "63",
- "name" : "RETAIL_WK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "64",
- "name" : "WEEK_IND",
- "datatype" : "string"
- }, {
- "id" : "65",
- "name" : "WEEK_NUM_DESC",
- "datatype" : "string"
- }, {
- "id" : "66",
- "name" : "WEEK_BEG_DATE",
- "datatype" : "string"
- }, {
- "id" : "67",
- "name" : "WEEK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "68",
- "name" : "WEEK_IN_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "69",
- "name" : "WEEK_ID",
- "datatype" : "string"
- }, {
- "id" : "70",
- "name" : "WEEK_BEG_END_DESC_MDY",
- "datatype" : "string"
- }, {
- "id" : "71",
- "name" : "WEEK_BEG_END_DESC_MD",
- "datatype" : "string"
- }, {
- "id" : "72",
- "name" : "YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "73",
- "name" : "YEAR_IND",
- "datatype" : "string"
- }, {
- "id" : "74",
- "name" : "CAL_DT_MNS_1YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "75",
- "name" : "CAL_DT_MNS_2YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "76",
- "name" : "CAL_DT_MNS_1QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "77",
- "name" : "CAL_DT_MNS_2QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "78",
- "name" : "CAL_DT_MNS_1MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "79",
- "name" : "CAL_DT_MNS_2MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "80",
- "name" : "CAL_DT_MNS_1WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "81",
- "name" : "CAL_DT_MNS_2WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "82",
- "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "83",
- "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "84",
- "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "85",
- "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "86",
- "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "87",
- "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "88",
- "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "89",
- "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "90",
- "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "91",
- "name" : "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "92",
- "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "93",
- "name" : "SEASON_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "94",
- "name" : "YTM_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "95",
- "name" : "YTQ_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "96",
- "name" : "YTW_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "97",
- "name" : "KYLIN_CAL_DT_CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "98",
- "name" : "KYLIN_CAL_DT_CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "99",
- "name" : "KYLIN_CAL_DT_UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "100",
- "name" : "KYLIN_CAL_DT_UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "default",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
deleted file mode 100644
index 8e97309..0000000
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
+++ /dev/null
@@ -1,152 +0,0 @@
-{
- "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
-
- "name" : "KYLIN_CATEGORY_GROUPINGS",
- "columns" : [ {
- "id" : "1",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "LEAF_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "3",
- "name" : "SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "4",
- "name" : "CATEG_BUSN_MGR",
- "datatype" : "string"
- }, {
- "id" : "5",
- "name" : "CATEG_BUSN_UNIT",
- "datatype" : "string"
- }, {
- "id" : "6",
- "name" : "REGN_CATEG",
- "datatype" : "string"
- }, {
- "id" : "7",
- "name" : "USER_DEFINED_FIELD1",
- "datatype" : "string"
- }, {
- "id" : "8",
- "name" : "USER_DEFINED_FIELD3",
- "datatype" : "string"
- }, {
- "id" : "9",
- "name" : "KYLIN_GROUPINGS_CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "10",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "11",
- "name" : "KYLIN_GROUPINGS_CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "12",
- "name" : "UPD_USER",
- "datatype" : "string"
- }, {
- "id" : "13",
- "name" : "META_CATEG_ID",
- "datatype" : "decimal"
- }, {
- "id" : "14",
- "name" : "META_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "15",
- "name" : "CATEG_LVL2_ID",
- "datatype" : "decimal"
- }, {
- "id" : "16",
- "name" : "CATEG_LVL3_ID",
- "datatype" : "decimal"
- }, {
- "id" : "17",
- "name" : "CATEG_LVL4_ID",
- "datatype" : "decimal"
- }, {
- "id" : "18",
- "name" : "CATEG_LVL5_ID",
- "datatype" : "decimal"
- }, {
- "id" : "19",
- "name" : "CATEG_LVL6_ID",
- "datatype" : "decimal"
- }, {
- "id" : "20",
- "name" : "CATEG_LVL7_ID",
- "datatype" : "decimal"
- }, {
- "id" : "21",
- "name" : "CATEG_LVL2_NAME",
- "datatype" : "string"
- }, {
- "id" : "22",
- "name" : "CATEG_LVL3_NAME",
- "datatype" : "string"
- }, {
- "id" : "23",
- "name" : "CATEG_LVL4_NAME",
- "datatype" : "string"
- }, {
- "id" : "24",
- "name" : "CATEG_LVL5_NAME",
- "datatype" : "string"
- }, {
- "id" : "25",
- "name" : "CATEG_LVL6_NAME",
- "datatype" : "string"
- }, {
- "id" : "26",
- "name" : "CATEG_LVL7_NAME",
- "datatype" : "string"
- }, {
- "id" : "27",
- "name" : "CATEG_FLAGS",
- "datatype" : "decimal"
- }, {
- "id" : "28",
- "name" : "ADULT_CATEG_YN",
- "datatype" : "string"
- }, {
- "id" : "29",
- "name" : "DOMAIN_ID",
- "datatype" : "decimal"
- }, {
- "id" : "30",
- "name" : "USER_DEFINED_FIELD5",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "VCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "32",
- "name" : "GCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "33",
- "name" : "MOVE_TO",
- "datatype" : "decimal"
- }, {
- "id" : "34",
- "name" : "SAP_CATEGORY_ID",
- "datatype" : "decimal"
- }, {
- "id" : "35",
- "name" : "SRC_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "36",
- "name" : "BSNS_VRTCL_NAME",
- "datatype" : "string"
- } ],
- "database" : "DEFAULT",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json b/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json
deleted file mode 100644
index 4927343..0000000
--- a/examples/sample_cube/metadata/table/DEFAULT.KYLIN_SALES.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
-
- "name" : "KYLIN_SALES",
- "columns" : [ {
- "id" : "1",
- "name" : "TRANS_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "PART_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "LSTG_FORMAT_NAME",
- "datatype" : "string"
- }, {
- "id" : "4",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "5",
- "name" : "LSTG_SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "6",
- "name" : "SLR_SEGMENT_CD",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "PRICE",
- "datatype" : "decimal(19,4)"
- }, {
- "id" : "8",
- "name" : "ITEM_COUNT",
- "datatype" : "bigint"
- }, {
- "id" : "9",
- "name" : "SELLER_ID",
- "datatype" : "bigint"
- } ],
- "database" : "DEFAULT",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/template/cube/kylin_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/cube/kylin_sales_cube.json b/examples/sample_cube/template/cube/kylin_sales_cube.json
new file mode 100644
index 0000000..86042ba
--- /dev/null
+++ b/examples/sample_cube/template/cube/kylin_sales_cube.json
@@ -0,0 +1,12 @@
+{
+ "uuid" : "2fbca32a-a33e-4b69-83dd-0bb8b1f8c53b",
+
+ "last_modified" : 0,
+ "name" : "kylin_sales_cube",
+ "owner" : null,
+ "descriptor" : "kylin_sales_cube_desc",
+ "status" : "DISABLED",
+ "segments" : [ ],
+ "create_time" : null,
+ "auto_merge_time_ranges" : [604800000, 2419200000]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
new file mode 100644
index 0000000..0953be6
--- /dev/null
+++ b/examples/sample_cube/template/cube_desc/kylin_sales_cube_desc.json
@@ -0,0 +1,183 @@
+{
+ "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
+ "name" : "kylin_sales_cube_desc",
+ "description" : null,
+ "dimensions" : [ {
+ "name" : "CAL_DT",
+ "table" : "DEFAULT.KYLIN_CAL_DT",
+ "column" : "{FK}",
+ "derived" : [ "WEEK_BEG_DT" ]
+ }, {
+ "name" : "CATEGORY",
+ "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "{FK}",
+ "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "META_CATEG_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL2_NAME",
+ "derived" : null
+ }, {
+ "name" : "CATEGORY_HIERARCHY",
+ "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "column" : "CATEG_LVL3_NAME",
+ "derived" : null
+ }, {
+ "name" : "LSTG_FORMAT_NAME",
+ "table" : "DEFAULT.KYLIN_SALES",
+ "column" : "LSTG_FORMAT_NAME",
+ "derived" : null
+ } ],
+ "measures" : [ {
+ "name" : "GMV_SUM",
+ "function" : {
+ "expression" : "SUM",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MIN",
+ "function" : {
+ "expression" : "MIN",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "GMV_MAX",
+ "function" : {
+ "expression" : "MAX",
+ "parameter" : {
+ "type" : "column",
+ "value" : "PRICE",
+ "next_parameter" : null
+ },
+ "returntype" : "decimal(19,4)"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "TRANS_CNT",
+ "function" : {
+ "expression" : "COUNT",
+ "parameter" : {
+ "type" : "constant",
+ "value" : "1",
+ "next_parameter" : null
+ },
+ "returntype" : "bigint"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "SELLER_CNT_HLL",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "SELLER_ID",
+ "next_parameter" : null
+ },
+ "returntype" : "hllc(10)"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name" : "SELLER_FORMAT_CNT",
+ "function" : {
+ "expression" : "COUNT_DISTINCT",
+ "parameter" : {
+ "type" : "column",
+ "value" : "LSTG_FORMAT_NAME",
+ "next_parameter" : null
+ },
+ "returntype" : "hllc(10)"
+ },
+ "dependent_measure_ref" : null
+ }, {
+ "name": "TOP_SELLER",
+ "function": {
+ "expression": "TOP_N",
+ "parameter": {
+ "type": "column",
+ "value": "PRICE",
+ "next_parameter": {
+ "type": "column",
+ "value": "SELLER_ID",
+ "next_parameter": null
+ }
+ },
+ "returntype": "topn(100)"
+ },
+ "dependent_measure_ref": null
+ } ],
+ "rowkey" : {
+ "rowkey_columns" : [ {
+ "column" : "PART_DT",
+ "encoding" : "dict"
+ }, {
+ "column" : "LEAF_CATEG_ID",
+ "encoding" : "dict"
+ }, {
+ "column" : "META_CATEG_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "CATEG_LVL2_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "CATEG_LVL3_NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "LSTG_FORMAT_NAME",
+ "encoding" : "fixed_length:12"
+ }, {
+ "column" : "LSTG_SITE_ID",
+ "encoding" : "dict"
+ } ]
+ },
+ "last_modified" : 1451468470824,
+ "model_name" : "kylin_sales_model",
+ "null_string" : null,
+ "hbase_mapping" : {
+ "column_family" : [ {
+ "name" : "f1",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
+ } ]
+ }, {
+ "name" : "f2",
+ "columns" : [ {
+ "qualifier" : "m",
+ "measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT", "TOP_SELLER" ]
+ } ]
+ } ]
+ },
+ "aggregation_groups" : [ {
+ "includes" : [ "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "META_CATEG_NAME", "PART_DT" ],
+ "select_rule" : {
+ "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ ]
+ }
+ } ],
+ "notify_list" : null,
+ "status_need_notify" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : ${default_engine_type},
+ "storage_type" : ${default_storage_type},
+ "override_kylin_properties" : { },
+ "partition_date_start" : 1325376000000
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/template/model_desc/kylin_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json b/examples/sample_cube/template/model_desc/kylin_sales_model.json
new file mode 100644
index 0000000..7ae47c1
--- /dev/null
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -0,0 +1,43 @@
+{
+ "uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f",
+ "name" : "kylin_sales_model",
+ "lookups" : [ {
+ "table" : "DEFAULT.KYLIN_CAL_DT",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "CAL_DT" ],
+ "foreign_key" : [ "PART_DT" ]
+ }
+ }, {
+ "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "join" : {
+ "type" : "inner",
+ "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
+ "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
+ }
+ }],
+ "dimensions" : [ {
+ "table" : "DEFAULT.KYLIN_CAL_DT",
+ "columns" : ["CAL_DT", "WEEK_BEG_DT"]
+ }, {
+ "table" : "DEFAULT.KYLIN_CATEGORY_GROUPINGS",
+ "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ]
+ }, {
+ "table" : "DEFAULT.KYLIN_SALES",
+ "columns" : [ "LSTG_FORMAT_NAME", "SELLER_ID", "PART_DT" ]
+ } ],
+ "metrics": [
+ "PRICE",
+ "ITEM_COUNT",
+ "SELLER_ID",
+ "LSTG_FORMAT_NAME"
+ ],
+ "last_modified" : 1422435345362,
+ "fact_table" : "DEFAULT.KYLIN_SALES",
+ "filter_condition" : null,
+ "partition_desc" : {
+ "partition_date_column" : "DEFAULT.KYLIN_SALES.PART_DT",
+ "partition_date_start" : 1325376000000,
+ "partition_type" : "APPEND"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/template/project/learn_kylin.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/project/learn_kylin.json b/examples/sample_cube/template/project/learn_kylin.json
new file mode 100644
index 0000000..e468214
--- /dev/null
+++ b/examples/sample_cube/template/project/learn_kylin.json
@@ -0,0 +1,19 @@
+{
+ "uuid": "2fbca32a-a33e-4b69-83dd-0bb8b1f8c91b",
+ "name": "learn_kylin",
+ "realizations": [
+ {
+ "name": "kylin_sales_cube",
+ "type": "CUBE",
+ "realization": "kylin_sales_cube"
+ }
+ ],
+ "tables": [
+ "DEFAULT.KYLIN_SALES",
+ "DEFAULT.KYLIN_CAL_DT",
+ "DEFAULT.KYLIN_CATEGORY_GROUPINGS"
+ ],
+ "models": [
+ "kylin_sales_model"
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/template/table/DEFAULT.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_CAL_DT.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_CAL_DT.json
new file mode 100644
index 0000000..d197115
--- /dev/null
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_CAL_DT.json
@@ -0,0 +1,408 @@
+{
+ "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+
+ "name" : "KYLIN_CAL_DT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "CAL_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "2",
+ "name" : "YEAR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "QTR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "4",
+ "name" : "MONTH_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "5",
+ "name" : "WEEK_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "6",
+ "name" : "AGE_FOR_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "AGE_FOR_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "8",
+ "name" : "AGE_FOR_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "9",
+ "name" : "AGE_FOR_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "10",
+ "name" : "AGE_FOR_DT_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "11",
+ "name" : "AGE_FOR_RTL_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "12",
+ "name" : "AGE_FOR_RTL_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "13",
+ "name" : "AGE_FOR_RTL_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "14",
+ "name" : "AGE_FOR_RTL_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "15",
+ "name" : "AGE_FOR_CS_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "16",
+ "name" : "DAY_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "17",
+ "name" : "DAY_OF_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "18",
+ "name" : "DAY_OF_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "19",
+ "name" : "DAY_OF_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "20",
+ "name" : "DAY_OF_WEEK_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "21",
+ "name" : "WEEK_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "22",
+ "name" : "WEEK_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "23",
+ "name" : "MONTH_OF_QTR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "24",
+ "name" : "MONTH_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "25",
+ "name" : "MONTH_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "26",
+ "name" : "QTR_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "27",
+ "name" : "QTR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "28",
+ "name" : "YEAR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "29",
+ "name" : "YEAR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "30",
+ "name" : "QTR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "MONTH_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "32",
+ "name" : "WEEK_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "33",
+ "name" : "CAL_DT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "34",
+ "name" : "CAL_DT_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "35",
+ "name" : "CAL_DT_SHORT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "36",
+ "name" : "YTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "37",
+ "name" : "QTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "38",
+ "name" : "MTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "39",
+ "name" : "WTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "40",
+ "name" : "SEASON_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "41",
+ "name" : "DAY_IN_YEAR_COUNT",
+ "datatype" : "smallint"
+ }, {
+ "id" : "42",
+ "name" : "DAY_IN_QTR_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "43",
+ "name" : "DAY_IN_MONTH_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "44",
+ "name" : "DAY_IN_WEEK_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "45",
+ "name" : "RTL_YEAR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "46",
+ "name" : "RTL_QTR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "47",
+ "name" : "RTL_MONTH_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "48",
+ "name" : "RTL_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "49",
+ "name" : "CS_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "50",
+ "name" : "CAL_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "51",
+ "name" : "DAY_OF_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "52",
+ "name" : "MONTH_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "53",
+ "name" : "PRD_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "54",
+ "name" : "PRD_FLAG",
+ "datatype" : "string"
+ }, {
+ "id" : "55",
+ "name" : "PRD_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "56",
+ "name" : "PRD_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "57",
+ "name" : "QTR_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "58",
+ "name" : "QTR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "59",
+ "name" : "QTR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "60",
+ "name" : "RETAIL_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "61",
+ "name" : "RETAIL_YEAR",
+ "datatype" : "string"
+ }, {
+ "id" : "62",
+ "name" : "RETAIL_START_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "63",
+ "name" : "RETAIL_WK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "64",
+ "name" : "WEEK_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "65",
+ "name" : "WEEK_NUM_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "66",
+ "name" : "WEEK_BEG_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "67",
+ "name" : "WEEK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "68",
+ "name" : "WEEK_IN_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "69",
+ "name" : "WEEK_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "70",
+ "name" : "WEEK_BEG_END_DESC_MDY",
+ "datatype" : "string"
+ }, {
+ "id" : "71",
+ "name" : "WEEK_BEG_END_DESC_MD",
+ "datatype" : "string"
+ }, {
+ "id" : "72",
+ "name" : "YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "73",
+ "name" : "YEAR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "74",
+ "name" : "CAL_DT_MNS_1YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "75",
+ "name" : "CAL_DT_MNS_2YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "76",
+ "name" : "CAL_DT_MNS_1QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "77",
+ "name" : "CAL_DT_MNS_2QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "78",
+ "name" : "CAL_DT_MNS_1MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "79",
+ "name" : "CAL_DT_MNS_2MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "80",
+ "name" : "CAL_DT_MNS_1WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "81",
+ "name" : "CAL_DT_MNS_2WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "82",
+ "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "83",
+ "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "84",
+ "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "85",
+ "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "86",
+ "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "87",
+ "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "88",
+ "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "89",
+ "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "90",
+ "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "91",
+ "name" : "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "92",
+ "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "93",
+ "name" : "SEASON_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "94",
+ "name" : "YTM_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "95",
+ "name" : "YTQ_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "96",
+ "name" : "YTW_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "97",
+ "name" : "KYLIN_CAL_DT_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "98",
+ "name" : "KYLIN_CAL_DT_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "99",
+ "name" : "KYLIN_CAL_DT_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "100",
+ "name" : "KYLIN_CAL_DT_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "default",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
new file mode 100644
index 0000000..8e97309
--- /dev/null
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_CATEGORY_GROUPINGS.json
@@ -0,0 +1,152 @@
+{
+ "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
+
+ "name" : "KYLIN_CATEGORY_GROUPINGS",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "LEAF_CATEG_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "LEAF_CATEG_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "3",
+ "name" : "SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "4",
+ "name" : "CATEG_BUSN_MGR",
+ "datatype" : "string"
+ }, {
+ "id" : "5",
+ "name" : "CATEG_BUSN_UNIT",
+ "datatype" : "string"
+ }, {
+ "id" : "6",
+ "name" : "REGN_CATEG",
+ "datatype" : "string"
+ }, {
+ "id" : "7",
+ "name" : "USER_DEFINED_FIELD1",
+ "datatype" : "string"
+ }, {
+ "id" : "8",
+ "name" : "USER_DEFINED_FIELD3",
+ "datatype" : "string"
+ }, {
+ "id" : "9",
+ "name" : "KYLIN_GROUPINGS_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "10",
+ "name" : "UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "11",
+ "name" : "KYLIN_GROUPINGS_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "12",
+ "name" : "UPD_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "13",
+ "name" : "META_CATEG_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "14",
+ "name" : "META_CATEG_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "15",
+ "name" : "CATEG_LVL2_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "16",
+ "name" : "CATEG_LVL3_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "17",
+ "name" : "CATEG_LVL4_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "18",
+ "name" : "CATEG_LVL5_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "19",
+ "name" : "CATEG_LVL6_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "20",
+ "name" : "CATEG_LVL7_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "21",
+ "name" : "CATEG_LVL2_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "22",
+ "name" : "CATEG_LVL3_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "23",
+ "name" : "CATEG_LVL4_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "24",
+ "name" : "CATEG_LVL5_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "25",
+ "name" : "CATEG_LVL6_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "26",
+ "name" : "CATEG_LVL7_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "27",
+ "name" : "CATEG_FLAGS",
+ "datatype" : "decimal"
+ }, {
+ "id" : "28",
+ "name" : "ADULT_CATEG_YN",
+ "datatype" : "string"
+ }, {
+ "id" : "29",
+ "name" : "DOMAIN_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "30",
+ "name" : "USER_DEFINED_FIELD5",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "VCS_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "32",
+ "name" : "GCS_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "33",
+ "name" : "MOVE_TO",
+ "datatype" : "decimal"
+ }, {
+ "id" : "34",
+ "name" : "SAP_CATEGORY_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "35",
+ "name" : "SRC_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "36",
+ "name" : "BSNS_VRTCL_NAME",
+ "datatype" : "string"
+ } ],
+ "database" : "DEFAULT",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d32ecb0/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
new file mode 100644
index 0000000..4927343
--- /dev/null
+++ b/examples/sample_cube/template/table/DEFAULT.KYLIN_SALES.json
@@ -0,0 +1,44 @@
+{
+ "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
+
+ "name" : "KYLIN_SALES",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "TRANS_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "PART_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "LSTG_FORMAT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "4",
+ "name" : "LEAF_CATEG_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "5",
+ "name" : "LSTG_SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "6",
+ "name" : "SLR_SEGMENT_CD",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "PRICE",
+ "datatype" : "decimal(19,4)"
+ }, {
+ "id" : "8",
+ "name" : "ITEM_COUNT",
+ "datatype" : "bigint"
+ }, {
+ "id" : "9",
+ "name" : "SELLER_ID",
+ "datatype" : "bigint"
+ } ],
+ "database" : "DEFAULT",
+ "last_modified" : 0
+}
\ No newline at end of file
[26/50] [abbrv] kylin git commit: minor use 'unset' to clear sys
variable
Posted by li...@apache.org.
minor use 'unset' to clear sys variable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/42f28688
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/42f28688
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/42f28688
Branch: refs/heads/1.5.x-CDH5.7
Commit: 42f286884598aac3e8d8a63c4462f6ead74bd8c2
Parents: 6c23313
Author: shaofengshi <sh...@apache.org>
Authored: Tue Aug 30 20:43:32 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Aug 30 20:43:32 2016 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/42f28688/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 695b7f6..201b5b6 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -213,7 +213,7 @@ then
retrieveDependency
#retrive $KYLIN_EXTRA_START_OPTS from a separate file called setenv-tool.sh
- reset KYLIN_EXTRA_START_OPTS # reset the global server setenv config first
+ unset KYLIN_EXTRA_START_OPTS # unset the global server setenv config first
if [ -f "${dir}/setenv-tool.sh" ]
then source ${dir}/setenv-tool.sh
fi
[21/50] [abbrv] kylin git commit: KYLIN-1904-WBE-DICTIONARY-UPDATE
Posted by li...@apache.org.
KYLIN-1904-WBE-DICTIONARY-UPDATE
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f48f7faa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f48f7faa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f48f7faa
Branch: refs/heads/1.5.x-CDH5.7
Commit: f48f7faa0de5aa8bcee05e7b92e708670b5beb7c
Parents: 1f4675f
Author: zx chen <34...@qq.com>
Authored: Mon Aug 29 11:19:29 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Mon Aug 29 11:31:44 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/cubeDesigner/advanced_settings.html | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f48f7faa/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index eac6944..29d55a6 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -420,4 +420,8 @@
</ol>
</div>
</script>
-<script type="text/ng-template" id="AdvancedDictionariesTip.html"
+<script type="text/ng-template" id="AdvancedDictionariesTip.html">
+ <div>
+ <h4>Special settings for dictionaries. Leave blank by default.</h4>
+ </div>
+</script>
[32/50] [abbrv] kylin git commit: KYLIN-1849 search cube by name
Posted by li...@apache.org.
KYLIN-1849 search cube by name
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64005c67
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64005c67
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64005c67
Branch: refs/heads/1.5.x-CDH5.7
Commit: 64005c67a4395d90b7732a069848f6aca0290934
Parents: e7d2956
Author: Jason <ji...@163.com>
Authored: Wed Aug 31 14:37:19 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Aug 31 14:39:00 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubes.js | 4 ++--
webapp/app/partials/cubes/cubes.html | 20 ++++++++++----------
2 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/64005c67/webapp/app/js/controllers/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
index 0ea8601..3f125a6 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -119,13 +119,13 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
})
$scope.loading = false;
- $scope.listParams.cubeName = '';
+ //$scope.listParams.cubeName = '';
defer.resolve(resp);
return defer.promise;
},function(resp){
$scope.loading = false;
- $scope.listParams.cubeName = '';
+ //$scope.listParams.cubeName = '';
defer.resolve([]);
SweetAlert.swal('Oops...', resp, 'error');
return defer.promise;
http://git-wip-us.apache.org/repos/asf/kylin/blob/64005c67/webapp/app/partials/cubes/cubes.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html
index 3f978a2..30981dc 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -17,28 +17,28 @@
-->
<div class="row" ng-controller="CubesCtrl" style="margin-top:15px;">
-<div ng-if="!loading && cubeList.cubes.length == 0">
- <div no-result text="No Cube."></div>
-</div>
<loading ng-if="loading" text="Loading Cubes..."></loading>
<!--Table-->
-<div ng-if="cubeList.cubes.length > 0" class="dataTables_wrapper no-footer">
+<div class="dataTables_wrapper no-footer">
<div class="row">
- <div class="col-xs-10">
+ <div class="col-xs-9">
<label class="table-header-text">Cubes</label>
</div>
- <div class="col-xs-2">
- <form class="" ng-submit="cubeList.removeAll();list(0,3)" style="display: inline" >
+ <div class="col-xs-3">
+ <form style="display: inline;float:right;" >
<span class="input-icon input-icon-right nav-search">
<input type="text" placeholder="Search by name" class="nav-search-input" ng-model="listParams.cubeName" />
- <i class="ace-icon fa fa-search blue" ng-click="cubeList.removeAll();list(0,3)"></i>
+ <i class="ace-icon fa fa-search blue" ng-click="cubeList.removeAll();reload()"></i>
</span>
</form>
</div>
</div>
-
- <table class="table table-striped table-bordered table-hover dataTable no-footer">
+
+ <div ng-if="!loading && cubeList.cubes.length == 0">
+ <div no-result text="No Cube."></div>
+ </div>
+ <table ng-if="cubeList.cubes.length > 0" class="table table-striped table-bordered table-hover dataTable no-footer">
<!--Header-->
<thead>
<tr style="cursor: pointer">
[48/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index fe65598..ae26508 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -25,7 +25,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -78,7 +79,8 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin admin = conn.getAdmin();
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -91,7 +93,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(tableName)) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -100,7 +102,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
admin.close();
@@ -109,8 +111,7 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -125,8 +126,7 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 7aecd7e..9dc9715 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -28,9 +28,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -99,19 +100,21 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index 4fe7748..65cf205 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -38,8 +38,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -64,7 +64,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final HTableInterface hTable;
+ private final Table hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -73,7 +73,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -132,7 +132,6 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
- hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
index 9adaf24..e1e2cba 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseStreamingOutput.java
@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.CubeSegment;
@@ -56,7 +57,7 @@ public class HBaseStreamingOutput implements IStreamingOutput {
try {
CubeSegment cubeSegment = (CubeSegment) buildable;
- final HTableInterface hTable;
+ final Table hTable;
hTable = createHTable(cubeSegment);
List<ICuboidWriter> cuboidWriters = Lists.newArrayList();
cuboidWriters.add(new HBaseCuboidWriter(cubeSegment, hTable));
@@ -88,10 +89,10 @@ public class HBaseStreamingOutput implements IStreamingOutput {
}
}
- private HTableInterface createHTable(final CubeSegment cubeSegment) throws IOException {
+ private Table createHTable(final CubeSegment cubeSegment) throws IOException {
final String hTableName = cubeSegment.getStorageLocationIdentifier();
CubeHTableUtil.createHTable(cubeSegment, null);
- final HTableInterface hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(hTableName);
+ final Table hTable = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getTable(TableName.valueOf(hTableName));
logger.info("hTable:" + hTableName + " for segment:" + cubeSegment.getName() + " created!");
return hTable;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
+ hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 3066fb5..45c6b91 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -86,7 +87,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -130,8 +131,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -229,6 +230,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -322,8 +324,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -445,11 +447,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -469,7 +471,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -500,8 +501,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -535,13 +536,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -558,7 +558,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 295750a..efda4e4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -60,7 +60,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -128,9 +128,8 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
-
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -187,10 +186,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index a1193e7..0aff276 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -40,7 +40,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -76,7 +77,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -159,10 +161,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -184,9 +186,9 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeObserverClass, hdfsCoprocessorJar, 1002, null);
}
- public static void resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static void resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
logger.info("Unset coprocessor on " + tableName);
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -214,13 +216,13 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
}
- private static List<String> resetCoprocessorOnHTables(HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static List<String> resetCoprocessorOnHTables(Admin hbaseAdmin, Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processed = new ArrayList<String>();
for (String tableName : tableNames) {
@@ -331,7 +333,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index a5a85fa..29ca7b2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -26,10 +26,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -231,9 +232,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -253,7 +254,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 86ba22f..dd5f8fa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ Connection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void prepareData(Connection conn) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 6749d6c..940d64a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,9 +24,11 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -55,8 +57,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -71,12 +73,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 346c3a2..3ae411b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -30,12 +31,15 @@ import java.util.TreeSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -57,30 +61,31 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ Table table = null;
+ Admin admin = null;
try {
+ table = hbaseConnection.getTable(TableName.valueOf(tableName));
+ admin = hbaseConnection.getAdmin();
+
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+ logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
+ List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
+ for (HRegionLocation hRegionLocation : regionLocationList) {
+ tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -104,7 +109,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- hBaseAdmin.close();
+ admin.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index 266f7e7..a2f60d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,9 +23,10 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -42,8 +43,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index f30f2c9..542df39 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -31,15 +31,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -57,11 +57,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- HConnection conn = getConnection();
- HBaseAdmin hadmin = new HBaseAdmin(conn);
+ Connection conn = getConnection();
+ Admin hadmin = conn.getAdmin();
try {
- boolean tableExist = hadmin.tableExists(tableName);
+ boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -118,8 +118,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
byte[] key = new byte[8 + 4];//time + id
@@ -134,7 +134,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -169,8 +169,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -205,7 +205,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(HTableInterface table) throws IOException {
+ private static long getFirstKeyTime(Table table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -223,8 +223,8 @@ public class HbaseStreamingInput {
}
- private static HConnection getConnection() throws IOException {
- return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
+ private static Connection getConnection() throws IOException {
+ return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ca1a060..ea05ab2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -50,8 +51,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index 8ff5b0f..df4e912 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,10 +30,14 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,9 +56,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
-
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -73,12 +77,13 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ TableName tableName = TableName.valueOf(htableName);
+ if (hbaseAdmin.tableExists(tableName)) {
+ if (hbaseAdmin.isTableEnabled(tableName)) {
+ hbaseAdmin.disableTable(tableName);
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(tableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index 58ef7cb..63b5a42 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -21,12 +21,13 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -57,12 +58,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- HConnection conn = null;
- HTableInterface table = null;
+ Connection conn = null;
+ Table table = null;
ResultScanner scanner = null;
try {
- conn = HConnectionManager.createConnection(hconf);
- table = conn.getTable(hbaseTable);
+ conn = ConnectionFactory.createConnection(hconf);
+ table = conn.getTable(TableName.valueOf(hbaseTable));
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index 01edb1f..db516bb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,11 +22,12 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -70,8 +71,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
+ Connection conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 4bd2c53..1681050 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -54,6 +56,7 @@ import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,7 +75,8 @@ public class StorageCleanupJob extends AbstractApplication {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
long TIME_THREADSHOLD = KylinConfig.getInstanceFromEnv().getStorageCleanupTimeThreshold();
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -150,22 +154,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- HBaseAdmin hbaseAdmin;
+ Admin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index e36f662..42a54c8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,16 +24,18 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,14 +51,15 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ hbaseAdmin = conn.getAdmin();
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -166,9 +169,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
index f8e2644..fe06a31 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserverTest.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.ColumnDesc;
@@ -230,15 +231,8 @@ public class AggregateRegionObserverTest {
return nextRaw(results);
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.InternalScanner#next(java.util
- * .List, int)
- */
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -307,6 +301,11 @@ public class AggregateRegionObserverTest {
return 0;
}
+ @Override
+ public int getBatch() {
+ return 0;
+ }
+
/*
* (non-Javadoc)
*
@@ -323,16 +322,9 @@ public class AggregateRegionObserverTest {
return i < input.size();
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.hadoop.hbase.regionserver.RegionScanner#nextRaw(java.util
- * .List, int)
- */
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return nextRaw(result);
+ public boolean nextRaw(List<Cell> list, ScannerContext scannerContext) throws IOException {
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
index 1d85922..04e2e8b 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/filter/TestFuzzyRowFilterV2EndToEnd.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -136,7 +137,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
Put p = new Put(rk);
p.setDurability(Durability.SKIP_WAL);
- p.add(cf.getBytes(), cq, Bytes.toBytes(c));
+ p.addColumn(cf.getBytes(), cq, Bytes.toBytes(c));
ht.put(p);
}
}
@@ -224,7 +225,7 @@ public class TestFuzzyRowFilterV2EndToEnd {
scan.addFamily(cf.getBytes());
scan.setFilter(filter);
List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes());
- HRegion first = regions.get(0);
+ Region first = regions.get(0);
first.getScanner(scan);
RegionScanner scanner = first.getScanner(scan);
List<Cell> results = new ArrayList<Cell>();
[16/50] [abbrv] kylin git commit: KYLIN-1965 Check duplicated measure
name
Posted by li...@apache.org.
KYLIN-1965 Check duplicated measure name
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/46be8055
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/46be8055
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/46be8055
Branch: refs/heads/1.5.x-CDH5.7
Commit: 46be80550e4cbd3403756d615d170dc244c97976
Parents: 1d04642
Author: kangkaisen <ka...@live.com>
Authored: Thu Aug 18 19:17:37 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Aug 28 14:19:07 2016 +0800
----------------------------------------------------------------------
.../model/validation/rule/FunctionRule.java | 20 ++++++
.../model/validation/rule/FunctionRuleTest.java | 68 ++++++++++++++++++++
webapp/app/js/controllers/cubeMeasures.js | 16 +++++
3 files changed, 104 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/46be8055/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
index 44b0be0..792f18d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
@@ -62,6 +62,10 @@ public class FunctionRule implements IValidatorRule<CubeDesc> {
public void validate(CubeDesc cube, ValidateContext context) {
List<MeasureDesc> measures = cube.getMeasures();
+ if (validateMeasureNamesDuplicated(measures, context)) {
+ return;
+ }
+
List<FunctionDesc> countFuncs = new ArrayList<FunctionDesc>();
Iterator<MeasureDesc> it = measures.iterator();
@@ -185,4 +189,20 @@ public class FunctionRule implements IValidatorRule<CubeDesc> {
}
}
+
+ /**
+ * @param measures
+ */
+ private boolean validateMeasureNamesDuplicated(List<MeasureDesc> measures, ValidateContext context) {
+ Set<String> nameSet = new HashSet<>();
+ for (MeasureDesc measure: measures){
+ if (nameSet.contains(measure.getName())){
+ context.addResult(ResultLevel.ERROR, "There is duplicated measure's name: " + measure.getName());
+ return true;
+ } else {
+ nameSet.add(measure.getName());
+ }
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/46be8055/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
new file mode 100644
index 0000000..48e01e3
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/FunctionRuleTest.java
@@ -0,0 +1,68 @@
+package org.apache.kylin.cube.model.validation.rule;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class FunctionRuleTest extends LocalFileMetadataTestCase {
+ private static KylinConfig config;
+ private static MetadataManager metadataManager;
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ config = KylinConfig.getInstanceFromEnv();
+ metadataManager = MetadataManager.getInstance(config);
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testGoodDesc() throws IOException {
+ FunctionRule rule = new FunctionRule();
+
+ File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/ssb.json");
+ CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
+ desc.init(config, metadataManager.getAllTablesMap());
+ ValidateContext vContext = new ValidateContext();
+ rule.validate(desc, vContext);
+ vContext.print(System.out);
+ assertTrue(vContext.getResults().length == 0);
+ }
+
+ @Test
+ public void testValidateMeasureNamesDuplicated() throws IOException {
+ FunctionRule rule = new FunctionRule();
+
+ File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/ssb.json");
+ CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
+
+ MeasureDesc measureDescDuplicated = desc.getMeasures().get(1);
+ desc.getMeasures().add(measureDescDuplicated);
+
+ desc.init(config, metadataManager.getAllTablesMap());
+ ValidateContext vContext = new ValidateContext();
+ rule.validate(desc, vContext);
+
+ vContext.print(System.out);
+ assertTrue(vContext.getResults().length >= 1);
+ assertEquals("There is duplicated measure's name: " + measureDescDuplicated.getName(), vContext.getResults()[0].getMessage());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/46be8055/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 8db1356..9dec379 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -107,6 +107,12 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
SweetAlert.swal('', '[TOP_N] Group by Column is required', 'warning');
return false;
}
+
+ if ($scope.isNameDuplicated($scope.cubeMetaFrame.measures, $scope.newMeasure) == true) {
+ SweetAlert.swal('', 'The measure name: ' + $scope.newMeasure.name + ' is duplicated', 'warning');
+ return false;
+ }
+
if($scope.nextPara.value!=="" && ($scope.newMeasure.function.expression == 'EXTENDED_COLUMN' || $scope.newMeasure.function.expression == 'TOP_N')){
$scope.newMeasure.function.parameter.next_parameter = jQuery.extend(true,{},$scope.nextPara);
}
@@ -117,12 +123,22 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
else {
$scope.cubeMetaFrame.measures.push($scope.newMeasure);
}
+
$scope.newMeasure = null;
$scope.initUpdateMeasureStatus();
$scope.nextParameterInit();
return true;
};
+ $scope.isNameDuplicated = function (measures, newMeasure) {
+ var names = [];
+ for(var i = 0;i < measures.length; i++){
+ names.push(measures[i].name);
+ }
+ var index = names.indexOf(newMeasure.name);
+ return index > -1;
+ }
+
$scope.nextParameterInit = function(){
$scope.nextPara = {
"type":"column",
[08/50] [abbrv] kylin git commit: minor, retrieve dependency on demand
Posted by li...@apache.org.
minor, retrieve dependency on demand
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/579dfcff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/579dfcff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/579dfcff
Branch: refs/heads/1.5.x-CDH5.7
Commit: 579dfcff33997dc7d74ab33d0913f404413b5dea
Parents: 5a77f5e
Author: lidongsjtu <li...@apache.org>
Authored: Wed Aug 24 16:49:32 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Aug 24 17:09:32 2016 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 29 +++++++++++++++++------------
1 file changed, 17 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/579dfcff/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 365e1fe..695b7f6 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -27,18 +27,19 @@ source ${dir}/check-env.sh
mkdir -p ${KYLIN_HOME}/logs
mkdir -p ${KYLIN_HOME}/ext
-#retrive $hive_dependency and $hbase_dependency
-source ${dir}/find-hive-dependency.sh
-source ${dir}/find-hbase-dependency.sh
+function retrieveDependency() {
+ #retrive $hive_dependency and $hbase_dependency
+ source ${dir}/find-hive-dependency.sh
+ source ${dir}/find-hbase-dependency.sh
+
+ #retrive $KYLIN_EXTRA_START_OPTS
+ if [ -f "${dir}/setenv.sh" ]
+ then source ${dir}/setenv.sh
+ fi
-#retrive $KYLIN_EXTRA_START_OPTS
-if [ -f "${dir}/setenv.sh" ]
- then source ${dir}/setenv.sh
-fi
-
-export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
-export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
-
+ export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+ export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
+}
# start command
if [ "$1" == "start" ]
@@ -75,6 +76,8 @@ then
echo "kylin.security.profile is set to $spring_profile"
fi
+ retrieveDependency
+
#additionally add tomcat libs to HBASE_CLASSPATH_PREFIX
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:${HBASE_CLASSPATH_PREFIX}
@@ -143,7 +146,7 @@ then
fi
if [ "$2" == "start" ]
then
-
+ retrieveDependency
source ${dir}/find-kafka-dependency.sh
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
@@ -182,6 +185,7 @@ elif [ "$1" == "monitor" ]
then
echo "monitor job"
+ retrieveDependency
source ${dir}/find-kafka-dependency.sh
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
@@ -206,6 +210,7 @@ then
# tool command
elif [[ "$1" = org.apache.kylin.* ]]
then
+ retrieveDependency
#retrive $KYLIN_EXTRA_START_OPTS from a separate file called setenv-tool.sh
reset KYLIN_EXTRA_START_OPTS # reset the global server setenv config first
[42/50] [abbrv] kylin git commit: minor,
remove EvaluatableLikeFunctionTransformer
Posted by li...@apache.org.
minor, remove EvaluatableLikeFunctionTransformer
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/16de8ba2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/16de8ba2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/16de8ba2
Branch: refs/heads/1.5.x-CDH5.7
Commit: 16de8ba2701881b08ab15f4c104726bcbd6fe596
Parents: cf10263
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Sep 1 23:38:06 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Sep 1 23:38:15 2016 +0800
----------------------------------------------------------------------
.../EvaluatableLikeFunctionTransformer.java | 76 --------------------
1 file changed, 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/16de8ba2/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
deleted file mode 100644
index 7295533..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.metadata.filter;
-
-import java.util.ListIterator;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class EvaluatableLikeFunctionTransformer {
- public static final Logger logger = LoggerFactory.getLogger(EvaluatableLikeFunctionTransformer.class);
-
- public static TupleFilter transform(TupleFilter tupleFilter) {
- TupleFilter translated = null;
- if (tupleFilter instanceof CompareTupleFilter) {
- CompareTupleFilter compTupleFilter = (CompareTupleFilter) tupleFilter;
- if (compTupleFilter.getFunction() != null && (compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter)) {
- throw new IllegalArgumentException("BuiltInFunctionTupleFilter not supported :" + ((BuiltInFunctionTupleFilter) compTupleFilter.getFunction()).getName());
- }
- } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
- BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) tupleFilter;
- if (isLikeFunction(builtInFunctionTupleFilter)) {
- for (TupleFilter child : builtInFunctionTupleFilter.getChildren()) {
- if (!(child instanceof ColumnTupleFilter) && !(child instanceof ConstantTupleFilter)) {
- throw new IllegalArgumentException("Only simple like clause is supported");
- }
- }
-
- translated = new EvaluatableLikeFunction(builtInFunctionTupleFilter.getName());
- for (TupleFilter child : builtInFunctionTupleFilter.getChildren()) {
- translated.addChild(child);
- }
-
- } else {
- throw new IllegalArgumentException("BuiltInFunctionTupleFilter not supported: " + builtInFunctionTupleFilter.getName());
- }
-
- } else if (tupleFilter instanceof LogicalTupleFilter) {
- @SuppressWarnings("unchecked")
- ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
- while (childIterator.hasNext()) {
- TupleFilter transformed = transform(childIterator.next());
- if (transformed != null) {
- childIterator.set(transformed);
- } else {
- throw new IllegalStateException("Should not be null");
- }
- }
- }
- return translated == null ? tupleFilter : translated;
- }
-
- private static boolean isLikeFunction(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) {
- return "like".equalsIgnoreCase(builtInFunctionTupleFilter.getName());
- }
-
-}
\ No newline at end of file
[17/50] [abbrv] kylin git commit: KYLIN-1979 Move
hackNoGroupByAggregation to cube-based storage implementations
Posted by li...@apache.org.
KYLIN-1979 Move hackNoGroupByAggregation to cube-based storage implementations
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae9d7479
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae9d7479
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae9d7479
Branch: refs/heads/1.5.x-CDH5.7
Commit: ae9d747904d02f89a4d6616ca201243afbc075cc
Parents: 46be805
Author: Hongbin Ma <ma...@apache.org>
Authored: Sun Aug 28 23:19:06 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sun Aug 28 23:19:06 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 2 +-
.../apache/kylin/cube/RawQueryLastHacker.java | 82 ++++++++++++++++++++
.../org/apache/kylin/measure/MeasureType.java | 2 +-
.../dim/DimCountDistinctMeasureType.java | 10 ++-
.../ExtendedColumnMeasureType.java | 26 ++++---
.../kylin/measure/raw/RawMeasureType.java | 47 +++++------
.../kylin/measure/topn/TopNMeasureType.java | 38 ++++-----
.../kylin/metadata/realization/SQLDigest.java | 5 +-
.../gtrecord/GTCubeStorageQueryBase.java | 23 +++++-
.../kylin/query/enumerator/OLAPEnumerator.java | 63 ---------------
.../storage/hbase/cube/v1/CubeStorageQuery.java | 23 +++++-
11 files changed, 195 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index e0d8dd3..79d1e3b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -88,7 +88,7 @@ public class CubeCapabilityChecker {
return result;
}
- if (digest.isRawQuery() && cube.getFactTable().equals(digest.factTable)) {
+ if (digest.isRawQuery && cube.getFactTable().equals(digest.factTable)) {
result.influences.add(new CapabilityInfluence() {
@Override
public double suggestCostMultiplier() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
new file mode 100644
index 0000000..63ddac5
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/RawQueryLastHacker.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube;
+
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RawQueryLastHacker {
+
+ private static final Logger logger = LoggerFactory.getLogger(RawQueryLastHacker.class);
+
+ public static void hackNoAggregations(SQLDigest sqlDigest, CubeDesc cubeDesc) {
+ if (!sqlDigest.isRawQuery) {
+ return;
+ }
+
+ // If no group by and metric found, then it's simple query like select ... from ... where ...,
+ // But we have no raw data stored, in order to return better results, we hack to output sum of metric column
+ logger.info("No group by and aggregation found in this query, will hack some result for better look of output...");
+
+ // If it's select * from ...,
+ // We need to retrieve cube to manually add columns into sqlDigest, so that we have full-columns results as output.
+ boolean isSelectAll = sqlDigest.allColumns.isEmpty() || sqlDigest.allColumns.equals(sqlDigest.filterColumns);
+ for (TblColRef col : cubeDesc.listAllColumns()) {
+ if (col.getTable().equals(sqlDigest.factTable) && (cubeDesc.listDimensionColumnsIncludingDerived().contains(col) || isSelectAll)) {
+ sqlDigest.allColumns.add(col);
+ }
+ }
+
+ for (TblColRef col : sqlDigest.allColumns) {
+ if (cubeDesc.listDimensionColumnsIncludingDerived().contains(col)) {
+ // For dimension columns, take them as group by columns.
+ sqlDigest.groupbyColumns.add(col);
+ } else {
+ // For measure columns, take them as metric columns with aggregation function SUM().
+ ParameterDesc colParameter = new ParameterDesc();
+ colParameter.setType("column");
+ colParameter.setValue(col.getName());
+ FunctionDesc sumFunc = new FunctionDesc();
+ sumFunc.setExpression("SUM");
+ sumFunc.setParameter(colParameter);
+
+ boolean measureHasSum = false;
+ for (MeasureDesc colMeasureDesc : cubeDesc.getMeasures()) {
+ if (colMeasureDesc.getFunction().equals(sumFunc)) {
+ measureHasSum = true;
+ break;
+ }
+ }
+ if (measureHasSum) {
+ sqlDigest.aggregations.add(sumFunc);
+ } else {
+ logger.warn("SUM is not defined for measure column " + col + ", output will be meaningless.");
+ }
+
+ sqlDigest.metricColumns.add(col);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 6786f4b..82618e9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -124,7 +124,7 @@ abstract public class MeasureType<T> {
* They need to adjust dimensions and measures in <code>sqlDigest</code> before scanning,
* such that correct cuboid and measures can be selected by storage.
*/
- public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
}
/** Return true if one storage record maps to multiple tuples, or false otherwise. */
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
index 7e4e529..9fe1075 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/dim/DimCountDistinctMeasureType.java
@@ -18,6 +18,8 @@
package org.apache.kylin.measure.dim;
+import java.util.List;
+
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
@@ -79,8 +81,10 @@ public class DimCountDistinctMeasureType extends MeasureType<Object> {
return DimCountDistinctAggFunc.class;
}
- public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
- sqlDigest.groupbyColumns.addAll(measureDesc.getFunction().getParameter().getColRefs());
- sqlDigest.aggregations.remove(measureDesc.getFunction());
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
+ for (MeasureDesc measureDesc : measureDescs) {
+ sqlDigest.groupbyColumns.addAll(measureDesc.getFunction().getParameter().getColRefs());
+ sqlDigest.aggregations.remove(measureDesc.getFunction());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
index dc718d1..796f1f7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/extendedcolumn/ExtendedColumnMeasureType.java
@@ -93,19 +93,21 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> {
}
@Override
- public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
- FunctionDesc extendColumnFunc = measureDesc.getFunction();
- List<TblColRef> hosts = getExtendedColumnHosts(extendColumnFunc);
- TblColRef extended = getExtendedColumn(extendColumnFunc);
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
+ for (MeasureDesc measureDesc : measureDescs) {
+ FunctionDesc extendColumnFunc = measureDesc.getFunction();
+ List<TblColRef> hosts = getExtendedColumnHosts(extendColumnFunc);
+ TblColRef extended = getExtendedColumn(extendColumnFunc);
+
+ if (!sqlDigest.groupbyColumns.contains(extended)) {
+ return;
+ }
- if (!sqlDigest.groupbyColumns.contains(extended)) {
- return;
+ sqlDigest.aggregations.add(extendColumnFunc);
+ sqlDigest.groupbyColumns.remove(extended);
+ sqlDigest.groupbyColumns.addAll(hosts);
+ sqlDigest.metricColumns.add(extended);
}
-
- sqlDigest.aggregations.add(extendColumnFunc);
- sqlDigest.groupbyColumns.remove(extended);
- sqlDigest.groupbyColumns.addAll(hosts);
- sqlDigest.metricColumns.add(extended);
}
@Override
@@ -151,7 +153,7 @@ public class ExtendedColumnMeasureType extends MeasureType<ByteArray> {
value = null;
return;
}
-
+
ByteArray byteArray = (ByteArray) measureValue;
//the array in ByteArray is guaranteed to be completed owned by the ByteArray
value = Bytes.toString(byteArray.array());
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index aa46149..50715ec 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -166,7 +166,7 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, MeasureDesc measureDesc) {
//is raw query
- if (!digest.isRawQuery())
+ if (!digest.isRawQuery)
return null;
TblColRef rawColumn = getRawColumn(measureDesc.getFunction());
@@ -196,27 +196,30 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
}
@Override
- public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
- if (sqlDigest.isRawQuery()) {
- TblColRef col = this.getRawColumn(measureDesc.getFunction());
- ParameterDesc colParameter = new ParameterDesc();
- colParameter.setType("column");
- colParameter.setValue(col.getName());
- FunctionDesc rawFunc = new FunctionDesc();
- rawFunc.setExpression("RAW");
- rawFunc.setParameter(colParameter);
-
- if (sqlDigest.allColumns.contains(col)) {
- if (measureDesc.getFunction().equals(rawFunc)) {
- FunctionDesc sumFunc = new FunctionDesc();
- sumFunc.setExpression("SUM");
- sumFunc.setParameter(colParameter);
- sqlDigest.aggregations.remove(sumFunc);
- sqlDigest.aggregations.add(rawFunc);
- logger.info("Add RAW measure on column " + col);
- }
- if (!sqlDigest.metricColumns.contains(col)) {
- sqlDigest.metricColumns.add(col);
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
+
+ if (sqlDigest.isRawQuery) {
+ for (MeasureDesc measureDesc : measureDescs) {
+ TblColRef col = this.getRawColumn(measureDesc.getFunction());
+ ParameterDesc colParameter = new ParameterDesc();
+ colParameter.setType("column");
+ colParameter.setValue(col.getName());
+ FunctionDesc rawFunc = new FunctionDesc();
+ rawFunc.setExpression("RAW");
+ rawFunc.setParameter(colParameter);
+
+ if (sqlDigest.allColumns.contains(col)) {
+ if (measureDesc.getFunction().equals(rawFunc)) {
+ FunctionDesc sumFunc = new FunctionDesc();
+ sumFunc.setExpression("SUM");
+ sumFunc.setParameter(colParameter);
+ sqlDigest.aggregations.remove(sumFunc);
+ sqlDigest.aggregations.add(rawFunc);
+ logger.info("Add RAW measure on column " + col);
+ }
+ if (!sqlDigest.metricColumns.contains(col)) {
+ sqlDigest.metricColumns.add(col);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index ad42824..ed22d61 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -39,8 +39,8 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.tuple.Tuple;
import org.apache.kylin.metadata.tuple.TupleInfo;
import org.slf4j.Logger;
@@ -290,28 +290,30 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
}
@Override
- public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
- FunctionDesc topnFunc = measureDesc.getFunction();
- List<TblColRef> topnLiteralCol = getTopNLiteralColumn(topnFunc);
+ public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
+ for (MeasureDesc measureDesc : measureDescs) {
+ FunctionDesc topnFunc = measureDesc.getFunction();
+ List<TblColRef> topnLiteralCol = getTopNLiteralColumn(topnFunc);
- if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false)
- return;
+ if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false)
+ return;
- if (sqlDigest.aggregations.size() > 1) {
- throw new IllegalStateException("When query with topN, only one metrics is allowed.");
- }
+ if (sqlDigest.aggregations.size() > 1) {
+ throw new IllegalStateException("When query with topN, only one metrics is allowed.");
+ }
- if (sqlDigest.aggregations.size() > 0) {
- FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
- if (origFunc.isSum() == false && origFunc.isCount() == false) {
- throw new IllegalStateException("When query with topN, only SUM function is allowed.");
+ if (sqlDigest.aggregations.size() > 0) {
+ FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
+ if (origFunc.isSum() == false && origFunc.isCount() == false) {
+ throw new IllegalStateException("When query with topN, only SUM function is allowed.");
+ }
+ logger.info("Rewrite function " + origFunc + " to " + topnFunc);
}
- logger.info("Rewrite function " + origFunc + " to " + topnFunc);
- }
- sqlDigest.aggregations = Lists.newArrayList(topnFunc);
- sqlDigest.groupbyColumns.removeAll(topnLiteralCol);
- sqlDigest.metricColumns.addAll(topnLiteralCol);
+ sqlDigest.aggregations = Lists.newArrayList(topnFunc);
+ sqlDigest.groupbyColumns.removeAll(topnLiteralCol);
+ sqlDigest.metricColumns.addAll(topnLiteralCol);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 1eee1e7..d2bba66 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -44,7 +44,9 @@ public class SQLDigest {
public Collection<FunctionDesc> aggregations;
public Collection<MeasureDesc> sortMeasures;
public Collection<OrderEnum> sortOrders;
+ public boolean isRawQuery;
+ //initialized when org.apache.kylin.query.routing.QueryRouter.selectRealization()
public SQLDigest(String factTable, TupleFilter filter, Collection<JoinDesc> joinDescs, Collection<TblColRef> allColumns, //
Collection<TblColRef> groupbyColumns, Collection<TblColRef> filterColumns, Collection<TblColRef> aggregatedColumns, Collection<FunctionDesc> aggregateFunnc, Collection<MeasureDesc> sortMeasures, Collection<OrderEnum> sortOrders) {
this.factTable = factTable;
@@ -57,9 +59,10 @@ public class SQLDigest {
this.aggregations = aggregateFunnc;
this.sortMeasures = sortMeasures;
this.sortOrders = sortOrders;
+ this.isRawQuery = isRawQuery();
}
- public boolean isRawQuery() {
+ private boolean isRawQuery() {
return this.groupbyColumns.isEmpty() && // select a group by a -> not raw
this.aggregations.isEmpty(); // has aggr -> not raw
//the reason to choose aggregations rather than metricColumns is because the former is set earlier at implOLAP
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index bacd293..86346f8 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -22,12 +22,14 @@ import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.RawQueryLastHacker;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
@@ -53,6 +55,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
public abstract class GTCubeStorageQueryBase implements IStorageQuery {
@@ -69,7 +72,11 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
- // allow custom measures hack
+
+ //cope with queries with no aggregations
+ RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
+
+ // Customized measure taking effect: e.g. allow custom measures to help raw queries
notifyBeforeStorageQuery(sqlDigest);
Collection<TblColRef> groups = sqlDigest.groupbyColumns;
@@ -418,9 +425,21 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
+ Map<String, List<MeasureDesc>> map = Maps.newHashMap();
for (MeasureDesc measure : cubeDesc.getMeasures()) {
MeasureType<?> measureType = measure.getFunction().getMeasureType();
- measureType.adjustSqlDigest(measure, sqlDigest);
+
+ String key = measureType.getClass().getCanonicalName();
+ List<MeasureDesc> temp = null;
+ if ((temp = map.get(key)) != null) {
+ temp.add(measure);
+ } else {
+ map.put(key, Lists.<MeasureDesc> newArrayList(measure));
+ }
+ }
+
+ for (List<MeasureDesc> sublist : map.values()) {
+ sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index 96e75f3..dafbb5f 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -25,14 +25,8 @@ import org.apache.calcite.DataContext;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
@@ -114,7 +108,6 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
// cube don't have correct result for simple query without group by, but let's try to return something makes sense
olapContext.resetSQLDigest();
SQLDigest sqlDigest = olapContext.getSQLDigest();
- hackNoGroupByAggregation(sqlDigest);
// query storage engine
IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
@@ -161,60 +154,4 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
olapContext.storageContext.setThreshold(threshold);
}
- // Hack no-group-by query for better results
- private void hackNoGroupByAggregation(SQLDigest sqlDigest) {
- if (!(olapContext.realization instanceof CubeInstance)) {
- //the hack only makes sense for cubes
- return;
- }
-
- if (!sqlDigest.isRawQuery()) {
- return;
- }
-
- // If no group by and metric found, then it's simple query like select ... from ... where ...,
- // But we have no raw data stored, in order to return better results, we hack to output sum of metric column
- logger.info("No group by and aggregation found in this query, will hack some result for better look of output...");
-
- // If it's select * from ...,
- // We need to retrieve cube to manually add columns into sqlDigest, so that we have full-columns results as output.
- IRealization cube = olapContext.realization;
- boolean isSelectAll = sqlDigest.allColumns.isEmpty() || sqlDigest.allColumns.equals(sqlDigest.filterColumns);
- for (TblColRef col : cube.getAllColumns()) {
- if (col.getTable().equals(sqlDigest.factTable) && (cube.getAllDimensions().contains(col) || isSelectAll)) {
- sqlDigest.allColumns.add(col);
- }
- }
-
- for (TblColRef col : sqlDigest.allColumns) {
- if (cube.getAllDimensions().contains(col)) {
- // For dimension columns, take them as group by columns.
- sqlDigest.groupbyColumns.add(col);
- } else {
- // For measure columns, take them as metric columns with aggregation function SUM().
- ParameterDesc colParameter = new ParameterDesc();
- colParameter.setType("column");
- colParameter.setValue(col.getName());
- FunctionDesc sumFunc = new FunctionDesc();
- sumFunc.setExpression("SUM");
- sumFunc.setParameter(colParameter);
-
- boolean measureHasSum = false;
- for (MeasureDesc colMeasureDesc : cube.getMeasures()) {
- if (colMeasureDesc.getFunction().equals(sumFunc)) {
- measureHasSum = true;
- break;
- }
- }
- if (measureHasSum) {
- sqlDigest.aggregations.add(sumFunc);
- } else {
- logger.warn("SUM is not defined for measure column " + col + ", output will be meaningless.");
- }
-
- sqlDigest.metricColumns.add(col);
- }
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae9d7479/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 46f16fe..ff729f4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -42,13 +42,14 @@ import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.RawQueryLastHacker;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
import org.apache.kylin.cube.model.RowKeyDesc;
+import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
@@ -97,7 +98,10 @@ public class CubeStorageQuery implements IStorageQuery {
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
- // allow custom measures hack
+ //cope with queries with no aggregations
+ RawQueryLastHacker.hackNoAggregations(sqlDigest, cubeDesc);
+
+ // Customized measure taking effect: e.g. allow custom measures to help raw queries
notifyBeforeStorageQuery(sqlDigest);
Collection<TblColRef> groups = sqlDigest.groupbyColumns;
@@ -762,9 +766,22 @@ public class CubeStorageQuery implements IStorageQuery {
}
private void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
+
+ Map<String, List<MeasureDesc>> map = Maps.newHashMap();
for (MeasureDesc measure : cubeDesc.getMeasures()) {
MeasureType<?> measureType = measure.getFunction().getMeasureType();
- measureType.adjustSqlDigest(measure, sqlDigest);
+
+ String key = measureType.getClass().getCanonicalName();
+ List<MeasureDesc> temp = null;
+ if ((temp = map.get(key)) != null) {
+ temp.add(measure);
+ } else {
+ map.put(key, Lists.<MeasureDesc> newArrayList(measure));
+ }
+ }
+
+ for (List<MeasureDesc> sublist : map.values()) {
+ sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest);
}
}
[41/50] [abbrv] kylin git commit: KYLIN-1993 update sample metadata
files
Posted by li...@apache.org.
KYLIN-1993 update sample metadata files
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cf102634
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cf102634
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cf102634
Branch: refs/heads/1.5.x-CDH5.7
Commit: cf102634137e2e72f980e51b9c997b8215dede07
Parents: 7d32ecb
Author: lidongsjtu <li...@apache.org>
Authored: Thu Sep 1 22:47:28 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Sep 1 22:47:35 2016 +0800
----------------------------------------------------------------------
build/bin/sample.sh | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf102634/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 06e05d1..c915fb4 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -51,6 +51,10 @@ fi
export default_engine_type
export default_storage_type
+mkdir -p ${KYLIN_HOME}/sample_cube/metadata
+cp -rf ${KYLIN_HOME}/sample_cube/template/* ${KYLIN_HOME}/sample_cube/metadata
+envsubst < ${KYLIN_HOME}/sample_cube/template/cube_desc/kylin_sales_cube_desc.json > ${KYLIN_HOME}/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+
cd ${KYLIN_HOME}
hbase org.apache.hadoop.util.RunJar ${job_jar} org.apache.kylin.common.persistence.ResourceTool upload ${KYLIN_HOME}/sample_cube/metadata || { exit 1; }
echo "Sample cube is created successfully in project 'learn_kylin'; Restart Kylin server or reload the metadata from web UI to see the change."
[33/50] [abbrv] kylin git commit: KYLIN-1849 support search cube in
web
Posted by li...@apache.org.
KYLIN-1849 support search cube in web
Signed-off-by: Jason <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7d29566
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7d29566
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7d29566
Branch: refs/heads/1.5.x-CDH5.7
Commit: e7d295669e023a4e28ac8ec3c6177f468371feb2
Parents: 3cb0894
Author: kangkaisen <ka...@live.com>
Authored: Mon Aug 15 23:46:23 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Wed Aug 31 14:39:00 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubes.js | 3 ++-
webapp/app/partials/cubes/cubes.html | 15 ++++++++++++---
2 files changed, 14 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7d29566/webapp/app/js/controllers/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
index 19de285..0ea8601 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -35,7 +35,6 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
$scope.loading = false;
$scope.action = {};
-
$scope.state = {
filterAttr: 'create_time', filterReverse: true, reverseColumn: 'create_time',
dimensionFilter: '', measureFilter: ''
@@ -120,11 +119,13 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
})
$scope.loading = false;
+ $scope.listParams.cubeName = '';
defer.resolve(resp);
return defer.promise;
},function(resp){
$scope.loading = false;
+ $scope.listParams.cubeName = '';
defer.resolve([]);
SweetAlert.swal('Oops...', resp, 'error');
return defer.promise;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7d29566/webapp/app/partials/cubes/cubes.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html
index fceca4c..3f978a2 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -25,10 +25,19 @@
<!--Table-->
<div ng-if="cubeList.cubes.length > 0" class="dataTables_wrapper no-footer">
<div class="row">
- <div class="col-xs-7"><label class="table-header-text">Cubes</label></div>
- <!--button-->
-
+ <div class="col-xs-10">
+ <label class="table-header-text">Cubes</label>
+ </div>
+ <div class="col-xs-2">
+ <form class="" ng-submit="cubeList.removeAll();list(0,3)" style="display: inline" >
+ <span class="input-icon input-icon-right nav-search">
+ <input type="text" placeholder="Search by name" class="nav-search-input" ng-model="listParams.cubeName" />
+ <i class="ace-icon fa fa-search blue" ng-click="cubeList.removeAll();list(0,3)"></i>
+ </span>
+ </form>
+ </div>
</div>
+
<table class="table table-striped table-bordered table-hover dataTable no-footer">
<!--Header-->
<thead>
[12/50] [abbrv] kylin git commit: minor, code refactor
Posted by li...@apache.org.
minor, code refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5ee76e8b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5ee76e8b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5ee76e8b
Branch: refs/heads/1.5.x-CDH5.7
Commit: 5ee76e8b5840e7f8c6da9a56e6f1f496f0821589
Parents: 57a259a
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Aug 25 14:17:19 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Aug 26 10:39:19 2016 +0800
----------------------------------------------------------------------
.../kylin/cube/CubeCapabilityChecker.java | 50 +-
.../org/apache/kylin/cube/CubeInstance.java | 2 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 10 +-
.../java/org/apache/kylin/cube/ISegment.java | 39 ++
.../java/org/apache/kylin/cube/JoinChecker.java | 68 +++
.../gridtable/AsymmetricRecordComparator.java | 55 ++
.../kylin/cube/gridtable/ComparatorEx.java | 64 +++
.../kylin/cube/gridtable/CubeGridTable.java | 21 -
.../cube/gridtable/CubeScanRangePlanner.java | 542 +------------------
.../gridtable/CuboidToGridTableMapping.java | 60 +-
.../kylin/cube/gridtable/RecordComparator.java | 46 ++
.../kylin/cube/gridtable/RecordComparators.java | 77 +++
.../cube/gridtable/ScanRangePlannerBase.java | 295 ++++++++++
.../cube/gridtable/SegmentGTStartAndEnd.java | 84 +++
.../java/org/apache/kylin/gridtable/GTInfo.java | 16 +-
.../apache/kylin/gridtable/GTScanRequest.java | 16 +-
.../kylin/gridtable/GTScanRequestBuilder.java | 13 +
.../apache/kylin/gridtable/ScannerWorker.java | 8 +-
.../kylin/gridtable/DictGridTableTest.java | 2 +-
.../kylin/metadata/project/ProjectL2Cache.java | 8 +
.../kylin/metadata/realization/SQLDigest.java | 6 +-
.../storage/gtrecord/CubeTupleConverter.java | 42 +-
.../gtrecord/GTCubeStorageQueryBase.java | 2 +-
.../gtrecord/SequentialCubeTupleIterator.java | 4 +-
.../apache/kylin/query/ITKylinQueryTest.java | 4 +-
pom.xml | 6 +
.../kylin/query/enumerator/OLAPEnumerator.java | 9 +-
.../apache/kylin/query/routing/Candidate.java | 1 +
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 6 +-
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 7 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 6 +-
31 files changed, 919 insertions(+), 650 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 1f16b1b..e0d8dd3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -18,7 +18,6 @@
package org.apache.kylin.cube;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
@@ -26,18 +25,16 @@ import java.util.List;
import java.util.Set;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.basic.BasicMeasureType;
import org.apache.kylin.metadata.filter.UDF.MassInTupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IStorageAware;
-import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,7 +50,7 @@ public class CubeCapabilityChecker {
result.capable = false;
// match joins
- boolean isJoinMatch = isJoinMatch(digest.joinDescs, cube);
+ boolean isJoinMatch = JoinChecker.isJoinMatch(digest.joinDescs, cube);
if (!isJoinMatch) {
logger.info("Exclude cube " + cube.getName() + " because unmatched joins");
return result;
@@ -89,7 +86,15 @@ public class CubeCapabilityChecker {
if (cube.getStorageType() == IStorageAware.ID_HBASE && MassInTupleFilter.containsMassInTupleFilter(digest.filter)) {
logger.info("Exclude cube " + cube.getName() + " because only v2 storage + v2 query engine supports massin");
return result;
+ }
+ if (digest.isRawQuery() && cube.getFactTable().equals(digest.factTable)) {
+ result.influences.add(new CapabilityInfluence() {
+ @Override
+ public double suggestCostMultiplier() {
+ return 100;
+ }
+ });
}
// cost will be minded by caller
@@ -121,41 +126,6 @@ public class CubeCapabilityChecker {
return result;
}
- private static boolean isJoinMatch(Collection<JoinDesc> joins, CubeInstance cube) {
- CubeDesc cubeDesc = cube.getDescriptor();
-
- List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(cubeDesc.getDimensions().size());
- for (DimensionDesc d : cubeDesc.getDimensions()) {
- if (d.getJoin() != null) {
- cubeJoins.add(d.getJoin());
- }
- }
- for (JoinDesc j : joins) {
- // optiq engine can't decide which one is fk or pk
- String pTable = j.getPrimaryKeyColumns()[0].getTable();
- String factTable = cubeDesc.getFactTable();
- if (factTable.equals(pTable)) {
- j.swapPKFK();
- }
-
- // check primary key, all PK column should refer to same tale, the Fact Table of cube.
- // Using first column's table name to check.
- String fTable = j.getForeignKeyColumns()[0].getTable();
- if (!factTable.equals(fTable)) {
- logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + cube.getName());
- return false;
- }
-
- // The hashcode() function of JoinDesc has been overwritten,
- // which takes into consideration: pk,fk,jointype
- if (!cubeJoins.contains(j)) {
- logger.info("Query joins don't macth on cube " + cube.getName());
- return false;
- }
- }
- return true;
- }
-
private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) {
CubeDesc cubeDesc = cube.getDescriptor();
Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 90da59d..151e142 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -352,7 +352,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return result;
}
- private int getCost(SQLDigest digest) {
+ public int getCost(SQLDigest digest) {
int calculatedCost = cost;
//the number of dimensions is not as accurate as number of row key cols
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index aaa88f1..79397c3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -35,21 +35,22 @@ import org.apache.kylin.common.util.ShardingHash;
import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IBuildable;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeSegment implements Comparable<CubeSegment>, IBuildable {
+public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegment {
@JsonBackReference
private CubeInstance cubeInstance;
@@ -175,6 +176,11 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable {
return status;
}
+ @Override
+ public DataModelDesc getModel() {
+ return this.getCubeDesc().getModel();
+ }
+
public void setStatus(SegmentStatusEnum status) {
this.status = status;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
new file mode 100644
index 0000000..2e1f214
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/ISegment.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube;
+
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+
+public interface ISegment {
+
+ public String getName();
+
+ public long getDateRangeStart();
+
+ public long getDateRangeEnd();
+
+ public long getSourceOffsetStart();
+
+ public long getSourceOffsetEnd();
+
+ public DataModelDesc getModel();
+
+ public SegmentStatusEnum getStatus();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
new file mode 100644
index 0000000..edd5be9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/JoinChecker.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.LookupDesc;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+public class JoinChecker {
+
+ private static final Logger logger = LoggerFactory.getLogger(CubeCapabilityChecker.class);
+
+ public static boolean isJoinMatch(Collection<JoinDesc> joins, IRealization realization) {
+
+ List<JoinDesc> realizationsJoins = Lists.newArrayList();
+ for (LookupDesc lookupDesc : realization.getDataModelDesc().getLookups()) {
+ realizationsJoins.add(lookupDesc.getJoin());
+ }
+
+ for (JoinDesc j : joins) {
+ // optiq engine can't decide which one is fk or pk
+ String pTable = j.getPrimaryKeyColumns()[0].getTable();
+ String factTable = realization.getFactTable();
+ if (factTable.equals(pTable)) {
+ j.swapPKFK();
+ }
+
+ // check primary key, all PK column should refer to same tale, the Fact Table of cube.
+ // Using first column's table name to check.
+ String fTable = j.getForeignKeyColumns()[0].getTable();
+ if (!factTable.equals(fTable)) {
+ logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + realization.getName());
+ return false;
+ }
+
+ // The hashcode() function of JoinDesc has been overwritten,
+ // which takes into consideration: pk,fk,jointype
+ if (!realizationsJoins.contains(j)) {
+ logger.info("Query joins don't macth on cube " + realization.getName());
+ return false;
+ }
+ }
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java
new file mode 100644
index 0000000..74dc855
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/AsymmetricRecordComparator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.gridtable.GTRecord;
+
+import java.util.Collection;
+
+/**
+ * asymmetric means compare(a,b) > 0 does not cause compare(b,a) < 0
+ * so min max functions will not be supported
+ */
+public class AsymmetricRecordComparator extends RecordComparator {
+
+ AsymmetricRecordComparator(ComparatorEx<ByteArray> byteComparator) {
+ super(byteComparator);
+ }
+
+ public GTRecord min(Collection<GTRecord> v) {
+ throw new UnsupportedOperationException();
+ }
+
+ public GTRecord max(Collection<GTRecord> v) {
+ throw new UnsupportedOperationException();
+ }
+
+ public GTRecord min(GTRecord a, GTRecord b) {
+ throw new UnsupportedOperationException();
+ }
+
+ public GTRecord max(GTRecord a, GTRecord b) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean between(GTRecord v, GTRecord start, GTRecord end) {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java
new file mode 100644
index 0000000..3f1e8af
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ComparatorEx.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+
+public abstract class ComparatorEx<T> implements Comparator<T> {
+
+ public T min(Collection<T> v) {
+ if (v.size() <= 0) {
+ return null;
+ }
+
+ Iterator<T> iterator = v.iterator();
+ T min = iterator.next();
+ while (iterator.hasNext()) {
+ min = min(min, iterator.next());
+ }
+ return min;
+ }
+
+ public T max(Collection<T> v) {
+ if (v.size() <= 0) {
+ return null;
+ }
+
+ Iterator<T> iterator = v.iterator();
+ T max = iterator.next();
+ while (iterator.hasNext()) {
+ max = max(max, iterator.next());
+ }
+ return max;
+ }
+
+ public T min(T a, T b) {
+ return compare(a, b) <= 0 ? a : b;
+ }
+
+ public T max(T a, T b) {
+ return compare(a, b) >= 0 ? a : b;
+ }
+
+ public boolean between(T v, T start, T end) {
+ return compare(start, v) <= 0 && compare(v, end) <= 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
index f2f3a25..563cf43 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeGridTable.java
@@ -18,11 +18,9 @@
package org.apache.kylin.cube.gridtable;
-import java.util.List;
import java.util.Map;
import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.CubeDimEncMap;
@@ -31,27 +29,8 @@ import org.apache.kylin.dimension.IDimensionEncodingMap;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.metadata.model.TblColRef;
-import com.google.common.collect.Maps;
-
public class CubeGridTable {
- public static Map<TblColRef, Dictionary<String>> getDimensionToDictionaryMap(CubeSegment cubeSeg, long cuboidId) {
- CubeDesc cubeDesc = cubeSeg.getCubeDesc();
- CubeManager cubeMgr = CubeManager.getInstance(cubeSeg.getCubeInstance().getConfig());
-
- // build a dictionary map
- Map<TblColRef, Dictionary<String>> dictionaryMap = Maps.newHashMap();
- List<TblColRef> dimCols = Cuboid.findById(cubeDesc, cuboidId).getColumns();
- for (TblColRef col : dimCols) {
- Dictionary<String> dictionary = cubeMgr.getDictionary(cubeSeg, col);
- if (dictionary != null) {
- dictionaryMap.put(col, dictionary);
- }
- }
-
- return dictionaryMap;
- }
-
public static GTInfo newGTInfo(CubeSegment cubeSeg, long cuboidId) {
Cuboid cuboid = Cuboid.findById(cubeSeg.getCubeDesc(), cuboidId);
return newGTInfo(cuboid, new CubeDimEncMap(cubeSeg));
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
index 58771da..a937045 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CubeScanRangePlanner.java
@@ -18,25 +18,17 @@
package org.apache.kylin.cube.gridtable;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.common.FuzzyValueCombination;
@@ -49,12 +41,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanRequestBuilder;
import org.apache.kylin.gridtable.GTUtil;
import org.apache.kylin.gridtable.IGTComparator;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -64,7 +51,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-public class CubeScanRangePlanner {
+public class CubeScanRangePlanner extends ScanRangePlannerBase {
private static final Logger logger = LoggerFactory.getLogger(CubeScanRangePlanner.class);
@@ -75,24 +62,6 @@ public class CubeScanRangePlanner {
protected CubeSegment cubeSegment;
protected CubeDesc cubeDesc;
protected Cuboid cuboid;
- protected TupleFilter filter;
- protected Set<TblColRef> dimensions;
- protected Set<TblColRef> groupbyDims;
- protected Set<TblColRef> filterDims;
- protected Collection<FunctionDesc> metrics;
-
- //GT
- protected TupleFilter gtFilter;
- protected GTInfo gtInfo;
- protected Pair<ByteArray, ByteArray> gtStartAndEnd;
- protected TblColRef gtPartitionCol;
- protected ImmutableBitSet gtDimensions;
- protected ImmutableBitSet gtAggrGroups;
- protected ImmutableBitSet gtAggrMetrics;
- protected String[] gtAggrFuncs;
- final protected RecordComparator rangeStartComparator;
- final protected RecordComparator rangeEndComparator;
- final protected RecordComparator rangeStartEndComparator;
public CubeScanRangePlanner(CubeSegment cubeSegment, Cuboid cuboid, TupleFilter filter, Set<TblColRef> dimensions, Set<TblColRef> groupbyDims, //
Collection<FunctionDesc> metrics) {
@@ -103,36 +72,35 @@ public class CubeScanRangePlanner {
this.cubeSegment = cubeSegment;
this.cubeDesc = cubeSegment.getCubeDesc();
this.cuboid = cuboid;
- this.dimensions = dimensions;
- this.groupbyDims = groupbyDims;
- this.filter = filter;
- this.metrics = metrics;
- this.filterDims = Sets.newHashSet();
- TupleFilter.collectColumns(filter, this.filterDims);
+
+ Set<TblColRef> filterDims = Sets.newHashSet();
+ TupleFilter.collectColumns(filter, filterDims);
this.gtInfo = CubeGridTable.newGTInfo(cubeSegment, cuboid.getId());
CuboidToGridTableMapping mapping = cuboid.getCuboidToGridTableMapping();
IGTComparator comp = gtInfo.getCodeSystem().getComparator();
//start key GTRecord compare to start key GTRecord
- this.rangeStartComparator = getRangeStartComparator(comp);
+ this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
//stop key GTRecord compare to stop key GTRecord
- this.rangeEndComparator = getRangeEndComparator(comp);
+ this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
//start key GTRecord compare to stop key GTRecord
- this.rangeStartEndComparator = getRangeStartEndComparator(comp);
+ this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
//replace the constant values in filter to dictionary codes
- this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), this.groupbyDims);
+ this.gtFilter = GTUtil.convertFilterColumnsAndConstants(filter, gtInfo, mapping.getCuboidDimensionsInGTOrder(), groupbyDims);
- this.gtDimensions = makeGridTableColumns(mapping, dimensions);
- this.gtAggrGroups = makeGridTableColumns(mapping, replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc()));
- this.gtAggrMetrics = makeGridTableColumns(mapping, metrics);
- this.gtAggrFuncs = makeAggrFuncs(mapping, metrics);
+ this.gtDimensions = mapping.makeGridTableColumns(dimensions);
+ this.gtAggrGroups = mapping.makeGridTableColumns(replaceDerivedColumns(groupbyDims, cubeSegment.getCubeDesc()));
+ this.gtAggrMetrics = mapping.makeGridTableColumns(metrics);
+ this.gtAggrFuncs = mapping.makeAggrFuncs(metrics);
- if (cubeSegment.getCubeDesc().getModel().getPartitionDesc().isPartitioned()) {
- int index = mapping.getIndexOf(cubeSegment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef());
+ if (cubeSegment.getModel().getPartitionDesc().isPartitioned()) {
+ int index = mapping.getIndexOf(cubeSegment.getModel().getPartitionDesc().getPartitionDateColumnRef());
if (index >= 0) {
- this.gtStartAndEnd = getSegmentStartAndEnd(index);
+ SegmentGTStartAndEnd segmentGTStartAndEnd = new SegmentGTStartAndEnd(cubeSegment, gtInfo);
+ this.gtStartAndEnd = segmentGTStartAndEnd.getSegmentStartAndEnd(index);
+ this.isPartitionColUsingDatetimeEncoding = segmentGTStartAndEnd.isUsingDatetimeEncoding(index);
this.gtPartitionCol = gtInfo.colRef(index);
}
}
@@ -155,11 +123,11 @@ public class CubeScanRangePlanner {
IGTComparator comp = gtInfo.getCodeSystem().getComparator();
//start key GTRecord compare to start key GTRecord
- this.rangeStartComparator = getRangeStartComparator(comp);
+ this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp);
//stop key GTRecord compare to stop key GTRecord
- this.rangeEndComparator = getRangeEndComparator(comp);
+ this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp);
//start key GTRecord compare to stop key GTRecord
- this.rangeStartEndComparator = getRangeStartEndComparator(comp);
+ this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp);
this.gtFilter = gtFilter;
this.gtStartAndEnd = gtStartAndEnd;
@@ -199,46 +167,6 @@ public class CubeScanRangePlanner {
return mergedRanges;
}
- private Pair<ByteArray, ByteArray> getSegmentStartAndEnd(int index) {
- ByteArray start;
- if (cubeSegment.getDateRangeStart() != Long.MIN_VALUE) {
- start = encodeTime(cubeSegment.getDateRangeStart(), index, 1);
- } else {
- start = new ByteArray();
- }
-
- ByteArray end;
- if (cubeSegment.getDateRangeEnd() != Long.MAX_VALUE) {
- end = encodeTime(cubeSegment.getDateRangeEnd(), index, -1);
- } else {
- end = new ByteArray();
- }
- return Pair.newPair(start, end);
-
- }
-
- private ByteArray encodeTime(long ts, int index, int roundingFlag) {
- String value;
- DataType partitionColType = gtInfo.getColumnType(index);
- if (partitionColType.isDate()) {
- value = DateFormat.formatToDateStr(ts);
- } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) {
- value = DateFormat.formatToTimeWithoutMilliStr(ts);
- } else if (partitionColType.isStringFamily()) {
- String partitionDateFormat = cubeSegment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateFormat();
- if (StringUtils.isEmpty(partitionDateFormat))
- partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
- value = DateFormat.formatToDateStr(ts, partitionDateFormat);
- } else {
- throw new RuntimeException("Type " + partitionColType + " is not valid partition column type");
- }
-
- ByteBuffer buffer = ByteBuffer.allocate(gtInfo.getMaxColumnLength());
- gtInfo.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer);
-
- return ByteArray.copyOf(buffer.array(), 0, buffer.position());
- }
-
private Set<TblColRef> replaceDerivedColumns(Set<TblColRef> input, CubeDesc cubeDesc) {
Set<TblColRef> ret = Sets.newHashSet();
for (TblColRef col : input) {
@@ -253,57 +181,6 @@ public class CubeScanRangePlanner {
return ret;
}
- private ImmutableBitSet makeGridTableColumns(CuboidToGridTableMapping mapping, Set<TblColRef> dimensions) {
- BitSet result = new BitSet();
- for (TblColRef dim : dimensions) {
- int idx = mapping.getIndexOf(dim);
- if (idx >= 0)
- result.set(idx);
- }
- return new ImmutableBitSet(result);
- }
-
- private ImmutableBitSet makeGridTableColumns(CuboidToGridTableMapping mapping, Collection<FunctionDesc> metrics) {
- BitSet result = new BitSet();
- for (FunctionDesc metric : metrics) {
- int idx = mapping.getIndexOf(metric);
- if (idx < 0)
- throw new IllegalStateException(metric + " not found in " + mapping);
- result.set(idx);
- }
- return new ImmutableBitSet(result);
- }
-
- private String[] makeAggrFuncs(final CuboidToGridTableMapping mapping, Collection<FunctionDesc> metrics) {
-
- //metrics are represented in ImmutableBitSet, which loses order information
- //sort the aggrFuns to align with metrics natural order
- List<FunctionDesc> metricList = Lists.newArrayList(metrics);
- Collections.sort(metricList, new Comparator<FunctionDesc>() {
- @Override
- public int compare(FunctionDesc o1, FunctionDesc o2) {
- int a = mapping.getIndexOf(o1);
- int b = mapping.getIndexOf(o2);
- return a - b;
- }
- });
-
- String[] result = new String[metricList.size()];
- int i = 0;
- for (FunctionDesc metric : metricList) {
- result[i++] = metric.getExpression();
- }
- return result;
- }
-
- private String makeReadable(ByteArray byteArray) {
- if (byteArray == null) {
- return null;
- } else {
- return byteArray.toReadableText();
- }
- }
-
protected GTScanRange newScanRange(Collection<ColumnRange> andDimRanges) {
GTRecord pkStart = new GTRecord(gtInfo);
GTRecord pkEnd = new GTRecord(gtInfo);
@@ -313,14 +190,14 @@ public class CubeScanRangePlanner {
for (ColumnRange range : andDimRanges) {
if (gtPartitionCol != null && range.column.equals(gtPartitionCol)) {
- if (rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end) <= 0 //
- && (rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond()) < 0 //
- || rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond()) == 0 //
- && (range.op == FilterOperatorEnum.EQ || range.op == FilterOperatorEnum.LTE || range.op == FilterOperatorEnum.GTE || range.op == FilterOperatorEnum.IN))) {
- //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded, so use <= when has equals in condition.
+ int beginCompare = rangeStartEndComparator.comparator.compare(range.begin, gtStartAndEnd.getSecond());
+ int endCompare = rangeStartEndComparator.comparator.compare(gtStartAndEnd.getFirst(), range.end);
+
+ if ((isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare < 0) || (!isPartitionColUsingDatetimeEncoding && endCompare <= 0 && beginCompare <= 0)) {
+ //segment range is [Closed,Open), but segmentStartAndEnd.getSecond() might be rounded when using dict encoding, so use <= when has equals in condition.
} else {
logger.debug("Pre-check partition col filter failed, partitionColRef {}, segment start {}, segment end {}, range begin {}, range end {}", //
- new Object[] { gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end) });
+ gtPartitionCol, makeReadable(gtStartAndEnd.getFirst()), makeReadable(gtStartAndEnd.getSecond()), makeReadable(range.begin), makeReadable(range.end));
return null;
}
}
@@ -337,7 +214,9 @@ public class CubeScanRangePlanner {
}
}
- fuzzyKeys = buildFuzzyKeys(fuzzyValues);
+ fuzzyKeys =
+
+ buildFuzzyKeys(fuzzyValues);
return new GTScanRange(pkStart, pkEnd, fuzzyKeys);
}
@@ -371,104 +250,6 @@ public class CubeScanRangePlanner {
return result;
}
- protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
- if (filter == null)
- return null;
-
- TupleFilter flatFilter = filter.flatFilter();
-
- // normalize to OR-AND filter
- if (flatFilter.getOperator() == FilterOperatorEnum.AND) {
- LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR);
- f.addChild(flatFilter);
- flatFilter = f;
- }
-
- if (flatFilter.getOperator() != FilterOperatorEnum.OR)
- throw new IllegalStateException();
-
- return flatFilter;
- }
-
- protected List<Collection<ColumnRange>> translateToOrAndDimRanges(TupleFilter flatFilter) {
- List<Collection<ColumnRange>> result = Lists.newArrayList();
-
- if (flatFilter == null) {
- result.add(Collections.<ColumnRange> emptyList());
- return result;
- }
-
- for (TupleFilter andFilter : flatFilter.getChildren()) {
- if (andFilter.getOperator() != FilterOperatorEnum.AND)
- throw new IllegalStateException("Filter should be AND instead of " + andFilter);
-
- Collection<ColumnRange> andRanges = translateToAndDimRanges(andFilter.getChildren());
- if (andRanges != null) {
- result.add(andRanges);
- }
- }
-
- return preEvaluateConstantConditions(result);
- }
-
- private Collection<ColumnRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters) {
- Map<TblColRef, ColumnRange> rangeMap = new HashMap<TblColRef, ColumnRange>();
- for (TupleFilter filter : andFilters) {
- if ((filter instanceof CompareTupleFilter) == false) {
- if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
- return null;
- } else {
- continue;
- }
- }
-
- CompareTupleFilter comp = (CompareTupleFilter) filter;
- if (comp.getColumn() == null) {
- continue;
- }
-
- @SuppressWarnings("unchecked")
- ColumnRange newRange = new ColumnRange(comp.getColumn(), (Set<ByteArray>) comp.getValues(), comp.getOperator());
- ColumnRange existing = rangeMap.get(newRange.column);
- if (existing == null) {
- rangeMap.put(newRange.column, newRange);
- } else {
- existing.andMerge(newRange);
- }
- }
- return rangeMap.values();
- }
-
- private List<Collection<ColumnRange>> preEvaluateConstantConditions(List<Collection<ColumnRange>> orAndRanges) {
- boolean globalAlwaysTrue = false;
- Iterator<Collection<ColumnRange>> iterator = orAndRanges.iterator();
- while (iterator.hasNext()) {
- Collection<ColumnRange> andRanges = iterator.next();
- Iterator<ColumnRange> iterator2 = andRanges.iterator();
- boolean hasAlwaysFalse = false;
- while (iterator2.hasNext()) {
- ColumnRange range = iterator2.next();
- if (range.satisfyAll())
- iterator2.remove();
- else if (range.satisfyNone())
- hasAlwaysFalse = true;
- }
- if (hasAlwaysFalse) {
- iterator.remove();
- } else if (andRanges.isEmpty()) {
- globalAlwaysTrue = true;
- break;
- }
- }
- // return empty OR list means global false
- // return an empty AND collection inside OR list means global true
- if (globalAlwaysTrue) {
- orAndRanges.clear();
- orAndRanges.add(Collections.<ColumnRange> emptyList());
- }
- return orAndRanges;
- }
-
protected List<GTScanRange> mergeOverlapRanges(List<GTScanRange> ranges) {
if (ranges.size() <= 1) {
return ranges;
@@ -556,269 +337,4 @@ public class CubeScanRangePlanner {
this.maxScanRanges = maxScanRanges;
}
- protected class ColumnRange {
- private TblColRef column;
- private ByteArray begin = ByteArray.EMPTY;
- private ByteArray end = ByteArray.EMPTY;
- private Set<ByteArray> valueSet;
- private FilterOperatorEnum op;
-
- public ColumnRange(TblColRef column, Set<ByteArray> values, FilterOperatorEnum op) {
- this.column = column;
- this.op = op;
-
- switch (op) {
- case EQ:
- case IN:
- valueSet = new HashSet<ByteArray>(values);
- refreshBeginEndFromEquals();
- break;
- case LT:
- case LTE:
- end = rangeEndComparator.comparator.max(values);
- break;
- case GT:
- case GTE:
- begin = rangeStartComparator.comparator.min(values);
- break;
- case NEQ:
- case NOTIN:
- case ISNULL:
- case ISNOTNULL:
- // let Optiq filter it!
- break;
- default:
- throw new UnsupportedOperationException(op.name());
- }
- }
-
- void copy(TblColRef column, ByteArray beginValue, ByteArray endValue, Set<ByteArray> equalValues) {
- this.column = column;
- this.begin = beginValue;
- this.end = endValue;
- this.valueSet = equalValues;
- }
-
- private void refreshBeginEndFromEquals() {
- if (valueSet.isEmpty()) {
- begin = ByteArray.EMPTY;
- end = ByteArray.EMPTY;
- } else {
- begin = rangeStartComparator.comparator.min(valueSet);
- end = rangeEndComparator.comparator.max(valueSet);
- }
- }
-
- public boolean satisfyAll() {
- return begin.array() == null && end.array() == null; // the NEQ case
- }
-
- public boolean satisfyNone() {
- if (valueSet != null) {
- return valueSet.isEmpty();
- } else if (begin.array() != null && end.array() != null) {
- return gtInfo.getCodeSystem().getComparator().compare(begin, end) > 0;
- } else {
- return false;
- }
- }
-
- public void andMerge(ColumnRange another) {
- assert this.column.equals(another.column);
-
- if (another.satisfyAll()) {
- return;
- }
-
- if (this.satisfyAll()) {
- copy(another.column, another.begin, another.end, another.valueSet);
- return;
- }
-
- if (this.valueSet != null && another.valueSet != null) {
- this.valueSet.retainAll(another.valueSet);
- refreshBeginEndFromEquals();
- return;
- }
-
- if (this.valueSet != null) {
- this.valueSet = filter(this.valueSet, another.begin, another.end);
- refreshBeginEndFromEquals();
- return;
- }
-
- if (another.valueSet != null) {
- this.valueSet = filter(another.valueSet, this.begin, this.end);
- refreshBeginEndFromEquals();
- return;
- }
-
- this.begin = rangeStartComparator.comparator.max(this.begin, another.begin);
- this.end = rangeEndComparator.comparator.min(this.end, another.end);
- }
-
- private Set<ByteArray> filter(Set<ByteArray> equalValues, ByteArray beginValue, ByteArray endValue) {
- Set<ByteArray> result = Sets.newHashSetWithExpectedSize(equalValues.size());
- for (ByteArray v : equalValues) {
- if (rangeStartEndComparator.comparator.compare(beginValue, v) <= 0 && rangeStartEndComparator.comparator.compare(v, endValue) <= 0) {
- result.add(v);
- }
- }
- return equalValues;
- }
-
- public String toString() {
- if (valueSet == null) {
- return column.getName() + " between " + begin + " and " + end;
- } else {
- return column.getName() + " in " + valueSet;
- }
- }
- }
-
- public static abstract class ComparatorEx<T> implements Comparator<T> {
-
- public T min(Collection<T> v) {
- if (v.size() <= 0) {
- return null;
- }
-
- Iterator<T> iterator = v.iterator();
- T min = iterator.next();
- while (iterator.hasNext()) {
- min = min(min, iterator.next());
- }
- return min;
- }
-
- public T max(Collection<T> v) {
- if (v.size() <= 0) {
- return null;
- }
-
- Iterator<T> iterator = v.iterator();
- T max = iterator.next();
- while (iterator.hasNext()) {
- max = max(max, iterator.next());
- }
- return max;
- }
-
- public T min(T a, T b) {
- return compare(a, b) <= 0 ? a : b;
- }
-
- public T max(T a, T b) {
- return compare(a, b) >= 0 ? a : b;
- }
-
- public boolean between(T v, T start, T end) {
- return compare(start, v) <= 0 && compare(v, end) <= 0;
- }
- }
-
- public static RecordComparator getRangeStartComparator(final IGTComparator comp) {
- return new RecordComparator(new ComparatorEx<ByteArray>() {
- @Override
- public int compare(ByteArray a, ByteArray b) {
- if (a.array() == null) {
- if (b.array() == null) {
- return 0;
- } else {
- return -1;
- }
- } else if (b.array() == null) {
- return 1;
- } else {
- return comp.compare(a, b);
- }
- }
- });
- }
-
- public static RecordComparator getRangeEndComparator(final IGTComparator comp) {
- return new RecordComparator(new ComparatorEx<ByteArray>() {
- @Override
- public int compare(ByteArray a, ByteArray b) {
- if (a.array() == null) {
- if (b.array() == null) {
- return 0;
- } else {
- return 1;
- }
- } else if (b.array() == null) {
- return -1;
- } else {
- return comp.compare(a, b);
- }
- }
- });
- }
-
- public static RecordComparator getRangeStartEndComparator(final IGTComparator comp) {
- return new AsymmetricRecordComparator(new ComparatorEx<ByteArray>() {
- @Override
- public int compare(ByteArray a, ByteArray b) {
- if (a.array() == null || b.array() == null) {
- return -1;
- } else {
- return comp.compare(a, b);
- }
- }
- });
- }
-
- private static class RecordComparator extends ComparatorEx<GTRecord> {
- final ComparatorEx<ByteArray> comparator;
-
- RecordComparator(ComparatorEx<ByteArray> byteComparator) {
- this.comparator = byteComparator;
- }
-
- @Override
- public int compare(GTRecord a, GTRecord b) {
- assert a.getInfo() == b.getInfo();
-
- int comp;
- ImmutableBitSet allColumns = a.getInfo().getAllColumns();
- for (int i = 0; i < allColumns.trueBitCount(); i++) {
- int c = allColumns.trueBitAt(i);
- comp = comparator.compare(a.get(c), b.get(c));
- if (comp != 0)
- return comp;
- }
- return 0; // equals
- }
- }
-
- /**
- * asymmetric means compare(a,b) > 0 does not cause compare(b,a) < 0
- * so min max functions will not be supported
- */
- private static class AsymmetricRecordComparator extends RecordComparator {
-
- AsymmetricRecordComparator(ComparatorEx<ByteArray> byteComparator) {
- super(byteComparator);
- }
-
- public GTRecord min(Collection<GTRecord> v) {
- throw new UnsupportedOperationException();
- }
-
- public GTRecord max(Collection<GTRecord> v) {
- throw new UnsupportedOperationException();
- }
-
- public GTRecord min(GTRecord a, GTRecord b) {
- throw new UnsupportedOperationException();
- }
-
- public GTRecord max(GTRecord a, GTRecord b) {
- throw new UnsupportedOperationException();
- }
-
- public boolean between(GTRecord v, GTRecord start, GTRecord end) {
- throw new UnsupportedOperationException();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
index 0904686..2e5dd12 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/CuboidToGridTableMapping.java
@@ -20,9 +20,12 @@ package org.apache.kylin.cube.gridtable;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -120,16 +123,8 @@ public class CuboidToGridTableMapping {
return nDimensions + nMetrics;
}
- public int getDimensionCount() {
- return nDimensions;
- }
-
- public int getMetricsCount() {
- return nMetrics;
- }
-
public DataType[] getDataTypes() {
- return (DataType[]) gtDataTypes.toArray(new DataType[gtDataTypes.size()]);
+ return gtDataTypes.toArray(new DataType[gtDataTypes.size()]);
}
public ImmutableBitSet getPrimaryKey() {
@@ -137,7 +132,7 @@ public class CuboidToGridTableMapping {
}
public ImmutableBitSet[] getColumnBlocks() {
- return (ImmutableBitSet[]) gtColBlocks.toArray(new ImmutableBitSet[gtColBlocks.size()]);
+ return gtColBlocks.toArray(new ImmutableBitSet[gtColBlocks.size()]);
}
public int getIndexOf(TblColRef dimension) {
@@ -147,7 +142,7 @@ public class CuboidToGridTableMapping {
public int getIndexOf(FunctionDesc metric) {
Integer r = metrics2gt.get(metric);
- return r == null ? -1 : r.intValue();
+ return r == null ? -1 : r;
}
public List<TblColRef> getCuboidDimensionsInGTOrder() {
@@ -185,4 +180,47 @@ public class CuboidToGridTableMapping {
return result.isEmpty() ? Collections.<Integer, Integer> emptyMap() : result;
}
+ public ImmutableBitSet makeGridTableColumns(Set<TblColRef> dimensions) {
+ BitSet result = new BitSet();
+ for (TblColRef dim : dimensions) {
+ int idx = this.getIndexOf(dim);
+ if (idx >= 0)
+ result.set(idx);
+ }
+ return new ImmutableBitSet(result);
+ }
+
+ public ImmutableBitSet makeGridTableColumns(Collection<FunctionDesc> metrics) {
+ BitSet result = new BitSet();
+ for (FunctionDesc metric : metrics) {
+ int idx = this.getIndexOf(metric);
+ if (idx < 0)
+ throw new IllegalStateException(metric + " not found in " + this);
+ result.set(idx);
+ }
+ return new ImmutableBitSet(result);
+ }
+
+ public String[] makeAggrFuncs(Collection<FunctionDesc> metrics) {
+
+ //metrics are represented in ImmutableBitSet, which loses order information
+ //sort the aggrFuns to align with metrics natural order
+ List<FunctionDesc> metricList = Lists.newArrayList(metrics);
+ Collections.sort(metricList, new Comparator<FunctionDesc>() {
+ @Override
+ public int compare(FunctionDesc o1, FunctionDesc o2) {
+ int a = CuboidToGridTableMapping.this.getIndexOf(o1);
+ int b = CuboidToGridTableMapping.this.getIndexOf(o2);
+ return a - b;
+ }
+ });
+
+ String[] result = new String[metricList.size()];
+ int i = 0;
+ for (FunctionDesc metric : metricList) {
+ result[i++] = metric.getExpression();
+ }
+ return result;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java
new file mode 100644
index 0000000..5360822
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.gridtable.GTRecord;
+
+public class RecordComparator extends ComparatorEx<GTRecord> {
+ final public ComparatorEx<ByteArray> comparator;
+
+ RecordComparator(ComparatorEx<ByteArray> byteComparator) {
+ this.comparator = byteComparator;
+ }
+
+ @Override
+ public int compare(GTRecord a, GTRecord b) {
+ assert a.getInfo() == b.getInfo();
+
+ int comp;
+ ImmutableBitSet allColumns = a.getInfo().getAllColumns();
+ for (int i = 0; i < allColumns.trueBitCount(); i++) {
+ int c = allColumns.trueBitAt(i);
+ comp = comparator.compare(a.get(c), b.get(c));
+ if (comp != 0)
+ return comp;
+ }
+ return 0; // equals
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java
new file mode 100644
index 0000000..43ea30b
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/RecordComparators.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.gridtable.IGTComparator;
+
+public class RecordComparators {
+
+ public static RecordComparator getRangeStartComparator(final IGTComparator comp) {
+ return new RecordComparator(new ComparatorEx<ByteArray>() {
+ @Override
+ public int compare(ByteArray a, ByteArray b) {
+ if (a.array() == null) {
+ if (b.array() == null) {
+ return 0;
+ } else {
+ return -1;
+ }
+ } else if (b.array() == null) {
+ return 1;
+ } else {
+ return comp.compare(a, b);
+ }
+ }
+ });
+ }
+
+ public static RecordComparator getRangeEndComparator(final IGTComparator comp) {
+ return new RecordComparator(new ComparatorEx<ByteArray>() {
+ @Override
+ public int compare(ByteArray a, ByteArray b) {
+ if (a.array() == null) {
+ if (b.array() == null) {
+ return 0;
+ } else {
+ return 1;
+ }
+ } else if (b.array() == null) {
+ return -1;
+ } else {
+ return comp.compare(a, b);
+ }
+ }
+ });
+ }
+
+ public static RecordComparator getRangeStartEndComparator(final IGTComparator comp) {
+ return new AsymmetricRecordComparator(new ComparatorEx<ByteArray>() {
+ @Override
+ public int compare(ByteArray a, ByteArray b) {
+ if (a.array() == null || b.array() == null) {
+ return -1;
+ } else {
+ return comp.compare(a, b);
+ }
+ }
+ });
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
new file mode 100644
index 0000000..d938f2b
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/ScanRangePlannerBase.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.ImmutableBitSet;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+public abstract class ScanRangePlannerBase {
+
+ //GT
+ protected TupleFilter gtFilter;
+ protected GTInfo gtInfo;
+ protected Pair<ByteArray, ByteArray> gtStartAndEnd;
+ protected TblColRef gtPartitionCol;
+ protected ImmutableBitSet gtDimensions;
+ protected ImmutableBitSet gtAggrGroups;
+ protected ImmutableBitSet gtAggrMetrics;
+ protected String[] gtAggrFuncs;
+ protected boolean isPartitionColUsingDatetimeEncoding = true;
+
+ protected RecordComparator rangeStartComparator;
+ protected RecordComparator rangeEndComparator;
+ protected RecordComparator rangeStartEndComparator;
+
+ public abstract GTScanRequest planScanRequest();
+
+ protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
+ if (filter == null)
+ return null;
+
+ TupleFilter flatFilter = filter.flatFilter();
+
+ // normalize to OR-AND filter
+ if (flatFilter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
+ LogicalTupleFilter f = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ f.addChild(flatFilter);
+ flatFilter = f;
+ }
+
+ if (flatFilter.getOperator() != TupleFilter.FilterOperatorEnum.OR)
+ throw new IllegalStateException();
+
+ return flatFilter;
+ }
+
+ protected List<Collection<ColumnRange>> translateToOrAndDimRanges(TupleFilter flatFilter) {
+ List<Collection<ColumnRange>> result = Lists.newArrayList();
+
+ if (flatFilter == null) {
+ result.add(Collections.<ColumnRange> emptyList());
+ return result;
+ }
+
+ for (TupleFilter andFilter : flatFilter.getChildren()) {
+ if (andFilter.getOperator() != TupleFilter.FilterOperatorEnum.AND)
+ throw new IllegalStateException("Filter should be AND instead of " + andFilter);
+
+ Collection<ColumnRange> andRanges = translateToAndDimRanges(andFilter.getChildren());
+ if (andRanges != null) {
+ result.add(andRanges);
+ }
+ }
+
+ return preEvaluateConstantConditions(result);
+ }
+
+ private Collection<ColumnRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters) {
+ Map<TblColRef, ColumnRange> rangeMap = new HashMap<TblColRef, ColumnRange>();
+ for (TupleFilter filter : andFilters) {
+ if ((filter instanceof CompareTupleFilter) == false) {
+ if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
+ return null;
+ } else {
+ continue;
+ }
+ }
+
+ CompareTupleFilter comp = (CompareTupleFilter) filter;
+ if (comp.getColumn() == null) {
+ continue;
+ }
+
+ @SuppressWarnings("unchecked")
+ ColumnRange newRange = new ColumnRange(comp.getColumn(), (Set<ByteArray>) comp.getValues(), comp.getOperator());
+ ColumnRange existing = rangeMap.get(newRange.column);
+ if (existing == null) {
+ rangeMap.put(newRange.column, newRange);
+ } else {
+ existing.andMerge(newRange);
+ }
+ }
+ return rangeMap.values();
+ }
+
+ private List<Collection<ColumnRange>> preEvaluateConstantConditions(List<Collection<ColumnRange>> orAndRanges) {
+ boolean globalAlwaysTrue = false;
+ Iterator<Collection<ColumnRange>> iterator = orAndRanges.iterator();
+ while (iterator.hasNext()) {
+ Collection<ColumnRange> andRanges = iterator.next();
+ Iterator<ColumnRange> iterator2 = andRanges.iterator();
+ boolean hasAlwaysFalse = false;
+ while (iterator2.hasNext()) {
+ ColumnRange range = iterator2.next();
+ if (range.satisfyAll())
+ iterator2.remove();
+ else if (range.satisfyNone())
+ hasAlwaysFalse = true;
+ }
+ if (hasAlwaysFalse) {
+ iterator.remove();
+ } else if (andRanges.isEmpty()) {
+ globalAlwaysTrue = true;
+ break;
+ }
+ }
+ // return empty OR list means global false
+ // return an empty AND collection inside OR list means global true
+ if (globalAlwaysTrue) {
+ orAndRanges.clear();
+ orAndRanges.add(Collections.<ColumnRange> emptyList());
+ }
+ return orAndRanges;
+ }
+
+ public class ColumnRange {
+ public TblColRef column;
+ public ByteArray begin = ByteArray.EMPTY;
+ public ByteArray end = ByteArray.EMPTY;
+ public Set<ByteArray> valueSet;
+ public boolean isBoundryInclusive;
+
+ public ColumnRange(TblColRef column, Set<ByteArray> values, TupleFilter.FilterOperatorEnum op) {
+ this.column = column;
+
+ //TODO: the treatment is un-precise
+ if (op == TupleFilter.FilterOperatorEnum.EQ || op == TupleFilter.FilterOperatorEnum.IN || op == TupleFilter.FilterOperatorEnum.LTE || op == TupleFilter.FilterOperatorEnum.GTE) {
+ isBoundryInclusive = true;
+ }
+
+ switch (op) {
+ case EQ:
+ case IN:
+ valueSet = new HashSet<ByteArray>(values);
+ refreshBeginEndFromEquals();
+ break;
+ case LT:
+ case LTE:
+ end = rangeEndComparator.comparator.max(values);
+ break;
+ case GT:
+ case GTE:
+ begin = rangeStartComparator.comparator.min(values);
+ break;
+ case NEQ:
+ case NOTIN:
+ case ISNULL:
+ case ISNOTNULL:
+ // let Optiq filter it!
+ break;
+ default:
+ throw new UnsupportedOperationException(op.name());
+ }
+ }
+
+ void copy(TblColRef column, ByteArray beginValue, ByteArray endValue, Set<ByteArray> equalValues) {
+ this.column = column;
+ this.begin = beginValue;
+ this.end = endValue;
+ this.valueSet = equalValues;
+ }
+
+ private void refreshBeginEndFromEquals() {
+ if (valueSet.isEmpty()) {
+ begin = ByteArray.EMPTY;
+ end = ByteArray.EMPTY;
+ } else {
+ begin = rangeStartComparator.comparator.min(valueSet);
+ end = rangeEndComparator.comparator.max(valueSet);
+ }
+ }
+
+ public boolean satisfyAll() {
+ return begin.array() == null && end.array() == null; // the NEQ case
+ }
+
+ public boolean satisfyNone() {
+ if (valueSet != null) {
+ return valueSet.isEmpty();
+ } else if (begin.array() != null && end.array() != null) {
+ return gtInfo.getCodeSystem().getComparator().compare(begin, end) > 0;
+ } else {
+ return false;
+ }
+ }
+
+ public void andMerge(ColumnRange another) {
+ assert this.column.equals(another.column);
+
+ if (another.satisfyAll()) {
+ return;
+ }
+
+ if (this.satisfyAll()) {
+ copy(another.column, another.begin, another.end, another.valueSet);
+ return;
+ }
+
+ if (this.valueSet != null && another.valueSet != null) {
+ this.valueSet.retainAll(another.valueSet);
+ refreshBeginEndFromEquals();
+ return;
+ }
+
+ if (this.valueSet != null) {
+ this.valueSet = filter(this.valueSet, another.begin, another.end);
+ refreshBeginEndFromEquals();
+ return;
+ }
+
+ if (another.valueSet != null) {
+ this.valueSet = filter(another.valueSet, this.begin, this.end);
+ refreshBeginEndFromEquals();
+ return;
+ }
+
+ this.begin = rangeStartComparator.comparator.max(this.begin, another.begin);
+ this.end = rangeEndComparator.comparator.min(this.end, another.end);
+ this.isBoundryInclusive |= another.isBoundryInclusive;
+ }
+
+ private Set<ByteArray> filter(Set<ByteArray> equalValues, ByteArray beginValue, ByteArray endValue) {
+ Set<ByteArray> result = Sets.newHashSetWithExpectedSize(equalValues.size());
+ for (ByteArray v : equalValues) {
+ if (rangeStartEndComparator.comparator.compare(beginValue, v) <= 0 && rangeStartEndComparator.comparator.compare(v, endValue) <= 0) {
+ result.add(v);
+ }
+ }
+ return equalValues;
+ }
+
+ public String toString() {
+ if (valueSet == null) {
+ return column.getName() + " between " + begin + " and " + end;
+ } else {
+ return column.getName() + " in " + valueSet;
+ }
+ }
+
+ }
+
+ protected String makeReadable(ByteArray byteArray) {
+ if (byteArray == null) {
+ return null;
+ } else {
+ return byteArray.toReadableText();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
new file mode 100644
index 0000000..e31111d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/SegmentGTStartAndEnd.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.gridtable;
+
+import java.nio.ByteBuffer;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.ISegment;
+import org.apache.kylin.dimension.AbstractDateDimEnc;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.metadata.datatype.DataType;
+
+public class SegmentGTStartAndEnd {
+ private ISegment segment;
+ private GTInfo info;
+
+ public SegmentGTStartAndEnd(ISegment segment, GTInfo info) {
+ this.segment = segment;
+ this.info = info;
+ }
+
+ public boolean isUsingDatetimeEncoding(int index) {
+ return info.getCodeSystem().getDimEnc(index) instanceof AbstractDateDimEnc;
+ }
+
+ public Pair<ByteArray, ByteArray> getSegmentStartAndEnd(int index) {
+ ByteArray start;
+ if (segment.getDateRangeStart() != Long.MIN_VALUE) {
+ start = encodeTime(segment.getDateRangeStart(), index, 1);
+ } else {
+ start = new ByteArray();
+ }
+
+ ByteArray end;
+ if (segment.getDateRangeEnd() != Long.MAX_VALUE) {
+ end = encodeTime(segment.getDateRangeEnd(), index, -1);
+ } else {
+ end = new ByteArray();
+ }
+ return Pair.newPair(start, end);
+
+ }
+
+ private ByteArray encodeTime(long ts, int index, int roundingFlag) {
+ String value;
+ DataType partitionColType = info.getColumnType(index);
+ if (partitionColType.isDate()) {
+ value = DateFormat.formatToDateStr(ts);
+ } else if (partitionColType.isDatetime() || partitionColType.isTimestamp()) {
+ value = DateFormat.formatToTimeWithoutMilliStr(ts);
+ } else if (partitionColType.isStringFamily()) {
+ String partitionDateFormat = segment.getModel().getPartitionDesc().getPartitionDateFormat();
+ if (StringUtils.isEmpty(partitionDateFormat))
+ partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
+ value = DateFormat.formatToDateStr(ts, partitionDateFormat);
+ } else {
+ throw new RuntimeException("Type " + partitionColType + " is not valid partition column type");
+ }
+
+ ByteBuffer buffer = ByteBuffer.allocate(info.getMaxColumnLength());
+ info.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer);
+
+ return ByteArray.copyOf(buffer.array(), 0, buffer.position());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
index 21da4ea..12a3593 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInfo.java
@@ -31,8 +31,11 @@ import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.cube.gridtable.TrimmedCubeCodeSystem;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class GTInfo {
+ private static final Logger logger = LoggerFactory.getLogger(GTInfo.class);
public static Builder builder() {
return new Builder();
@@ -151,7 +154,7 @@ public class GTInfo {
if (!expected.equals(ref))
throw new IllegalArgumentException();
}
-
+
void validate() {
if (codeSystem == null)
throw new IllegalStateException();
@@ -271,7 +274,7 @@ public class GTInfo {
public IGTCodeSystem getCodeSystem() {
return codeSystem;
}
-
+
public int getMaxLength() {
int ret = 0;
for (int i = 0; i < colAll.trueBitCount(); i++) {
@@ -291,7 +294,7 @@ public class GTInfo {
BytesUtil.writeAsciiString(GTSampleCodeSystem.class.getCanonicalName(), out);
GTSampleCodeSystem.serializer.serialize((GTSampleCodeSystem) value.codeSystem, out);
} else {
- throw new IllegalArgumentException("Can't recognize code system " + value.codeSystem.getClass());
+ BytesUtil.writeAsciiString(value.codeSystem.getClass().getCanonicalName(), out);
}
BytesUtil.writeUTFString(value.tableName, out);
@@ -317,7 +320,11 @@ public class GTInfo {
} else if (GTSampleCodeSystem.class.getCanonicalName().equals(codeSystemType)) {
codeSystem = GTSampleCodeSystem.serializer.deserialize(in);
} else {
- throw new IllegalArgumentException("Can't recognize code system " + codeSystemType);
+ try {
+ codeSystem = (IGTCodeSystem) Class.forName(codeSystemType).newInstance();
+ } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
}
String newTableName = BytesUtil.readUTFString(in);
@@ -349,5 +356,4 @@ public class GTInfo {
}
};
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index e2bac3d..4cfba1b 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -54,8 +54,8 @@ public class GTScanRequest {
// optional aggregation
private ImmutableBitSet aggrGroupBy;
private ImmutableBitSet aggrMetrics;
- private String[] aggrMetricsFuncs;
-
+ private String[] aggrMetricsFuncs;//
+
// hint to storage behavior
private boolean allowStorageAggregation;
private double aggCacheMemThreshold;
@@ -204,7 +204,7 @@ public class GTScanRequest {
//TODO BUG? select sum() from fact, no aggr by
public boolean hasAggregation() {
- return aggrGroupBy != null && aggrMetrics != null && aggrMetricsFuncs != null;
+ return !aggrGroupBy.isEmpty() || !aggrMetrics.isEmpty();
}
public GTInfo getInfo() {
@@ -284,16 +284,6 @@ public class GTScanRequest {
this.storagePushDownLimit = limit;
}
- public List<Integer> getRequiredMeasures() {
- List<Integer> measures = Lists.newArrayList();
- int numDim = info.getPrimaryKey().trueBitCount();
- for (int i = 0; i < aggrMetrics.trueBitCount(); i++) {
- int index = aggrMetrics.trueBitAt(i);
- measures.add(index - numDim);
- }
- return measures;
- }
-
@Override
public String toString() {
return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]";
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
index 49ec759..c4390cd 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
@@ -18,6 +18,7 @@
package org.apache.kylin.gridtable;
+import java.util.BitSet;
import java.util.List;
import org.apache.kylin.common.util.ImmutableBitSet;
@@ -92,6 +93,18 @@ public class GTScanRequestBuilder {
}
public GTScanRequest createGTScanRequest() {
+ if (aggrGroupBy == null) {
+ aggrGroupBy = new ImmutableBitSet(new BitSet());
+ }
+
+ if (aggrMetrics == null) {
+ aggrMetrics = new ImmutableBitSet(new BitSet());
+ }
+
+ if (aggrMetricsFuncs == null) {
+ aggrMetricsFuncs = new String[0];
+ }
+
return new GTScanRequest(info, ranges, dimensions, aggrGroupBy, aggrMetrics, aggrMetricsFuncs, filterPushDown, allowStorageAggregation, aggCacheMemThreshold, storageScanRowNumThreshold, storagePushDownLimit);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
index 5be91be..bb7503a 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.Iterator;
-import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,9 +34,9 @@ public class ScannerWorker {
private static final Logger logger = LoggerFactory.getLogger(ScannerWorker.class);
private IGTScanner internal = null;
- public ScannerWorker(CubeSegment cubeSeg, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) {
+ public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) {
if (scanRequest == null) {
- logger.info("Segment {} will be skipped", cubeSeg);
+ logger.info("Segment {} will be skipped", segment);
internal = new EmptyGTScanner(0);
return;
}
@@ -44,7 +44,7 @@ public class ScannerWorker {
final GTInfo info = scanRequest.getInfo();
try {
- IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(CubeSegment.class, Cuboid.class, GTInfo.class).newInstance(cubeSeg, cuboid, info); // default behavior
+ IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(ISegment.class, Cuboid.class, GTInfo.class).newInstance(segment, cuboid, info); // default behavior
internal = rpc.getGTScanner(scanRequest);
} catch (IOException | InstantiationException | InvocationTargetException | IllegalAccessException | ClassNotFoundException | NoSuchMethodException e) {
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
index 2bbf6b7..7b6d3fa 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DictGridTableTest.java
@@ -132,7 +132,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
LogicalTupleFilter filter = and(timeComp4, ageComp1);
CubeScanRangePlanner planner = new CubeScanRangePlanner(info, Pair.newPair(segmentStart, segmentEnd), info.colRef(0), filter);
List<GTScanRange> r = planner.planScanRanges();
- assertEquals(1, r.size());
+ assertEquals(0, r.size());
}
{
LogicalTupleFilter filter = and(timeComp5, ageComp1);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index da2cf98..1f883bd 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -32,6 +32,7 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.metadata.realization.RealizationType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -206,6 +207,13 @@ class ProjectL2Cache {
} else {
logger.warn("Realization '" + entry + "' defined under project '" + project + "' is not found");
}
+
+ //check if there's raw table parasite
+ //TODO: ugly impl here
+ IRealization parasite = registry.getRealization(RealizationType.INVERTED_INDEX, entry.getRealization());
+ if (parasite != null) {
+ projectCache.realizations.add(parasite);
+ }
}
for (IRealization realization : projectCache.realizations) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index aa90fc5..1eee1e7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -44,7 +44,6 @@ public class SQLDigest {
public Collection<FunctionDesc> aggregations;
public Collection<MeasureDesc> sortMeasures;
public Collection<OrderEnum> sortOrders;
- private boolean isRawQuery = false;
public SQLDigest(String factTable, TupleFilter filter, Collection<JoinDesc> joinDescs, Collection<TblColRef> allColumns, //
Collection<TblColRef> groupbyColumns, Collection<TblColRef> filterColumns, Collection<TblColRef> aggregatedColumns, Collection<FunctionDesc> aggregateFunnc, Collection<MeasureDesc> sortMeasures, Collection<OrderEnum> sortOrders) {
@@ -58,11 +57,12 @@ public class SQLDigest {
this.aggregations = aggregateFunnc;
this.sortMeasures = sortMeasures;
this.sortOrders = sortOrders;
- this.isRawQuery = this.groupbyColumns.isEmpty() && this.metricColumns.isEmpty();
}
public boolean isRawQuery() {
- return isRawQuery;
+ return this.groupbyColumns.isEmpty() && // select a group by a -> not raw
+ this.aggregations.isEmpty(); // has aggr -> not raw
+ //the reason to choose aggregations rather than metricColumns is because the former is set earlier at implOLAP
}
@Override
[29/50] [abbrv] kylin git commit: minor, fix checkstyle
Posted by li...@apache.org.
minor, fix checkstyle
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a9ba1da1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a9ba1da1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a9ba1da1
Branch: refs/heads/1.5.x-CDH5.7
Commit: a9ba1da138966cb3fd6c4bd6950fe4b48597cf8d
Parents: 9e0b8d8
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Aug 31 11:54:02 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Aug 31 11:54:02 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/gridtable/GTUtil.java | 383 ++++++++++---------
1 file changed, 199 insertions(+), 184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9ba1da1/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index ce0f016..7406e81 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -75,224 +75,239 @@ public class GTUtil {
IFilterCodeSystem<ByteArray> filterCodeSystem = wrap(info.codeSystem.getComparator());
- byte[] bytes = TupleFilterSerializer.serialize(rootFilter, new TupleFilterSerializer.Decorator() {
- @Override
- public TupleFilter onSerialize(TupleFilter filter) {
- if (filter == null)
- return null;
-
- // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
- // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter)
- // will always return FALSE.
- if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
- TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
- return ConstantTupleFilter.TRUE;
- }
+ byte[] bytes = TupleFilterSerializer.serialize(rootFilter, new GTConvertDecorator(unevaluatableColumnCollector, colMapping, info, encodeConstants), filterCodeSystem);
- // shortcut for unEvaluatable filter
- if (!filter.isEvaluable()) {
- TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
- return ConstantTupleFilter.TRUE;
- }
+ return TupleFilterSerializer.deserialize(bytes, filterCodeSystem);
+ }
- // map to column onto grid table
- if (colMapping != null && filter instanceof ColumnTupleFilter) {
- ColumnTupleFilter colFilter = (ColumnTupleFilter) filter;
- int gtColIdx = colMapping.indexOf(colFilter.getColumn());
- return new ColumnTupleFilter(info.colRef(gtColIdx));
- }
+ public static IFilterCodeSystem<ByteArray> wrap(final IGTComparator comp) {
+ return new IFilterCodeSystem<ByteArray>() {
- // encode constants
- if (encodeConstants && filter instanceof CompareTupleFilter) {
- return encodeConstants((CompareTupleFilter) filter);
- }
- if (encodeConstants && filter instanceof BuiltInFunctionTupleFilter) {
- if (!((BuiltInFunctionTupleFilter) filter).hasNested()) {
- return encodeConstants((BuiltInFunctionTupleFilter) filter);
- } else {
- throw new IllegalStateException("Nested BuiltInFunctionTupleFilter is not supported to be pushed down");
- }
- }
+ @Override
+ public int compare(ByteArray o1, ByteArray o2) {
+ return comp.compare(o1, o2);
+ }
- return filter;
+ @Override
+ public boolean isNull(ByteArray code) {
+ return comp.isNull(code);
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
- private TupleFilter encodeConstants(CompareTupleFilter oldCompareFilter) {
- // extract ColumnFilter & ConstantFilter
- TblColRef externalCol = oldCompareFilter.getColumn();
+ @Override
+ public void serialize(ByteArray code, ByteBuffer buffer) {
+ if (code == null)
+ BytesUtil.writeByteArray(null, 0, 0, buffer);
+ else
+ BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
+ }
- if (externalCol == null) {
- return oldCompareFilter;
- }
+ @Override
+ public ByteArray deserialize(ByteBuffer buffer) {
+ return new ByteArray(BytesUtil.readByteArray(buffer));
+ }
+ };
+ }
- Collection constValues = oldCompareFilter.getValues();
- if (constValues == null || constValues.isEmpty()) {
- return oldCompareFilter;
- }
+ private static class GTConvertDecorator implements TupleFilterSerializer.Decorator {
+ private final Set<TblColRef> unevaluatableColumnCollector;
+ private final List<TblColRef> colMapping;
+ private final GTInfo info;
+ private final boolean encodeConstants;
+
+ public GTConvertDecorator(Set<TblColRef> unevaluatableColumnCollector, List<TblColRef> colMapping, GTInfo info, boolean encodeConstants) {
+ this.unevaluatableColumnCollector = unevaluatableColumnCollector;
+ this.colMapping = colMapping;
+ this.info = info;
+ this.encodeConstants = encodeConstants;
+ buf = ByteBuffer.allocate(info.getMaxColumnLength());
+ }
+
+ @Override
+ public TupleFilter onSerialize(TupleFilter filter) {
+ if (filter == null)
+ return null;
+
+ // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
+ // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter)
+ // will always return FALSE.
+ if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
+ TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
+ return ConstantTupleFilter.TRUE;
+ }
- //CompareTupleFilter containing BuiltInFunctionTupleFilter will not reach here caz it will be transformed by BuiltInFunctionTransformer
- CompareTupleFilter newCompareFilter = new CompareTupleFilter(oldCompareFilter.getOperator());
- newCompareFilter.addChild(new ColumnTupleFilter(externalCol));
-
- //for CompareTupleFilter containing dynamicVariables, the below codes will actually replace dynamicVariables
- //with normal ConstantTupleFilter
-
- Object firstValue = constValues.iterator().next();
- int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
-
- TupleFilter result;
- ByteArray code;
-
- // translate constant into code
- switch (newCompareFilter.getOperator()) {
- case EQ:
- case IN:
- Set newValues = Sets.newHashSet();
- for (Object value : constValues) {
- code = translate(col, value, 0);
- if (code != null)
- newValues.add(code);
- }
- if (newValues.isEmpty()) {
- result = ConstantTupleFilter.FALSE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(newValues));
- result = newCompareFilter;
- }
- break;
- case NEQ:
- code = translate(col, firstValue, 0);
- if (code == null) {
- result = ConstantTupleFilter.TRUE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(code));
- result = newCompareFilter;
- }
- break;
- case LT:
- code = translate(col, firstValue, 1);
- if (code == null) {
- result = ConstantTupleFilter.TRUE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(code));
- result = newCompareFilter;
- }
- break;
- case LTE:
- code = translate(col, firstValue, -1);
- if (code == null) {
- result = ConstantTupleFilter.FALSE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(code));
- result = newCompareFilter;
- }
- break;
- case GT:
- code = translate(col, firstValue, -1);
- if (code == null) {
- result = ConstantTupleFilter.TRUE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(code));
- result = newCompareFilter;
- }
- break;
- case GTE:
- code = translate(col, firstValue, 1);
- if (code == null) {
- result = ConstantTupleFilter.FALSE;
- } else {
- newCompareFilter.addChild(new ConstantTupleFilter(code));
- result = newCompareFilter;
- }
- break;
- default:
- throw new IllegalStateException("Cannot handle operator " + newCompareFilter.getOperator());
- }
- return result;
+ // shortcut for unEvaluatable filter
+ if (!filter.isEvaluable()) {
+ TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
+ return ConstantTupleFilter.TRUE;
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
- private TupleFilter encodeConstants(BuiltInFunctionTupleFilter funcFilter) {
- // extract ColumnFilter & ConstantFilter
- TblColRef externalCol = funcFilter.getColumn();
+ // map to column onto grid table
+ if (colMapping != null && filter instanceof ColumnTupleFilter) {
+ ColumnTupleFilter colFilter = (ColumnTupleFilter) filter;
+ int gtColIdx = colMapping.indexOf(colFilter.getColumn());
+ return new ColumnTupleFilter(info.colRef(gtColIdx));
+ }
- if (externalCol == null) {
- return funcFilter;
+ // encode constants
+ if (encodeConstants && filter instanceof CompareTupleFilter) {
+ return encodeConstants((CompareTupleFilter) filter);
+ }
+ if (encodeConstants && filter instanceof BuiltInFunctionTupleFilter) {
+ if (!((BuiltInFunctionTupleFilter) filter).hasNested()) {
+ return encodeConstants((BuiltInFunctionTupleFilter) filter);
+ } else {
+ throw new IllegalStateException("Nested BuiltInFunctionTupleFilter is not supported to be pushed down");
}
+ }
- Collection constValues = funcFilter.getConstantTupleFilter().getValues();
- if (constValues == null || constValues.isEmpty()) {
- return funcFilter;
- }
+ return filter;
+ }
- BuiltInFunctionTupleFilter newFuncFilter;
- try {
- newFuncFilter = funcFilter.getClass().getConstructor(String.class).newInstance(funcFilter.getName());
- } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
- throw new RuntimeException(e);
- }
- newFuncFilter.addChild(new ColumnTupleFilter(externalCol));
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private TupleFilter encodeConstants(CompareTupleFilter oldCompareFilter) {
+ // extract ColumnFilter & ConstantFilter
+ TblColRef externalCol = oldCompareFilter.getColumn();
- int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
+ if (externalCol == null) {
+ return oldCompareFilter;
+ }
+
+ Collection constValues = oldCompareFilter.getValues();
+ if (constValues == null || constValues.isEmpty()) {
+ return oldCompareFilter;
+ }
+
+ //CompareTupleFilter containing BuiltInFunctionTupleFilter will not reach here caz it will be transformed by BuiltInFunctionTransformer
+ CompareTupleFilter newCompareFilter = new CompareTupleFilter(oldCompareFilter.getOperator());
+ newCompareFilter.addChild(new ColumnTupleFilter(externalCol));
+
+ //for CompareTupleFilter containing dynamicVariables, the below codes will actually replace dynamicVariables
+ //with normal ConstantTupleFilter
- ByteArray code;
+ Object firstValue = constValues.iterator().next();
+ int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
- // translate constant into code
+ TupleFilter result;
+ ByteArray code;
+
+ // translate constant into code
+ switch (newCompareFilter.getOperator()) {
+ case EQ:
+ case IN:
Set newValues = Sets.newHashSet();
for (Object value : constValues) {
code = translate(col, value, 0);
- if (code == null) {
- throw new IllegalStateException("Cannot serialize BuiltInFunctionTupleFilter");
- }
- newValues.add(code);
+ if (code != null)
+ newValues.add(code);
+ }
+ if (newValues.isEmpty()) {
+ result = ConstantTupleFilter.FALSE;
+ } else {
+ newCompareFilter.addChild(new ConstantTupleFilter(newValues));
+ result = newCompareFilter;
+ }
+ break;
+ case NEQ:
+ code = translate(col, firstValue, 0);
+ if (code == null) {
+ result = ConstantTupleFilter.TRUE;
+ } else {
+ newCompareFilter.addChild(new ConstantTupleFilter(code));
+ result = newCompareFilter;
+ }
+ break;
+ case LT:
+ code = translate(col, firstValue, 1);
+ if (code == null) {
+ result = ConstantTupleFilter.TRUE;
+ } else {
+ newCompareFilter.addChild(new ConstantTupleFilter(code));
+ result = newCompareFilter;
+ }
+ break;
+ case LTE:
+ code = translate(col, firstValue, -1);
+ if (code == null) {
+ result = ConstantTupleFilter.FALSE;
+ } else {
+ newCompareFilter.addChild(new ConstantTupleFilter(code));
+ result = newCompareFilter;
}
- newFuncFilter.addChild(new ConstantTupleFilter(newValues));
+ break;
+ case GT:
+ code = translate(col, firstValue, -1);
+ if (code == null) {
+ result = ConstantTupleFilter.TRUE;
+ } else {
+ newCompareFilter.addChild(new ConstantTupleFilter(code));
+ result = newCompareFilter;
+ }
+ break;
+ case GTE:
+ code = translate(col, firstValue, 1);
+ if (code == null) {
+ result = ConstantTupleFilter.FALSE;
+ } else {
+ newCompareFilter.addChild(new ConstantTupleFilter(code));
+ result = newCompareFilter;
+ }
+ break;
+ default:
+ throw new IllegalStateException("Cannot handle operator " + newCompareFilter.getOperator());
+ }
+ return result;
+ }
- return newFuncFilter;
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private TupleFilter encodeConstants(BuiltInFunctionTupleFilter funcFilter) {
+ // extract ColumnFilter & ConstantFilter
+ TblColRef externalCol = funcFilter.getColumn();
+
+ if (externalCol == null) {
+ return funcFilter;
}
- transient ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
+ Collection constValues = funcFilter.getConstantTupleFilter().getValues();
+ if (constValues == null || constValues.isEmpty()) {
+ return funcFilter;
+ }
- private ByteArray translate(int col, Object value, int roundingFlag) {
- try {
- buf.clear();
- info.codeSystem.encodeColumnValue(col, value, roundingFlag, buf);
- return ByteArray.copyOf(buf.array(), 0, buf.position());
- } catch (IllegalArgumentException ex) {
- return null;
- }
+ BuiltInFunctionTupleFilter newFuncFilter;
+ try {
+ newFuncFilter = funcFilter.getClass().getConstructor(String.class).newInstance(funcFilter.getName());
+ } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
+ throw new RuntimeException(e);
}
- }, filterCodeSystem);
+ newFuncFilter.addChild(new ColumnTupleFilter(externalCol));
- return TupleFilterSerializer.deserialize(bytes, filterCodeSystem);
- }
+ int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
- public static IFilterCodeSystem<ByteArray> wrap(final IGTComparator comp) {
- return new IFilterCodeSystem<ByteArray>() {
+ ByteArray code;
- @Override
- public int compare(ByteArray o1, ByteArray o2) {
- return comp.compare(o1, o2);
+ // translate constant into code
+ Set newValues = Sets.newHashSet();
+ for (Object value : constValues) {
+ code = translate(col, value, 0);
+ if (code == null) {
+ throw new IllegalStateException("Cannot serialize BuiltInFunctionTupleFilter");
+ }
+ newValues.add(code);
}
+ newFuncFilter.addChild(new ConstantTupleFilter(newValues));
- @Override
- public boolean isNull(ByteArray code) {
- return comp.isNull(code);
- }
+ return newFuncFilter;
+ }
- @Override
- public void serialize(ByteArray code, ByteBuffer buffer) {
- if (code == null)
- BytesUtil.writeByteArray(null, 0, 0, buffer);
- else
- BytesUtil.writeByteArray(code.array(), code.offset(), code.length(), buffer);
- }
+ transient ByteBuffer buf;
- @Override
- public ByteArray deserialize(ByteBuffer buffer) {
- return new ByteArray(BytesUtil.readByteArray(buffer));
+ private ByteArray translate(int col, Object value, int roundingFlag) {
+ try {
+ buf.clear();
+ info.codeSystem.encodeColumnValue(col, value, roundingFlag, buf);
+ return ByteArray.copyOf(buf.array(), 0, buf.position());
+ } catch (IllegalArgumentException ex) {
+ return null;
}
- };
+ }
}
}
[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
Posted by li...@apache.org.
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6ee0e7c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6ee0e7c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6ee0e7c
Branch: refs/heads/1.5.x-CDH5.7
Commit: f6ee0e7c83eb0fa7b363fc734398baf5d00baaaf
Parents: 67bcec2
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sat Sep 3 14:35:08 2016 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/hbase-site.xml | 19 +---
.../test_case_data/sandbox/kylin_job_conf.xml | 86 +++++++++---------
examples/test_case_data/sandbox/mapred-site.xml | 23 +++--
.../kylin/provision/BuildCubeWithEngine.java | 13 ++-
pom.xml | 12 +--
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 ++++----------------
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 +++---
.../apache/kylin/rest/service/CubeService.java | 36 +++-----
.../apache/kylin/rest/service/QueryService.java | 24 +++--
.../apache/kylin/rest/service/UserService.java | 17 ++--
.../kylin/storage/hbase/HBaseConnection.java | 44 ++++-----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++----
.../kylin/storage/hbase/HBaseStorage.java | 3 +-
.../storage/hbase/cube/SimpleHBaseStore.java | 20 ++---
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 11 +--
.../storage/hbase/cube/v1/CubeStorageQuery.java | 6 +-
.../hbase/cube/v1/RegionScannerAdapter.java | 10 ++-
.../cube/v1/SerializedHBaseTupleIterator.java | 4 +-
.../observer/AggregateRegionObserver.java | 4 +-
.../observer/AggregationScanner.java | 14 ++-
.../observer/ObserverAggregationCache.java | 10 ++-
.../coprocessor/observer/ObserverEnabler.java | 4 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../hbase/cube/v2/ExpectedSizeIterator.java | 10 +--
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
.../storage/hbase/steps/DeprecatedGCStep.java | 23 ++---
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../hbase/steps/HBaseStreamingOutput.java | 9 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++---
.../storage/hbase/util/CleanHtableCLI.java | 12 +--
.../storage/hbase/util/CubeMigrationCLI.java | 36 ++++----
.../hbase/util/CubeMigrationCheckCLI.java | 17 ++--
.../hbase/util/DeployCoprocessorCLI.java | 22 ++---
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 +++----
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++--
.../hbase/util/HBaseRegionSizeCalculator.java | 35 ++++----
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 +++----
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
.../storage/hbase/util/StorageCleanupJob.java | 20 +++--
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
.../observer/AggregateRegionObserverTest.java | 26 ++----
.../v1/filter/TestFuzzyRowFilterV2EndToEnd.java | 5 +-
52 files changed, 464 insertions(+), 525 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
- <property>
- <name>hbase.client.pause</name>
- <value>100</value>
- <description>General client pause value. Used mostly as value to wait
- before running a retry of a failed get, region lookup, etc.
- See hbase.client.retries.number for description of how we backoff from
- this initial pause amount and how this pause works w/ retries.</description>
- </property>
- <property>
- <name>hbase.client.retries.number</name>
- <value>5</value>
- <description>Maximum retries. Used as maximum for all retryable
- operations such as the getting of a cell's value, starting a row update,
- etc. Retry interval is a rough function based on hbase.client.pause. At
- first we retry at this interval but then with backoff, we pretty quickly reach
- retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
- ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/examples/test_case_data/sandbox/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.xml b/examples/test_case_data/sandbox/kylin_job_conf.xml
index bd947af..6082fa9 100644
--- a/examples/test_case_data/sandbox/kylin_job_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_job_conf.xml
@@ -1,20 +1,18 @@
<?xml version="1.0"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
-->
+
<configuration>
<property>
@@ -26,44 +24,41 @@
</description>
</property>
- <property>
- <name>mapreduce.map.maxattempts</name>
- <value>2</value>
- </property>
+ <!-- uncomment the following 5 properties to enable lzo compressing
- <!--
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
+ <property>
+ <name>mapred.compress.map.output</name>
+ <value>true</value>
+ <description>Compress map outputs</description>
+ </property>
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
+ <property>
+ <name>mapred.map.output.compression.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description>The compression codec to use for map outputs
+ </description>
+ </property>
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
+ <property>
+ <name>mapred.output.compress</name>
+ <value>true</value>
+ <description>Compress the output of a MapReduce job</description>
+ </property>
- <property>
- <name>mapred.output.compression.codec</name>
- <value>org.apache.hadoop.io.compress.SnappyCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
+ <property>
+ <name>mapred.output.compression.codec</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <description>The compression codec to use for job outputs
+ </description>
+ </property>
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
--->
+ <property>
+ <name>mapred.output.compression.type</name>
+ <value>BLOCK</value>
+ <description>The compression type to use for job outputs</description>
+ </property>
+
+ !-->
<property>
<name>mapreduce.job.max.split.locations</name>
@@ -76,5 +71,4 @@
<value>2</value>
<description>Block replication</description>
</property>
-
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index 18f6feb..ff1c7eb 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -18,7 +18,7 @@
<property>
<name>io.sort.mb</name>
- <value>128</value>
+ <value>64</value>
</property>
<property>
@@ -28,12 +28,12 @@
<property>
<name>mapred.job.map.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
<name>mapred.job.reduce.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -58,7 +58,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
@@ -81,10 +81,9 @@
<value>false</value>
</property>
- <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
<property>
<name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>1</value>
+ <value>0.05</value>
</property>
<property>
@@ -114,7 +113,7 @@
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -124,7 +123,7 @@
<property>
<name>mapreduce.map.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -169,7 +168,7 @@
<property>
<name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
@@ -219,7 +218,7 @@
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>128</value>
+ <value>64</value>
</property>
<property>
@@ -234,7 +233,7 @@
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -244,7 +243,7 @@
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
- <value>512</value>
+ <value>250</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 3d60a3c..0910df5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -35,8 +35,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -55,6 +54,7 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.job.manager.ExecutableManager;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.StorageCleanupJob;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
@@ -419,10 +419,10 @@ public class BuildCubeWithEngine {
}
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
- String tableName = segment.getStorageLocationIdentifier();
- try (HTable table = new HTable(conf, tableName)) {
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
+ String tableName = segment.getStorageLocationIdentifier();
+
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
@@ -448,5 +448,4 @@ public class BuildCubeWithEngine {
}
}
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d5925b9..93b6b46 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,20 +46,20 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.6.0</hadoop2.version>
- <yarn.version>2.6.0</yarn.version>
+ <hadoop2.version>2.7.1</hadoop2.version>
+ <yarn.version>2.7.1</yarn.version>
<!-- Hive versions -->
- <hive.version>0.14.0</hive.version>
- <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
+ <hive.version>1.2.1</hive.version>
+ <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
<kafka.version>0.8.1</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.6.0</curator.version>
+ <curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index 38f299e..bfb5fe4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
/**
*/
@@ -37,6 +37,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- HTableInterface getTable(String tableName) throws IOException;
+ Table getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index d9326f5..cc76b87 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
+ private Table mockedAclTable;
+ private Table mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index d0aa0ed..972eea9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements HTableInterface {
+public class MockHTable implements Table {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,14 +114,6 @@ public class MockHTable implements HTableInterface {
this.columnFamilies.add(columnFamily);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[] getTableName() {
- return tableName.getBytes();
- }
-
@Override
public TableName getName() {
return null;
@@ -200,8 +192,8 @@ public class MockHTable implements HTableInterface {
}
@Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- return new Boolean[0];
+ public boolean[] existsAll(List<Get> list) throws IOException {
+ return new boolean[0];
}
/**
@@ -306,15 +298,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- // FIXME: implement
- return null;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -446,7 +429,7 @@ public class MockHTable implements HTableInterface {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow(nkvs);
+ filter.filterRow();
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -535,6 +518,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -555,7 +543,7 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDeleteFamily()) {
+ if (kv.isDelete()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -592,6 +580,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -605,7 +598,7 @@ public class MockHTable implements HTableInterface {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, true);
+ return incrementColumnValue(row, family, qualifier, amount, null);
}
@Override
@@ -617,37 +610,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if (check(row, family, qualifier, null)) {
- Put put = new Put(row);
- put.add(family, qualifier, Bytes.toBytes(amount));
- put(put);
- return amount;
- }
- long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
- data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
- return newValue;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isAutoFlush() {
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void flushCommits() throws IOException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public void close() throws IOException {
}
@@ -673,29 +635,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public void setAutoFlush(boolean autoFlush) {
- throw new NotImplementedException();
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- throw new NotImplementedException();
-
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- throw new NotImplementedException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index ab18029..d55edc3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,7 +21,8 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -57,11 +58,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index d693a67..3e3efec 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- HTableInterface htable = null;
+ Table htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,17 +226,16 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -250,7 +249,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -266,7 +265,6 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -284,7 +282,7 @@ public class AclService implements MutableAclService {
throw e;
}
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -295,17 +293,16 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 4cd527c..945a111 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -28,8 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.WeakHashMap;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -424,35 +423,24 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- HTable table = null;
+ Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- try {
- table = new HTable(hconf, tableName);
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
-
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
-
- regionCount = sizeMap.size();
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
- } finally {
- if (null != table) {
- table.close();
- }
+ for (long s : sizeMap.values()) {
+ tableSize += s;
}
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 3acaeb8..96661ac 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -45,11 +45,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
@@ -137,14 +137,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -170,14 +169,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -189,12 +187,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- HTableInterface htable = null;
+ Table htable = null;
try {
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = conn.getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 07c7c6f..ab54882 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
- put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,14 +163,13 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -185,7 +184,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -216,7 +215,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- HTableInterface htable = null;
+ Table htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..b769391 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -64,7 +64,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
+ private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -75,7 +75,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (HConnection conn : connPool.values()) {
+ for (Connection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -144,7 +144,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -213,9 +213,9 @@ public class HBaseConnection {
// ============================================================================
- // returned HConnection can be shared by multiple threads and does not require close()
+ // returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static HConnection get(String url) {
+ public static Connection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -223,13 +223,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- HConnection connection = connPool.get(url);
+ Connection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = HConnectionManager.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
connPool.put(url, connection);
}
@@ -248,8 +248,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static boolean tableExists(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -269,18 +269,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
+ public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
+ Admin hbase = conn.getAdmin();
+ TableName tableName = TableName.valueOf(table);
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(table, newFamilyDescriptor(family));
+ hbase.addColumn(tableName, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -333,8 +333,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void deleteTable(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (!tableExists(conn, tableName)) {
@@ -344,10 +344,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(tableName)) {
- hbase.disableTable(tableName);
+ if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
+ hbase.disableTable(TableName.valueOf(tableName));
}
- hbase.deleteTable(tableName);
+ hbase.deleteTable(TableName.valueOf(tableName));
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index e2f3661..50524b6 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -31,14 +31,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +70,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- private HConnection getConnection() throws IOException {
+ private Connection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -120,7 +121,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,13 +239,12 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -252,7 +252,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -265,8 +265,6 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
- table.flushCommits();
-
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -275,7 +273,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -288,7 +286,6 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
- table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -310,7 +307,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -318,7 +315,7 @@ public class HBaseResourceStore extends ResourceStore {
}
}
- private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -337,7 +334,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
@@ -363,7 +360,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
int kvSizeLimit = this.kylinConfig.getHBaseKeyValueSize();
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -371,8 +368,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.addColumn(B_FAMILY, B_COLUMN, content);
+ put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index f4dfd2b..3d82105 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -18,7 +18,6 @@
package org.apache.kylin.storage.hbase;
-import com.google.common.base.Preconditions;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.cube.CubeInstance;
@@ -36,6 +35,8 @@ import org.apache.kylin.storage.IStorageQuery;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput;
import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition;
+import com.google.common.base.Preconditions;
+
@SuppressWarnings("unused")
//used by reflection
public class HBaseStorage implements IStorage {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b141190..f63d9c2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,12 +26,13 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -86,14 +87,13 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final HTableInterface table;
+ final BufferedMutator table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getTable(htableName);
- table.setAutoFlush(false, true);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getBufferedMutator(htableName);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.put(put);
+ table.mutate(put);
}
@Override
public void close() throws IOException {
- table.flushCommits();
+ table.flush();
table.close();
}
}
class Reader implements IGTScanner {
- final HTableInterface table;
+ final Table table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 8ac3832..982a044 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -25,11 +25,12 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FuzzyRowFilter;
@@ -70,7 +71,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
protected final List<RowValueDecoder> rowValueDecoders;
private final StorageContext context;
private final String tableName;
- private final HTableInterface table;
+ private final Table table;
protected CubeTupleConverter tupleConverter;
protected final Iterator<HBaseKeyRange> rangeIterator;
@@ -88,7 +89,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
private int advMeasureRowsRemaining;
private int advMeasureRowIndex;
- public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, HConnection conn, //
+ public CubeSegmentTupleIterator(CubeSegment cubeSeg, List<HBaseKeyRange> keyRanges, Connection conn, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, //
List<RowValueDecoder> rowValueDecoders, StorageContext context, TupleInfo returnTupleInfo) {
this.cubeSeg = cubeSeg;
@@ -108,7 +109,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
this.rangeIterator = keyRanges.iterator();
try {
- this.table = conn.getTable(tableName);
+ this.table = conn.getTable(TableName.valueOf(tableName));
} catch (Throwable t) {
throw new StorageException("Error when open connection to table " + tableName, t);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index ff729f4..1944327 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -33,7 +33,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.Dictionary;
@@ -46,10 +46,10 @@ import org.apache.kylin.cube.RawQueryLastHacker;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseMappingDesc;
import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.CubeDesc.DeriveInfo;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
@@ -152,7 +152,7 @@ public class CubeStorageQuery implements IStorageQuery {
setCoprocessor(groupsCopD, valueDecoders, context); // enable coprocessor if beneficial
setLimit(filter, context);
- HConnection conn = HBaseConnection.get(context.getConnUrl());
+ Connection conn = HBaseConnection.get(context.getConnUrl());
// notice we're passing filterD down to storage instead of flatFilter
return new SerializedHBaseTupleIterator(conn, scans, cubeInstance, dimensionsD, filterD, groupsCopD, valueDecoders, context, returnTupleInfo);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
index 6342c5c..9a171e9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/RegionScannerAdapter.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
/**
* @author yangli9
@@ -50,7 +51,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -60,7 +61,7 @@ public class RegionScannerAdapter implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -94,4 +95,9 @@ public class RegionScannerAdapter implements RegionScanner {
return Long.MAX_VALUE;
}
+ @Override
+ public int getBatch() {
+ return -1;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
index e8dd5b9..d033c77 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/SerializedHBaseTupleIterator.java
@@ -25,7 +25,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -57,7 +57,7 @@ public class SerializedHBaseTupleIterator implements ITupleIterator {
private int scanCount;
private ITuple next;
- public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
+ public SerializedHBaseTupleIterator(Connection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, //
Set<TblColRef> dimensions, TupleFilter filter, Set<TblColRef> groupBy, List<RowValueDecoder> rowValueDecoders, //
StorageContext context, TupleInfo returnTupleInfo) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
index c7b650a..8dba1b1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregateRegionObserver.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
@@ -99,7 +99,7 @@ public class AggregateRegionObserver extends BaseRegionObserver {
// start/end region operation & sync on scanner is suggested by the
// javadoc of RegionScanner.nextRaw()
// FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
- HRegion region = ctxt.getEnvironment().getRegion();
+ Region region = ctxt.getEnvironment().getRegion();
region.startRegionOperation();
try {
synchronized (innerScanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
index be26142..5857435 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/AggregationScanner.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
@@ -116,8 +117,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
- return outerScanner.next(result, limit);
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.next(result, scannerContext);
}
@Override
@@ -126,8 +127,8 @@ public class AggregationScanner implements RegionScanner {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
- return outerScanner.nextRaw(result, limit);
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
+ return outerScanner.nextRaw(result, scannerContext);
}
@Override
@@ -160,6 +161,11 @@ public class AggregationScanner implements RegionScanner {
return outerScanner.getMvccReadPoint();
}
+ @Override
+ public int getBatch() {
+ return outerScanner.getBatch();
+ }
+
private static class Stats {
long inputRows = 0;
long inputBytes = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
index 8404262..331e34d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverAggregationCache.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScannerContext;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.storage.hbase.common.coprocessor.AggrKey;
import org.apache.kylin.storage.hbase.common.coprocessor.AggregationCache;
@@ -112,7 +113,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean next(List<Cell> result, int limit) throws IOException {
+ public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -122,7 +123,7 @@ public class ObserverAggregationCache extends AggregationCache {
}
@Override
- public boolean nextRaw(List<Cell> result, int limit) throws IOException {
+ public boolean nextRaw(List<Cell> result, ScannerContext scannerContext) throws IOException {
return next(result);
}
@@ -161,6 +162,11 @@ public class ObserverAggregationCache extends AggregationCache {
// AggregateRegionObserver.LOG.info("Kylin Scanner getMvccReadPoint()");
return Long.MAX_VALUE;
}
+
+ @Override
+ public int getBatch() {
+ return innerScanner.getBatch();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
index f0e9bed..e02ba1c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/ObserverEnabler.java
@@ -23,9 +23,9 @@ import java.util.Collection;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
@@ -60,7 +60,7 @@ public class ObserverEnabler {
static final Map<String, Boolean> CUBE_OVERRIDES = Maps.newConcurrentMap();
public static ResultScanner scanWithCoprocessorIfBeneficial(CubeSegment segment, Cuboid cuboid, TupleFilter tupleFiler, //
- Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, HTableInterface table, Scan scan) throws IOException {
+ Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context, Table table, Scan scan) throws IOException {
if (context.isCoprocessorEnabled() == false) {
return table.getScanner(scan);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 07a3cc3..c42cd05 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,8 +26,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -50,10 +51,10 @@ import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorBehavior;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -120,7 +121,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -198,7 +199,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final boolean[] abnormalFinish = new boolean[1];
try {
- HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
+ Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index a359d19..a940dfc 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -24,11 +24,12 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -164,8 +165,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
index 7d48c1a..be88c47 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/ExpectedSizeIterator.java
@@ -18,6 +18,11 @@
package org.apache.kylin.storage.hbase.cube.v2;
+import java.util.Iterator;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
@@ -27,11 +32,6 @@ import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Iterator;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
-
class ExpectedSizeIterator implements Iterator<byte[]> {
private static final Logger logger = LoggerFactory.getLogger(ExpectedSizeIterator.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6ee0e7c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index b29d0d1..2d449d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -145,7 +145,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -181,7 +181,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try {
this.serviceStartTime = System.currentTimeMillis();
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
[24/50] [abbrv] kylin git commit: minor remove googleapi fonts from
dependency
Posted by li...@apache.org.
minor remove googleapi fonts from dependency
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/27426aef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/27426aef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/27426aef
Branch: refs/heads/1.5.x-CDH5.7
Commit: 27426aef28f57813db90c6713ea7fd21f0151cbe
Parents: fdad2d8
Author: Jason <ji...@163.com>
Authored: Tue Aug 30 11:32:19 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Tue Aug 30 11:32:19 2016 +0800
----------------------------------------------------------------------
pom.xml | 1 +
webapp/app/css/messenger-theme-ice.css | 125 ++++++++++++++++++++++++++++
webapp/app/index.html | 2 +-
webapp/grunt.json | 2 +-
4 files changed, 128 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/27426aef/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ea29958..d5925b9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1186,6 +1186,7 @@
<!-- MIT license -->
<exclude>webapp/app/css/AdminLTE.css</exclude>
+ <exclude>webapp/app/css/messenger-theme-ice.css</exclude>
<exclude>webapp/app/js/directives/kylin_abn_tree_directive.js</exclude>
<exclude>webapp/app/js/directives/angular-tree-control.js</exclude>
<exclude>webapp/app/js/directives/datetimepicker.js</exclude>
http://git-wip-us.apache.org/repos/asf/kylin/blob/27426aef/webapp/app/css/messenger-theme-ice.css
----------------------------------------------------------------------
diff --git a/webapp/app/css/messenger-theme-ice.css b/webapp/app/css/messenger-theme-ice.css
new file mode 100644
index 0000000..faf5f66
--- /dev/null
+++ b/webapp/app/css/messenger-theme-ice.css
@@ -0,0 +1,125 @@
+/*!
+Copyright (c) 2013 HubSpot, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+*/
+
+/* line 12, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice {
+ -moz-user-select: none;
+ -webkit-user-select: none;
+ -o-user-select: none;
+ user-select: none;
+ font-family: "Raleway", sans-serif;
+}
+/* line 16, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message {
+ -webkit-border-radius: 5px;
+ -moz-border-radius: 5px;
+ -ms-border-radius: 5px;
+ -o-border-radius: 5px;
+ border-radius: 5px;
+ -webkit-box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.14), 0 4px #aaaaaa, 0 5px rgba(0, 0, 0, 0.05);
+ -moz-box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.14), 0 4px #aaaaaa, 0 5px rgba(0, 0, 0, 0.05);
+ box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.14), 0 4px #aaaaaa, 0 5px rgba(0, 0, 0, 0.05);
+ border: 0px;
+ background-color: #f6f6f6;
+ position: relative;
+ margin-bottom: 1.5em;
+ font-size: 13px;
+ color: #666666;
+ font-weight: 500;
+ padding: 12px 22px;
+}
+/* line 28, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message .messenger-close {
+ position: absolute;
+ top: 0px;
+ right: 0px;
+ color: #888888;
+ opacity: 1;
+ font-weight: bold;
+ display: block;
+ font-size: 20px;
+ line-height: 20px;
+ padding: 8px 10px 7px 7px;
+ cursor: pointer;
+ background: transparent;
+ border: 0;
+ -webkit-appearance: none;
+}
+/* line 44, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message .messenger-close:hover {
+ color: #444444;
+}
+/* line 47, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message .messenger-close:active {
+ color: #222222;
+}
+/* line 50, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message .messenger-actions {
+ float: none;
+ margin-top: 10px;
+}
+/* line 54, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message .messenger-actions a {
+ -webkit-box-shadow: 0px 0px 0px 1px rgba(0, 0, 0, 0.1), inset 0px 1px rgba(255, 255, 255, 0.05);
+ -moz-box-shadow: 0px 0px 0px 1px rgba(0, 0, 0, 0.1), inset 0px 1px rgba(255, 255, 255, 0.05);
+ box-shadow: 0px 0px 0px 1px rgba(0, 0, 0, 0.1), inset 0px 1px rgba(255, 255, 255, 0.05);
+ -webkit-border-radius: 4px;
+ -moz-border-radius: 4px;
+ -ms-border-radius: 4px;
+ -o-border-radius: 4px;
+ border-radius: 4px;
+ position: relative;
+ text-decoration: none;
+ display: inline-block;
+ padding: 10px;
+ color: #888888;
+ margin-right: 10px;
+ padding: 3px 10px 5px;
+ text-transform: capitalize;
+}
+/* line 66, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message .messenger-actions a:hover, ul.messenger-theme-ice .messenger-message .messenger-actions a:active {
+ -webkit-box-shadow: 0px 0px 0px 1px rgba(0, 0, 0, 0.1), inset 0px 1px rgba(255, 255, 255, 0.15), 0 2px #aaaaaa;
+ -moz-box-shadow: 0px 0px 0px 1px rgba(0, 0, 0, 0.1), inset 0px 1px rgba(255, 255, 255, 0.15), 0 2px #aaaaaa;
+ box-shadow: 0px 0px 0px 1px rgba(0, 0, 0, 0.1), inset 0px 1px rgba(255, 255, 255, 0.15), 0 2px #aaaaaa;
+ color: #444444;
+}
+/* line 70, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message .messenger-actions a:active {
+ -webkit-box-shadow: 0px 0px 0px 1px rgba(0, 0, 0, 0.1), inset 0px 1px rgba(255, 255, 255, 0.15), 0 1px #aaaaaa;
+ -moz-box-shadow: 0px 0px 0px 1px rgba(0, 0, 0, 0.1), inset 0px 1px rgba(255, 255, 255, 0.15), 0 1px #aaaaaa;
+ box-shadow: 0px 0px 0px 1px rgba(0, 0, 0, 0.1), inset 0px 1px rgba(255, 255, 255, 0.15), 0 1px #aaaaaa;
+ top: 1px;
+}
+/* line 74, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message .messenger-actions .messenger-phrase {
+ display: none;
+}
+/* line 77, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message .messenger-message-inner:before {
+ display: block;
+ z-index: 20;
+ font-weight: bold;
+ margin-bottom: 2px;
+}
+/* line 84, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message.alert-success .messenger-message-inner:before {
+ content: "Success";
+}
+/* line 88, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message.alert-error .messenger-message-inner:before {
+ content: "Error";
+}
+/* line 92, ../../src/sass/messenger-theme-ice.sass */
+ul.messenger-theme-ice .messenger-message.alert-info .messenger-message-inner:before {
+ content: "Information";
+}
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/27426aef/webapp/app/index.html
----------------------------------------------------------------------
diff --git a/webapp/app/index.html b/webapp/app/index.html
index 282567c..8fb2cb9 100644
--- a/webapp/app/index.html
+++ b/webapp/app/index.html
@@ -34,7 +34,7 @@
<link rel="stylesheet" type="text/css" href="components/angular-tree-control/css/tree-control.css">
<link rel="stylesheet" type="text/css" href="components/angular-tree-control/css/tree-control-attribute.css">
<link rel="stylesheet" type="text/css" href="components/messenger/build/css/messenger.css">
- <link rel="stylesheet" type="text/css" href="components/messenger/build/css/messenger-theme-ice.css">
+ <link rel="stylesheet" type="text/css" href="css/messenger-theme-ice.css">
<link rel="stylesheet" type="text/css" href="components/ng-grid/ng-grid.css">
<link rel="stylesheet" type="text/css" href="components/angular-ui-grid/ui-grid.min.css">
<link rel="stylesheet" type="text/css" href="components/chosen/chosen.css">
http://git-wip-us.apache.org/repos/asf/kylin/blob/27426aef/webapp/grunt.json
----------------------------------------------------------------------
diff --git a/webapp/grunt.json b/webapp/grunt.json
index 3c2cbf8..d0ee545 100755
--- a/webapp/grunt.json
+++ b/webapp/grunt.json
@@ -62,7 +62,7 @@
"app/components/angular-ui-grid/ui-grid.min.css",
"app/components/animate.css/animate.min.css",
"app/components/messenger/build/css/messenger.css",
- "app/components/messenger/build/css/messenger-theme-ice.css",
+ "app/css/messenger-theme-ice.css",
"app/components/chosen/chosen.css",
"app/components/angular-chosen-localytics/chosen-spinner.css",
"app/components/nvd3/nv.d3.min.css",
[46/50] [abbrv] kylin git commit: KYLIN-1767 td witdh update
Posted by li...@apache.org.
KYLIN-1767 td witdh update
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7dae9778
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7dae9778
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7dae9778
Branch: refs/heads/1.5.x-CDH5.7
Commit: 7dae9778c1f360261b465e38db47e954cf226590
Parents: 9e2970b
Author: Jason <ji...@163.com>
Authored: Fri Sep 2 19:56:47 2016 +0800
Committer: Jason <ji...@163.com>
Committed: Fri Sep 2 19:57:40 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/cubeDesigner/measures.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7dae9778/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 065b735..56c86be 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -251,7 +251,7 @@
<td>
<select class="form-control" chosen ng-if="nextPara.type !== 'constant'" required
ng-model="groupby_column.name"
- ng-options="column as column for column in getGroupBYColumns()" style="width:323px;">
+ ng-options="column as column for column in getGroupBYColumns()" style="width:200px;">
<option value="">--Select A Column--</option>
</select>
</td>
[45/50] [abbrv] kylin git commit: KYLIN-1767-WEB-UI-FOR-TOPN
Posted by li...@apache.org.
KYLIN-1767-WEB-UI-FOR-TOPN
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e2970bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e2970bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e2970bc
Branch: refs/heads/1.5.x-CDH5.7
Commit: 9e2970bc4a87d3848ba6677fd2d05c9ed86e15d9
Parents: 111e984
Author: zx chen <34...@qq.com>
Authored: Thu Sep 1 15:16:30 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Sep 2 19:42:22 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 25 ++++
webapp/app/js/controllers/cubeMeasures.js | 130 +++++++++++++++++++-
webapp/app/js/directives/directives.js | 29 ++++-
webapp/app/js/model/cubeDescModel.js | 6 +-
webapp/app/partials/cubeDesigner/measures.html | 77 ++++++++++--
5 files changed, 250 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e2970bc/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index b620d7f..e2d0ab5 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -98,6 +98,31 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
return me_columns;
};
+ $scope.getGroupBYColumns = function () {
+ //metric from model
+ var me_columns = [];
+ var table_columns=[];
+ var groupby_columns=[];
+ var tableColumns = $scope.getColumnsByTable($scope.metaModel.model.fact_table);
+ if($scope.metaModel.model.metrics){
+ angular.forEach($scope.metaModel.model.metrics,function(metric,index){
+ me_columns.push(metric);
+ });
+ }
+ angular.forEach($scope.cubeMetaFrame.dimensions,function(dimension){
+ if(dimension.table==$scope.metaModel.model.fact_table) {
+ table_columns.push(dimension.column);
+ }
+ });
+ angular.forEach(me_columns,function(column){
+ if(table_columns.indexOf(column)==-1) {
+ groupby_columns.push(column);
+ }
+ });
+
+ return groupby_columns;
+ };
+
$scope.getExtendedHostColumn = function(){
var me_columns = [];
//add cube dimension column for specific measure
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e2970bc/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 9dec379..794d2b2 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -19,7 +19,9 @@
'use strict';
KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubesManager,CubeDescModel,SweetAlert) {
-
+ $scope.num=0;
+ $scope.convertedColumns=[];
+ $scope.groupby=[];
$scope.initUpdateMeasureStatus = function(){
$scope.updateMeasureStatus = {
isEdit:false,
@@ -38,6 +40,31 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
if(!!measure && measure.function.parameter.next_parameter){
$scope.nextPara.value = measure.function.parameter.next_parameter.value;
}
+ if($scope.newMeasure.function.expression=="TOP_N"){
+ $scope.convertedColumns=[];
+ for(var configuration in $scope.newMeasure.function.configuration) {
+ var _name=configuration.slice(14);
+ var item=$scope.newMeasure.function.configuration[configuration];
+ var _isFixedLength = item.substring(0,12) === "fixed_length"?"true":"false";//fixed_length:12
+ var _isIntLength = item.substring(0,3) === "int"?"true":"false";//fixed_length:12
+ var _encoding = item;
+ var _valueLength = 0 ;
+ if(_isFixedLength !=="false"){
+ _valueLength = item.substring(13,item.length);
+ _encoding = "fixed_length";
+ }
+ if(_isIntLength!="false"){
+ _valueLength = item.substring(4,item.length);
+ _encoding = "int";
+ }
+ $scope.GroupBy = {
+ name:_name,
+ encoding:_encoding,
+ valueLength:_valueLength,
+ }
+ $scope.convertedColumns.push($scope.GroupBy);
+ };
+ }
};
@@ -103,9 +130,52 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
$scope.saveNewMeasure = function () {
- if ($scope.newMeasure.function.expression === 'TOP_N' && $scope.nextPara.value == "") {
- SweetAlert.swal('', '[TOP_N] Group by Column is required', 'warning');
- return false;
+ if ($scope.newMeasure.function.expression === 'TOP_N' ) {
+ if($scope.newMeasure.function.parameter.value == ""){
+ SweetAlert.swal('', '[TOP_N] ORDER|SUM by Column is required', 'warning');
+ return false;
+ }
+ if($scope.convertedColumns.length<1){
+ SweetAlert.swal('', '[TOP_N] Group by Column is required', 'warning');
+ return false;
+ }
+
+ var hasExisted = [];
+
+ for (var column in $scope.convertedColumns){
+ if(hasExisted.indexOf($scope.convertedColumns[column].name)==-1){
+ hasExisted.push($scope.convertedColumns[column].name);
+ }
+ else{
+ SweetAlert.swal('', 'The column named ['+$scope.convertedColumns[column].name+'] already exits.', 'warning');
+ return false;
+ }
+ if ($scope.convertedColumns[column].encoding == 'int' && ($scope.convertedColumns[column].valueLength < 1 || $scope.convertedColumns[column].valueLength > 8)) {
+ SweetAlert.swal('', 'int encoding column length should between 1 and 8.', 'warning');
+ return false;
+ }
+ }
+ $scope.nextPara.next_parameter={};
+ $scope.newMeasure.function.configuration={};
+ $scope.groupby($scope.nextPara);
+ angular.forEach($scope.convertedColumns,function(item){
+ var a='topn.encoding.'+item.name;
+ var encoding="";
+ if(item.encoding!=="dict" && item.encoding!=="date"&& item.encoding!=="time"){
+ if(item.encoding=="fixed_length" && item.valueLength){
+ encoding = "fixed_length:"+item.valueLength;
+ }
+ else if(item.encoding=="int" && item.valueLength){
+ encoding = "int:"+item.valueLength;
+ }else{
+ encoding = item.encoding;
+ }
+ }else{
+ encoding = item.encoding;
+ item.valueLength=0;
+ }
+ $scope.newMeasure.function.configuration[a]=encoding;
+ });
}
if ($scope.isNameDuplicated($scope.cubeMetaFrame.measures, $scope.newMeasure) == true) {
@@ -143,17 +213,67 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
$scope.nextPara = {
"type":"column",
"value":"",
- "next_parameter":null
+ "next_parameter":{}
}
if($scope.newMeasure){
$scope.newMeasure.function.parameter.next_parameter = null;
}
}
+ $scope.addNewGroupByColumn = function () {
+ $scope.nextGroupBy = {
+ name:null,
+ encoding:"dict",
+ valueLength:0,
+ }
+ $scope.convertedColumns.push($scope.nextGroupBy);
+
+ };
+
+ $scope.removeColumn = function(arr,index){
+ if (index > -1) {
+ arr.splice(index, 1);
+ }
+ };
+ $scope.refreshGroupBy=function (list,index,item) {
+ var encoding;
+ var name = item.name;
+ if(item.encoding=="dict" || item.encoding=="date"|| item.encoding=="time"){
+ item.valueLength=0;
+ }
+ };
+
+ $scope.groupby= function (next_parameter){
+ if($scope.num<$scope.convertedColumns.length-1){
+ next_parameter.value=$scope.convertedColumns[$scope.num].name;
+ next_parameter.type="column";
+ next_parameter.next_parameter={};
+ $scope.num++;
+ $scope.groupby(next_parameter.next_parameter);
+ }
+ else{
+ next_parameter.value=$scope.convertedColumns[$scope.num].name;
+ next_parameter.type="column";
+ next_parameter.next_parameter=null;
+ $scope.num=0;
+ return false;
+ }
+ }
+ $scope.converted = function (next_parameter) {
+ if (next_parameter != null) {
+ $scope.groupby.push(next_parameter.value);
+ converted(next_parameter.next_parameter)
+ }
+ else {
+ $scope.groupby.push(next_parameter.value);
+ return false;
+ }
+ }
//map right return type for param
$scope.measureReturnTypeUpdate = function(){
if($scope.newMeasure.function.expression == 'TOP_N'){
+ $scope.convertedColumns=[];
$scope.newMeasure.function.parameter.type= 'column';
$scope.newMeasure.function.returntype = "topn(100)";
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e2970bc/webapp/app/js/directives/directives.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/directives/directives.js b/webapp/app/js/directives/directives.js
index 4b83865..f5051e8 100644
--- a/webapp/app/js/directives/directives.js
+++ b/webapp/app/js/directives/directives.js
@@ -295,7 +295,30 @@ KylinApp.directive('kylinPagination', function ($parse, $q) {
},
template:
'<li class="parent_li">Value:<b>{{nextpara.value}}</b>, Type:<b>{{ nextpara.type }}</b></li>' +
- '<parametertree ng-if="nextpara.next_parameter!=null" nextpara="nextpara.next_parameter"></parameterTree>',
+ '<parametertree ng-if="nextpara.next_parameter!=null" nextpara="nextpara.next_parameter"></parameterTree>',
+ compile: function(tElement, tAttr, transclude) {
+ var contents = tElement.contents().remove();
+ var compiledContents;
+ return function(scope, iElement, iAttr) {
+ if(!compiledContents) {
+ compiledContents = $compile(contents, transclude);
+ }
+ compiledContents(scope, function(clone, scope) {
+ iElement.append(clone);
+ });
+ };
+ }
+ };
+ }).directive("groupbytree", function($compile) {
+ return {
+ restrict: "E",
+ transclude: true,
+ scope: {
+ nextpara: '=',
+ },
+ template:
+ '<b>{{nextpara.value}}<b ng-if="nextpara.next_parameter!=null">,</b></b>'+
+ '<groupbytree ng-if="nextpara.next_parameter!=null" nextpara="nextpara.next_parameter"></groupbytree>',
compile: function(tElement, tAttr, transclude) {
var contents = tElement.contents().remove();
var compiledContents;
@@ -318,7 +341,9 @@ KylinApp.directive('kylinPagination', function ($parse, $q) {
},
template:
'<li class="parent_li">SUM|ORDER BY:<b>{{nextpara.value}}</b></b></li>' +
- '<li class="parent_li">GROUP BY:<b>{{nextpara.next_parameter.value}}</b></li>',
+ '<li class="parent_li">Group By:'+
+ '<groupbytree nextpara="nextpara.next_parameter"></groupbytree>'+
+ '</li>',
compile: function(tElement, tAttr, transclude) {
var contents = tElement.contents().remove();
var compiledContents;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e2970bc/webapp/app/js/model/cubeDescModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeDescModel.js b/webapp/app/js/model/cubeDescModel.js
index 57d7e6e..c981249 100644
--- a/webapp/app/js/model/cubeDescModel.js
+++ b/webapp/app/js/model/cubeDescModel.js
@@ -37,7 +37,8 @@ KylinApp.service('CubeDescModel', function () {
"type": "constant",
"value": "1",
"next_parameter":null
- }
+ },
+ "configuration":{}
}
}
],
@@ -74,7 +75,8 @@ KylinApp.service('CubeDescModel', function () {
"type": "",
"value": "",
"next_parameter":null
- }
+ },
+ "configuration":null
}
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e2970bc/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 53806b4..065b735 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -218,19 +218,80 @@
</div>
</div>
</div>
-
- <div class="form-group" ng-if="newMeasure.function.expression == 'TOP_N'">
+ <!--Group by Column-->
+ <div class="form-group" ng-if="newMeasure.function.expression == 'TOP_N'" >
<div class="row">
<label class="col-xs-12 col-sm-3 control-label no-padding-right font-color-default">
<b>Group by Column</b>
</label>
- <div class="col-xs-12 col-sm-6">
- <select class="form-control" chosen ng-if="nextPara.type !== 'constant'" required
- ng-model="nextPara.value"
- ng-options="column as column for column in getCommonMetricColumns()" >
- <option value=""></option>
- </select>
+ <div class="form-group large-popover" >
+ <div class="box-body">
+ <table style="margin-left:width:92%"
+ class="table table-hover table-bordered list">
+ <thead>
+ <tr>
+ <th>ID</th>
+ <th>Column</th>
+ <th>Encoding</th>
+ <th>Length</th>
+ <th ng-if="state.mode=='edit'"></th>
+ </tr>
+ </thead>
+
+ <tbody>
+
+ <tr ng-repeat="groupby_column in convertedColumns track by $index"
+ ng-class="state.mode=='edit'?'sort-item':''">
+
+ <td>
+ <!-- ID -->
+ <span class="ng-binding" ng-class="state.mode=='edit'?'badge':''">{{($index + 1)}}</span>
+ </td>
+ <!--Column Name -->
+ <td>
+ <select class="form-control" chosen ng-if="nextPara.type !== 'constant'" required
+ ng-model="groupby_column.name"
+ ng-options="column as column for column in getGroupBYColumns()" style="width:323px;">
+ <option value="">--Select A Column--</option>
+ </select>
+ </td>
+ <!--Column Encoding -->
+ <td>
+ <select ng-if="state.mode=='edit'" style="width:180px;"
+ chosen ng-model="groupby_column.encoding"
+ ng-change="refreshGroupBy(convertedColumns,$index,groupby_column)"
+ ng-options="dt as dt for dt in store.supportedEncoding">
+ <option value=""></option>
+ </select>
+ <span ng-if="state.mode=='view'">{{groupby_column.encoding}}</span>
+
+ </td>
+ <td>
+ <!--Column Length -->
+ <input type="text" class="form-control" placeholder="Column Length.." ng-if="state.mode=='edit'"
+ tooltip="group by column length.." tooltip-trigger="focus"
+ ng-change="refreshGroupBy(convertedColumns,$index,groupby_column);"
+ ng-disabled="groupby_column.encoding=='dict'||groupby_column.encoding=='date'||groupby_column.encoding=='time'"
+ ng-model="groupby_column.valueLength" class="form-control">
+
+ <small class="help-block red" ng-show="state.mode=='edit' && groupby_column.encoding=='int' && (groupby_column.valueLength>8 || groupby_column.valueLength<1)">int encoding column length should between 1 and 8</small>
+ <span ng-if="state.mode=='view'">{{groupby_column.valueLength}}</span>
+ </td>
+ <td ng-if="state.mode=='edit'">
+ <button class="btn btn-xs btn-info"
+ ng-click="removeColumn(convertedColumns, $index)"><i
+ class="fa fa-minus"></i>
+ </button>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ <button class="btn btn-sm btn-info" style="margin-left:10px"
+ ng-click="addNewGroupByColumn()" ng-show="state.mode=='edit'">New Column<i class="fa fa-plus"></i>
+ </button>
</div>
+
</div>
</div>
[23/50] [abbrv] kylin git commit: KYLIN-1954 refind and fix CI
Posted by li...@apache.org.
KYLIN-1954 refind and fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdad2d8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdad2d8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdad2d8d
Branch: refs/heads/1.5.x-CDH5.7
Commit: fdad2d8d3ff34b069f646794226bcb73543a9474
Parents: cc9acbc
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Aug 30 09:55:57 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Aug 30 11:05:26 2016 +0800
----------------------------------------------------------------------
.../kylin/job/dataGen/FactTableGenerator.java | 10 +++--
.../kylin/dict/BuildInFunctionTransformer.java | 5 ++-
.../localmeta/data/data_gen_config.json | 2 +-
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 4 +-
.../apache/kylin/query/ITCombinationTest.java | 14 ++++--
.../apache/kylin/query/ITKylinQueryTest.java | 13 +++++-
.../test/resources/query/sql_like/query01.sql | 31 +++++++++++++
.../test/resources/query/sql_like/query02.sql | 22 +++++++++
.../apache/kylin/query/routing/RoutingRule.java | 2 +
.../rules/RemoveBlackoutRealizationsRule.java | 47 ++++++++++++++++++++
10 files changed, 136 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 9373a02..5aac4de 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -586,10 +586,12 @@ public class FactTableGenerator {
}
for (Integer index : differentiateColumns) {
- if (currentRowTime >= differentiateBoundary) {
- columnValues.set(index, columnValues.get(index) + "_B");
- } else {
- columnValues.set(index, columnValues.get(index) + "_A");
+ if (r.nextBoolean()) {//only change half of data
+ if (currentRowTime >= differentiateBoundary) {
+ columnValues.set(index, columnValues.get(index) + "_B");
+ } else {
+ columnValues.set(index, columnValues.get(index) + "_A");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
index 9156b67..665a7e6 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuildInFunctionTransformer.java
@@ -70,8 +70,11 @@ public class BuildInFunctionTransformer implements ITupleFilterTransformer {
ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
while (childIterator.hasNext()) {
TupleFilter transformed = transform(childIterator.next());
- if (transformed != null)
+ if (transformed != null) {
childIterator.set(transformed);
+ } else {
+ throw new IllegalStateException("Should not be null");
+ }
}
}
return translated == null ? tupleFilter : translated;
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/examples/test_case_data/localmeta/data/data_gen_config.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/data_gen_config.json b/examples/test_case_data/localmeta/data/data_gen_config.json
index f730058..16ba3d5 100644
--- a/examples/test_case_data/localmeta/data/data_gen_config.json
+++ b/examples/test_case_data/localmeta/data/data_gen_config.json
@@ -1,5 +1,5 @@
{
- "differentiateBoundary": "2013-01-01",
+ "differentiateBoundary": "2013-07-01",
"columnConfigs": [
{
"columnName": "lstg_format_name",
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index ca3a9bc..4c847bf 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -217,7 +217,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact " + "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME");
- statement.setString(1, "FP-GTC");
+ statement.setString(1, "FP-GTC_A");
ResultSet rs = statement.executeQuery();
@@ -225,7 +225,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
String format_name = rs.getString(1);
- Assert.assertTrue("FP-GTC".equals(format_name));
+ Assert.assertTrue("FP-GTC_A".equals(format_name));
rs.close();
statement.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
index cf18b20..cbd4e44 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.storage.hbase.HBaseStorage;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.junit.AfterClass;
@@ -64,13 +65,12 @@ public class ITCombinationTest extends ITKylinQueryTest {
*/
@Parameterized.Parameters
public static Collection<Object[]> configs() {
- // return Arrays.asList(new Object[][] { { "inner", "unset" }, { "left", "unset" }, { "inner", "off" }, { "left", "off" }, { "inner", "on" }, { "left", "on" }, });
- return Arrays.asList(new Object[][] { { "inner", "on", "v2" }, { "left", "on", "v1" }, { "left", "on", "v2" } });
+ return Arrays.asList(new Object[][] { { "inner", "on", "v2", false }, { "left", "on", "v1", false }, { "left", "on", "v2", false }, { "inner", "on", "v2", true }, { "left", "on", "v2", true } });
}
- public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine) throws Exception {
+ public ITCombinationTest(String joinType, String coprocessorToggle, String queryEngine, boolean excludeViewCubes) throws Exception {
- printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " + coprocessorToggle + ", query engine: " + queryEngine);
+ printInfo("Into combination join type: " + joinType + ", coprocessor toggle: " + coprocessorToggle + ", query engine: " + queryEngine + ", excludeViewCubes: " + excludeViewCubes);
ITKylinQueryTest.clean();
@@ -85,6 +85,12 @@ public class ITCombinationTest extends ITKylinQueryTest {
// unset
}
+ RemoveBlackoutRealizationsRule.blackouts.clear();
+ if (excludeViewCubes) {
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
+ }
+
if ("v1".equalsIgnoreCase(queryEngine))
HBaseStorage.overwriteStorageQuery = HBaseStorage.v1CubeStorageQuery;
else
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index dcacb06..4657a5c 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -34,6 +34,7 @@ import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.storage.hbase.HBaseStorage;
import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
@@ -46,7 +47,6 @@ import org.junit.Test;
import com.google.common.collect.Maps;
-@Ignore("KylinQueryTest is contained by ITCombinationTest")
public class ITKylinQueryTest extends KylinTestBase {
@BeforeClass
@@ -57,9 +57,12 @@ public class ITKylinQueryTest extends KylinTestBase {
priorities.put(RealizationType.CUBE, 0);
Candidate.setPriorities(priorities);
- joinType = "left";
+ joinType = "inner";
setupAll();
+
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_left_join_empty]");
+ RemoveBlackoutRealizationsRule.blackouts.add("CUBE[name=test_kylin_cube_with_view_inner_join_empty]");
}
@AfterClass
@@ -96,6 +99,7 @@ public class ITKylinQueryTest extends KylinTestBase {
ObserverEnabler.forceCoprocessorUnset();
HBaseMetadataTestCase.staticCleanupTestMetadata();
+ RemoveBlackoutRealizationsRule.blackouts.clear();
}
@@ -147,6 +151,11 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
+ public void testLikeQuery() throws Exception {
+ execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_like", null, true);
+ }
+
+ @Test
public void testVerifyQuery() throws Exception {
verifyResultRowCount(getQueryFolderPrefix() + "src/test/resources/query/sql_verifyCount");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/resources/query/sql_like/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query01.sql b/kylin-it/src/test/resources/query/sql_like/query01.sql
new file mode 100644
index 0000000..e1f131d
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query01.sql
@@ -0,0 +1,31 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select lstg_format_name as lstg_format_name, count(*) as cnt
+
+ from test_kylin_fact
+inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
+
+
+where lstg_format_name like '%BIN%'
+group by lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/kylin-it/src/test/resources/query/sql_like/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_like/query02.sql b/kylin-it/src/test/resources/query/sql_like/query02.sql
new file mode 100644
index 0000000..ef0b4a0
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_like/query02.sql
@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select upper(lstg_format_name) as lstg_format_name, count(*) as cnt from test_kylin_fact
+where lower(lstg_format_name)='abin_a' and substring(lstg_format_name,1,3) in ('ABI') and upper(lstg_format_name) > 'AAAA' and
+lower(lstg_format_name) like '%b%' and char_length(lstg_format_name) < 10 and char_length(lstg_format_name) > 3 and lstg_format_name||'a'='ABIN_Aa'
+group by lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
index 11498d6..53f871d 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/RoutingRule.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.query.routing.rules.RealizationSortRule;
+import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule;
import org.apache.kylin.query.routing.rules.RemoveUncapableRealizationsRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,6 +38,7 @@ public abstract class RoutingRule {
private static List<RoutingRule> rules = Lists.newLinkedList();
static {
+ rules.add(new RemoveBlackoutRealizationsRule());
rules.add(new RemoveUncapableRealizationsRule());
rules.add(new RealizationSortRule());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdad2d8d/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
new file mode 100644
index 0000000..9c3d7c9
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/routing/rules/RemoveBlackoutRealizationsRule.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.routing.rules;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.query.routing.Candidate;
+import org.apache.kylin.query.routing.RoutingRule;
+
+import com.google.common.collect.Sets;
+
+/**
+ * for IT use, exclude some cubes
+ */
+public class RemoveBlackoutRealizationsRule extends RoutingRule {
+ public static Set<String> blackouts = Sets.newHashSet();
+
+ @Override
+ public void apply(List<Candidate> candidates) {
+ for (Iterator<Candidate> iterator = candidates.iterator(); iterator.hasNext();) {
+ Candidate candidate = iterator.next();
+
+ if (blackouts.contains(candidate.getRealization().getCanonicalName())) {
+ iterator.remove();
+ }
+ }
+ }
+
+}
[18/50] [abbrv] kylin git commit: Add db info to table name when
creating table snapshot
Posted by li...@apache.org.
Add db info to table name when creating table snapshot
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ab6b73be
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ab6b73be
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ab6b73be
Branch: refs/heads/1.5.x-CDH5.7
Commit: ab6b73be75542a8e3e18f52aa52001286a57334b
Parents: 7815224
Author: Zhong <nj...@apache.org>
Authored: Mon Aug 29 09:35:21 2016 +0800
Committer: Zhong <nj...@apache.org>
Committed: Mon Aug 29 09:35:50 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/dict/lookup/SnapshotManager.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ab6b73be/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index b15847b..0d3848c 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -115,7 +115,7 @@ public class SnapshotManager {
}
public SnapshotTable buildSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException {
- SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getName());
+ SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity());
snapshot.updateRandomUuid();
String dup = checkDupByInfo(snapshot);
@@ -135,7 +135,7 @@ public class SnapshotManager {
}
public SnapshotTable rebuildSnapshot(ReadableTable table, TableDesc tableDesc, String overwriteUUID) throws IOException {
- SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getName());
+ SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity());
snapshot.setUuid(overwriteUUID);
snapshot.takeSnapshot(table, tableDesc);
[22/50] [abbrv] kylin git commit: KYLIN-1954
BuildInFunctionTransformer should be executed per CubeSegmentScanner
Posted by li...@apache.org.
KYLIN-1954 BuildInFunctionTransformer should be executed per CubeSegmentScanner
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cc9acbc2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cc9acbc2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cc9acbc2
Branch: refs/heads/1.5.x-CDH5.7
Commit: cc9acbc2e30b5a2c4a2a0d4b6bb056bc716f1fbb
Parents: f48f7fa
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Aug 29 18:28:30 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Aug 29 18:28:37 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/job/dataGen/ColumnConfig.java | 9 ++++
.../kylin/job/dataGen/FactTableGenerator.java | 44 ++++++++++++++++++--
.../org/apache/kylin/job/dataGen/GenConfig.java | 11 +++++
.../storage/gtrecord/CubeSegmentScanner.java | 9 +++-
.../localmeta/data/data_gen_config.json | 4 +-
5 files changed, 72 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cc9acbc2/assembly/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
index 44ba8f4..5e1c09f 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
@@ -35,6 +35,8 @@ public class ColumnConfig {
private boolean exclusive;
@JsonProperty("asRange")
private boolean asRange;
+ @JsonProperty("differentiateByDateBoundary")
+ private boolean differentiateByDateBoundary;
public boolean isAsRange() {
return asRange;
@@ -68,4 +70,11 @@ public class ColumnConfig {
this.valueSet = valueSet;
}
+ public boolean isDifferentiateByDateBoundary() {
+ return differentiateByDateBoundary;
+ }
+
+ public void setDifferentiateByDateBoundary(boolean differentiateByDateBoundary) {
+ this.differentiateByDateBoundary = differentiateByDateBoundary;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cc9acbc2/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
index 368f509..9373a02 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
@@ -50,6 +50,8 @@ import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import com.google.common.collect.Lists;
+
/**
*/
public class FactTableGenerator {
@@ -70,6 +72,11 @@ public class FactTableGenerator {
double conflictRatio;
double linkableRatio;
+ long differentiateBoundary = -1;
+ List<Integer> differentiateColumns = Lists.newArrayList();
+
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
+
// the names of lookup table columns which is in relation with fact
// table(appear as fk in fact table)
TreeMap<String, LinkedList<String>> lookupTableKeys = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
@@ -207,6 +214,25 @@ public class FactTableGenerator {
// load config
loadConfig();
+ int index = 0;
+ for (ColumnDesc cDesc : MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(factTableName).getColumns()) {
+ ColumnConfig cConfig = genConf.getColumnConfigByName(cDesc.getName());
+
+ if (cConfig != null && cConfig.isDifferentiateByDateBoundary()) {
+ if (!cDesc.getType().isStringFamily()) {
+ throw new IllegalStateException("differentiateByDateBoundary only applies to text types, actual:" + cDesc.getType());
+ }
+ if (genConf.getDifferentiateBoundary() == null) {
+ throw new IllegalStateException("differentiateBoundary not provided");
+ }
+ if (differentiateBoundary == -1) {
+ differentiateBoundary = format.parse(genConf.getDifferentiateBoundary()).getTime();
+ }
+ differentiateColumns.add(index);
+ }
+ index++;
+ }
+
TreeSet<String> factTableColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
for (DimensionDesc dim : desc.getDimensions()) {
@@ -359,7 +385,6 @@ public class FactTableGenerator {
throw new RuntimeException("Does not support " + type);
}
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
Date start = format.parse(range.get(0));
Date end = format.parse(range.get(1));
long diff = end.getTime() - start.getTime();
@@ -533,6 +558,8 @@ public class FactTableGenerator {
KylinConfig config = KylinConfig.getInstanceFromEnv();
LinkedList<String> columnValues = new LinkedList<String>();
+ long currentRowTime = -1;
+
for (ColumnDesc cDesc : MetadataManager.getInstance(config).getTableDesc(factTableName).getColumns()) {
String colName = cDesc.getName();
@@ -544,8 +571,7 @@ public class FactTableGenerator {
columnValues.add(candidates.get(r.nextInt(candidates.size())));
} else if (usedCols.contains(colName)) {
-
- // if the current column is a metric column in fact table
+ // if the current column is a metric or dimension column in fact table
columnValues.add(createCell(cDesc));
} else {
@@ -553,6 +579,18 @@ public class FactTableGenerator {
columnValues.add(createDefaultsCell(cDesc.getTypeName()));
defaultColumns.add(colName);
}
+
+ if (cDesc.getRef().equals(this.cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateColumnRef())) {
+ currentRowTime = format.parse(columnValues.get(columnValues.size() - 1)).getTime();
+ }
+ }
+
+ for (Integer index : differentiateColumns) {
+ if (currentRowTime >= differentiateBoundary) {
+ columnValues.set(index, columnValues.get(index) + "_B");
+ } else {
+ columnValues.set(index, columnValues.get(index) + "_A");
+ }
}
return columnValues;
http://git-wip-us.apache.org/repos/asf/kylin/blob/cc9acbc2/assembly/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
index c58cfb6..5204d2a 100644
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
+++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
@@ -38,8 +38,19 @@ public class GenConfig {
@JsonProperty("columnConfigs")
private ArrayList<ColumnConfig> columnConfigs;
+ @JsonProperty("differentiateBoundary")
+ private String differentiateBoundary; //data before and after the provided date will be different, so that different segments will have different segments
+
private HashMap<String, ColumnConfig> cache = new HashMap<String, ColumnConfig>();
+ public String getDifferentiateBoundary() {
+ return differentiateBoundary;
+ }
+
+ public void setDifferentiateBoundary(String differentiateBoundary) {
+ this.differentiateBoundary = differentiateBoundary;
+ }
+
public ArrayList<ColumnConfig> getColumnConfigs() {
return columnConfigs;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cc9acbc2/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 4365ee2..6ed7d3b 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -34,7 +34,9 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.ScannerWorker;
import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
+import org.apache.kylin.metadata.filter.StringCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.storage.StorageContext;
@@ -52,10 +54,15 @@ public class CubeSegmentScanner implements IGTScanner {
final GTScanRequest scanRequest;
public CubeSegmentScanner(CubeSegment cubeSeg, Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, //
- Collection<FunctionDesc> metrics, TupleFilter filter, StorageContext context, String gtStorage) {
+ Collection<FunctionDesc> metrics, TupleFilter originalfilter, StorageContext context, String gtStorage) {
this.cuboid = cuboid;
this.cubeSeg = cubeSeg;
+ //the filter might be changed later in this CubeSegmentScanner (In ITupleFilterTransformer)
+ //to avoid issues like in https://issues.apache.org/jira/browse/KYLIN-1954, make sure each CubeSegmentScanner
+ //is working on its own copy
+ byte[] serialize = TupleFilterSerializer.serialize(originalfilter, StringCodeSystem.INSTANCE);
+ TupleFilter filter = TupleFilterSerializer.deserialize(serialize, StringCodeSystem.INSTANCE);
// translate FunctionTupleFilter to IN clause
ITupleFilterTransformer translator = new BuildInFunctionTransformer(cubeSeg.getDimensionEncodingMap());
filter = translator.transform(filter);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cc9acbc2/examples/test_case_data/localmeta/data/data_gen_config.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/data_gen_config.json b/examples/test_case_data/localmeta/data/data_gen_config.json
index ff3f676..f730058 100644
--- a/examples/test_case_data/localmeta/data/data_gen_config.json
+++ b/examples/test_case_data/localmeta/data/data_gen_config.json
@@ -1,4 +1,5 @@
{
+ "differentiateBoundary": "2013-01-01",
"columnConfigs": [
{
"columnName": "lstg_format_name",
@@ -9,7 +10,8 @@
"Auction",
"Others"
],
- "exclusive": true
+ "exclusive": true,
+ "differentiateByDateBoundary": true
},
{
"columnName": "SELLER_ID",
[28/50] [abbrv] kylin git commit: minor,
add LIKE FilterOperatorEnum in case it will be pushed down(it won't
for now)
Posted by li...@apache.org.
minor, add LIKE FilterOperatorEnum in case it will be pushed down(it won't for now)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9e0b8d80
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9e0b8d80
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9e0b8d80
Branch: refs/heads/1.5.x-CDH5.7
Commit: 9e0b8d8033dbe4e86d299adbd102cf52792cd5fe
Parents: c5fdacd
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Aug 31 11:47:37 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Aug 31 11:47:42 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/gridtable/GTUtil.java | 53 +++++++++++
.../filter/BuiltInFunctionTupleFilter.java | 22 ++++-
.../filter/EvaluatableLikeFunction.java | 96 ++++++++++++++++++++
.../EvaluatableLikeFunctionTransformer.java | 76 ++++++++++++++++
.../kylin/metadata/filter/TupleFilter.java | 2 +-
.../metadata/filter/TupleFilterSerializer.java | 3 +
.../org/apache/kylin/query/KylinTestBase.java | 8 +-
7 files changed, 254 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index 4b50176..ce0f016 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -18,6 +18,7 @@
package org.apache.kylin.gridtable;
+import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
@@ -25,6 +26,7 @@ import java.util.Set;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.filter.BuiltInFunctionTupleFilter;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
@@ -104,6 +106,13 @@ public class GTUtil {
if (encodeConstants && filter instanceof CompareTupleFilter) {
return encodeConstants((CompareTupleFilter) filter);
}
+ if (encodeConstants && filter instanceof BuiltInFunctionTupleFilter) {
+ if (!((BuiltInFunctionTupleFilter) filter).hasNested()) {
+ return encodeConstants((BuiltInFunctionTupleFilter) filter);
+ } else {
+ throw new IllegalStateException("Nested BuiltInFunctionTupleFilter is not supported to be pushed down");
+ }
+ }
return filter;
}
@@ -122,9 +131,13 @@ public class GTUtil {
return oldCompareFilter;
}
+ //CompareTupleFilter containing BuiltInFunctionTupleFilter will not reach here caz it will be transformed by BuiltInFunctionTransformer
CompareTupleFilter newCompareFilter = new CompareTupleFilter(oldCompareFilter.getOperator());
newCompareFilter.addChild(new ColumnTupleFilter(externalCol));
+ //for CompareTupleFilter containing dynamicVariables, the below codes will actually replace dynamicVariables
+ //with normal ConstantTupleFilter
+
Object firstValue = constValues.iterator().next();
int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
@@ -199,6 +212,46 @@ public class GTUtil {
return result;
}
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private TupleFilter encodeConstants(BuiltInFunctionTupleFilter funcFilter) {
+ // extract ColumnFilter & ConstantFilter
+ TblColRef externalCol = funcFilter.getColumn();
+
+ if (externalCol == null) {
+ return funcFilter;
+ }
+
+ Collection constValues = funcFilter.getConstantTupleFilter().getValues();
+ if (constValues == null || constValues.isEmpty()) {
+ return funcFilter;
+ }
+
+ BuiltInFunctionTupleFilter newFuncFilter;
+ try {
+ newFuncFilter = funcFilter.getClass().getConstructor(String.class).newInstance(funcFilter.getName());
+ } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
+ throw new RuntimeException(e);
+ }
+ newFuncFilter.addChild(new ColumnTupleFilter(externalCol));
+
+ int col = colMapping == null ? externalCol.getColumnDesc().getZeroBasedIndex() : colMapping.indexOf(externalCol);
+
+ ByteArray code;
+
+ // translate constant into code
+ Set newValues = Sets.newHashSet();
+ for (Object value : constValues) {
+ code = translate(col, value, 0);
+ if (code == null) {
+ throw new IllegalStateException("Cannot serialize BuiltInFunctionTupleFilter");
+ }
+ newValues.add(code);
+ }
+ newFuncFilter.addChild(new ConstantTupleFilter(newValues));
+
+ return newFuncFilter;
+ }
+
transient ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength());
private ByteArray translate(int col, Object value, int roundingFlag) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 5a10371..40afb18 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -25,6 +25,7 @@ import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
+import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.filter.function.BuiltInMethod;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
@@ -40,13 +41,18 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
protected String name;
// FIXME Only supports single parameter functions currently
protected TupleFilter columnContainerFilter;//might be a ColumnTupleFilter(simple case) or FunctionTupleFilter(complex case like substr(lower()))
+ protected ConstantTupleFilter constantTupleFilter;
protected int colPosition;
protected Method method;
protected List<Serializable> methodParams;
protected boolean isValidFunc = false;
public BuiltInFunctionTupleFilter(String name) {
- super(Lists.<TupleFilter> newArrayList(), FilterOperatorEnum.FUNCTION);
+ this(name, null);
+ }
+
+ public BuiltInFunctionTupleFilter(String name, FilterOperatorEnum filterOperatorEnum) {
+ super(Lists.<TupleFilter> newArrayList(), filterOperatorEnum == null ? FilterOperatorEnum.FUNCTION : filterOperatorEnum);
this.methodParams = Lists.newArrayList();
if (name != null) {
@@ -59,6 +65,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
return name;
}
+ public ConstantTupleFilter getConstantTupleFilter() {
+ return constantTupleFilter;
+ }
+
public TblColRef getColumn() {
if (columnContainerFilter == null)
return null;
@@ -71,6 +81,10 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
throw new UnsupportedOperationException("Wrong type TupleFilter in FunctionTupleFilter.");
}
+ public boolean hasNested() {
+ return (columnContainerFilter != null && columnContainerFilter instanceof BuiltInFunctionTupleFilter);
+ }
+
public Object invokeFunction(Object input) throws InvocationTargetException, IllegalAccessException {
if (columnContainerFilter instanceof ColumnTupleFilter)
methodParams.set(colPosition, (Serializable) input);
@@ -90,6 +104,7 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
colPosition = methodParams.size();
methodParams.add(null);
} else if (child instanceof ConstantTupleFilter) {
+ this.constantTupleFilter = (ConstantTupleFilter) child;
Serializable constVal = (Serializable) child.getValues().iterator().next();
try {
Class<?> clazz = Primitives.wrap(method.getParameterTypes()[methodParams.size()]);
@@ -122,12 +137,13 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
@Override
public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- throw new UnsupportedOperationException("Function filter cannot serialized");
+ BytesUtil.writeUTFString(name, buffer);
}
@Override
public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
- throw new UnsupportedOperationException("Function filter cannot serialized");
+ this.name = BytesUtil.readUTFString(buffer);
+ this.initMethod();
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
new file mode 100644
index 0000000..59961ac
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunction.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.lang.reflect.InvocationTargetException;
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.StringSerializer;
+import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
+
+public class EvaluatableLikeFunction extends BuiltInFunctionTupleFilter {
+
+ public EvaluatableLikeFunction(String name) {
+ super(name, FilterOperatorEnum.LIKE);
+ }
+
+ @Override
+ public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
+
+ // extract tuple value
+ Object tupleValue = null;
+ for (TupleFilter filter : this.children) {
+ if (!isConstant(filter)) {
+ filter.evaluate(tuple, cs);
+ tupleValue = filter.getValues().iterator().next();
+ }
+ }
+
+ // consider null case
+ if (cs.isNull(tupleValue)) {
+ return false;
+ }
+
+ ByteArray valueByteArray = (ByteArray) tupleValue;
+ StringSerializer serializer = new StringSerializer(DataType.getType("string"));
+ String value = serializer.deserialize(ByteBuffer.wrap(valueByteArray.array(), valueByteArray.offset(), valueByteArray.length()));
+ try {
+ return (Boolean) invokeFunction(value);
+ } catch (InvocationTargetException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ if (!isValid()) {
+ throw new IllegalStateException("must be valid");
+ }
+ if (methodParams.size() != 2 || methodParams.get(0) != null || methodParams.get(1) == null) {
+ throw new IllegalArgumentException("bad methodParams: " + methodParams);
+ }
+ BytesUtil.writeUTFString(name, buffer);
+ }
+
+ @Override
+ public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
+ this.name = BytesUtil.readUTFString(buffer);
+ this.initMethod();
+ }
+
+ @Override
+ public boolean isEvaluable() {
+ return true;
+ }
+
+ private boolean isConstant(TupleFilter filter) {
+ return (filter instanceof ConstantTupleFilter) || (filter instanceof DynamicTupleFilter);
+ }
+
+ public String getLikePattern() {
+ ByteArray byteArray = (ByteArray) methodParams.get(1);
+ StringSerializer s = new StringSerializer(DataType.getType("string"));
+ String pattern = s.deserialize(ByteBuffer.wrap(byteArray.array(), byteArray.offset(), byteArray.length()));
+ return pattern;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
new file mode 100644
index 0000000..7295533
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/EvaluatableLikeFunctionTransformer.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import java.util.ListIterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class EvaluatableLikeFunctionTransformer {
+ public static final Logger logger = LoggerFactory.getLogger(EvaluatableLikeFunctionTransformer.class);
+
+ public static TupleFilter transform(TupleFilter tupleFilter) {
+ TupleFilter translated = null;
+ if (tupleFilter instanceof CompareTupleFilter) {
+ CompareTupleFilter compTupleFilter = (CompareTupleFilter) tupleFilter;
+ if (compTupleFilter.getFunction() != null && (compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter)) {
+ throw new IllegalArgumentException("BuiltInFunctionTupleFilter not supported :" + ((BuiltInFunctionTupleFilter) compTupleFilter.getFunction()).getName());
+ }
+ } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
+ BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) tupleFilter;
+ if (isLikeFunction(builtInFunctionTupleFilter)) {
+ for (TupleFilter child : builtInFunctionTupleFilter.getChildren()) {
+ if (!(child instanceof ColumnTupleFilter) && !(child instanceof ConstantTupleFilter)) {
+ throw new IllegalArgumentException("Only simple like clause is supported");
+ }
+ }
+
+ translated = new EvaluatableLikeFunction(builtInFunctionTupleFilter.getName());
+ for (TupleFilter child : builtInFunctionTupleFilter.getChildren()) {
+ translated.addChild(child);
+ }
+
+ } else {
+ throw new IllegalArgumentException("BuiltInFunctionTupleFilter not supported: " + builtInFunctionTupleFilter.getName());
+ }
+
+ } else if (tupleFilter instanceof LogicalTupleFilter) {
+ @SuppressWarnings("unchecked")
+ ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren().listIterator();
+ while (childIterator.hasNext()) {
+ TupleFilter transformed = transform(childIterator.next());
+ if (transformed != null) {
+ childIterator.set(transformed);
+ } else {
+ throw new IllegalStateException("Should not be null");
+ }
+ }
+ }
+ return translated == null ? tupleFilter : translated;
+ }
+
+ private static boolean isLikeFunction(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) {
+ return "like".equalsIgnoreCase(builtInFunctionTupleFilter.getName());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 3250640..2fb4e1f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -38,7 +38,7 @@ import com.google.common.collect.Maps;
public abstract class TupleFilter {
public enum FilterOperatorEnum {
- EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36);
+ EQ(1), NEQ(2), GT(3), LT(4), GTE(5), LTE(6), ISNULL(7), ISNOTNULL(8), IN(9), NOTIN(10), AND(20), OR(21), NOT(22), COLUMN(30), CONSTANT(31), DYNAMIC(32), EXTRACT(33), CASE(34), FUNCTION(35), MASSIN(36), LIKE(37);
private final int value;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
index f6c8bc4..0aca9a1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilterSerializer.java
@@ -188,6 +188,9 @@ public class TupleFilterSerializer {
case FUNCTION:
filter = new BuiltInFunctionTupleFilter(null);
break;
+ case LIKE:
+ filter = new EvaluatableLikeFunction(null);
+ break;
case MASSIN:
filter = new MassInTupleFilter();
break;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9e0b8d80/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 4e59815..2ad1105 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -63,6 +63,8 @@ import com.google.common.io.Files;
*/
public class KylinTestBase {
+ public static boolean PRINT_RESULT = false;
+
class ObjectArray {
Object[] data;
@@ -224,7 +226,8 @@ public class KylinTestBase {
if (needSort) {
queryTable = new SortedTable(queryTable, columnNames);
}
- //printResult(queryTable);
+ if (PRINT_RESULT)
+ printResult(queryTable);
return queryTable;
}
@@ -280,7 +283,8 @@ public class KylinTestBase {
if (needSort) {
queryTable = new SortedTable(queryTable, columnNames);
}
- //printResult(queryTable);
+ if (PRINT_RESULT)
+ printResult(queryTable);
return queryTable;
}
[11/50] [abbrv] kylin git commit: minor, code refactor
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index 0f96e3c..7ec24b2 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@ -53,19 +53,19 @@ public class CubeTupleConverter {
final CubeSegment cubeSeg;
final Cuboid cuboid;
final TupleInfo tupleInfo;
- final List<IDerivedColumnFiller> derivedColFillers;
+ private final List<IDerivedColumnFiller> derivedColFillers;
- final int[] gtColIdx;
- final int[] tupleIdx;
- final Object[] gtValues;
- final MeasureType<?>[] measureTypes;
+ private final int[] gtColIdx;
+ private final int[] tupleIdx;
+ private final Object[] gtValues;
+ private final MeasureType<?>[] measureTypes;
- final List<IAdvMeasureFiller> advMeasureFillers;
- final List<Integer> advMeasureIndexInGTValues;
+ private final List<IAdvMeasureFiller> advMeasureFillers;
+ private final List<Integer> advMeasureIndexInGTValues;
- final int nSelectedDims;
+ private final int nSelectedDims;
- final int[] dimensionIndexOnTuple;
+ private final int[] dimensionIndexOnTuple;
public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo) {
@@ -102,44 +102,44 @@ public class CubeTupleConverter {
////////////
- int iii = 0;
+ int i = 0;
// pre-calculate dimension index mapping to tuple
for (TblColRef dim : selectedDimensions) {
- int i = mapping.getIndexOf(dim);
- gtColIdx[iii] = i;
- tupleIdx[iii] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
+ int dimIndex = mapping.getIndexOf(dim);
+ gtColIdx[i] = dimIndex;
+ tupleIdx[i] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
// if (tupleIdx[iii] == -1) {
// throw new IllegalStateException("dim not used in tuple:" + dim);
// }
- iii++;
+ i++;
}
for (FunctionDesc metric : selectedMetrics) {
- int i = mapping.getIndexOf(metric);
- gtColIdx[iii] = i;
+ int metricIndex = mapping.getIndexOf(metric);
+ gtColIdx[i] = metricIndex;
if (metric.needRewrite()) {
String rewriteFieldName = metric.getRewriteFieldName();
- tupleIdx[iii] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
+ tupleIdx[i] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
} else {
// a non-rewrite metrics (like sum, or dimension playing as metrics) is like a dimension column
TblColRef col = metric.getParameter().getColRefs().get(0);
- tupleIdx[iii] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
+ tupleIdx[i] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
}
MeasureType<?> measureType = metric.getMeasureType();
if (measureType.needAdvancedTupleFilling()) {
Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap(measureType.getColumnsNeedDictionary(metric));
advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, dictionaryMap));
- advMeasureIndexInGTValues.add(iii);
+ advMeasureIndexInGTValues.add(i);
} else {
- measureTypes[iii] = measureType;
+ measureTypes[i] = measureType;
}
- iii++;
+ i++;
}
// prepare derived columns and filler
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index ae5240b..bacd293 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -127,7 +127,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
if (scanners.isEmpty())
return ITupleIterator.EMPTY_TUPLE_ITERATOR;
- return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context);
+ return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo, context, cubeDesc.supportsLimitPushDown());
}
protected boolean skipZeroInputSegment(CubeSegment cubeSegment) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index 2cff76c..7059473 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -54,7 +54,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
private int scanCountDelta;
public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid, Set<TblColRef> selectedDimensions, //
- Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context) {
+ Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context, boolean supportLimitPushDown) {
this.context = context;
this.scanners = scanners;
@@ -63,8 +63,6 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context));
}
- boolean supportLimitPushDown = scanners.get(0).getSegment().getCubeDesc().supportsLimitPushDown();
-
this.storagePushDownLimit = context.getStoragePushDownLimit();
if (!supportLimitPushDown || storagePushDownLimit > KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax()) {
//normal case
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 5db4342..dcacb06 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -51,12 +51,12 @@ public class ITKylinQueryTest extends KylinTestBase {
@BeforeClass
public static void setUp() throws Exception {
+ printInfo("setUp in ITKylinQueryTest");
Map<RealizationType, Integer> priorities = Maps.newHashMap();
priorities.put(RealizationType.HYBRID, 0);
priorities.put(RealizationType.CUBE, 0);
Candidate.setPriorities(priorities);
- printInfo("setUp in ITKylinQueryTest");
joinType = "left";
setupAll();
@@ -64,8 +64,8 @@ public class ITKylinQueryTest extends KylinTestBase {
@AfterClass
public static void tearDown() throws Exception {
- Candidate.restorePriorities();
printInfo("tearDown in ITKylinQueryTest");
+ Candidate.restorePriorities();
clean();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0a82bf8..ea29958 100644
--- a/pom.xml
+++ b/pom.xml
@@ -366,6 +366,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <version>${hadoop2.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<version>${hadoop2.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index bf93300..96e75f3 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -25,6 +25,7 @@ import org.apache.calcite.DataContext;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -162,8 +163,14 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
// Hack no-group-by query for better results
private void hackNoGroupByAggregation(SQLDigest sqlDigest) {
- if (!sqlDigest.groupbyColumns.isEmpty() || !sqlDigest.metricColumns.isEmpty())
+ if (!(olapContext.realization instanceof CubeInstance)) {
+ //the hack only makes sense for cubes
return;
+ }
+
+ if (!sqlDigest.isRawQuery()) {
+ return;
+ }
// If no group by and metric found, then it's simple query like select ... from ... where ...,
// But we have no raw data stored, in order to return better results, we hack to output sum of metric column
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
index ab7884a..9ea8961 100644
--- a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
+++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java
@@ -35,6 +35,7 @@ public class Candidate implements Comparable<Candidate> {
static {
DEFAULT_PRIORITIES.put(RealizationType.HYBRID, 0);
DEFAULT_PRIORITIES.put(RealizationType.CUBE, 1);
+ DEFAULT_PRIORITIES.put(RealizationType.INVERTED_INDEX, 1);
}
/** for test only */
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 200c040..07a3cc3 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -40,7 +40,7 @@ import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LoggableCachedThreadPool;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTScanRange;
@@ -67,8 +67,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
private static ExecutorService executorService = new LoggableCachedThreadPool();
- public CubeHBaseEndpointRPC(CubeSegment cubeSeg, Cuboid cuboid, GTInfo fullGTInfo) {
- super(cubeSeg, cuboid, fullGTInfo);
+ public CubeHBaseEndpointRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo) {
+ super(segment, cuboid, fullGTInfo);
}
private byte[] getByteArrayForShort(short v) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 687ee83..c318cba 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -31,6 +31,7 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.FuzzyKeyEncoder;
import org.apache.kylin.cube.kv.FuzzyMaskEncoder;
@@ -60,8 +61,10 @@ public abstract class CubeHBaseRPC implements IGTStorage {
final private RowKeyEncoder fuzzyKeyEncoder;
final private RowKeyEncoder fuzzyMaskEncoder;
- public CubeHBaseRPC(CubeSegment cubeSeg, Cuboid cuboid, GTInfo fullGTInfo) {
- this.cubeSeg = cubeSeg;
+ public CubeHBaseRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo) {
+ Preconditions.checkArgument(segment instanceof CubeSegment, "segment must be CubeSegment");
+
+ this.cubeSeg = (CubeSegment) segment;
this.cuboid = cuboid;
this.fullGTInfo = fullGTInfo;
http://git-wip-us.apache.org/repos/asf/kylin/blob/5ee76e8b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 238939c..a359d19 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
-import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.ISegment;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.dimension.DimensionEncoding;
@@ -89,8 +89,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
}
- public CubeHBaseScanRPC(CubeSegment cubeSeg, Cuboid cuboid, final GTInfo fullGTInfo) {
- super(cubeSeg, cuboid, fullGTInfo);
+ public CubeHBaseScanRPC(ISegment segment, Cuboid cuboid, final GTInfo fullGTInfo) {
+ super(segment, cuboid, fullGTInfo);
MassInTupleFilter.VALUE_PROVIDER_FACTORY = new MassInValueProviderFactoryImpl(new MassInValueProviderFactoryImpl.DimEncAware() {
@Override
public DimensionEncoding getDimEnc(TblColRef col) {
[50/50] [abbrv] kylin git commit: KYLIN-1672 support kylin on cdh 5.7
Posted by li...@apache.org.
KYLIN-1672 support kylin on cdh 5.7
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c7d02166
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c7d02166
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c7d02166
Branch: refs/heads/1.5.x-CDH5.7
Commit: c7d021661ebc8c0d4a58933032a47815d6eb0637
Parents: f6ee0e7
Author: Lynne Jiang <ly...@hotmail.com>
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu <li...@apache.org>
Committed: Sat Sep 3 20:41:56 2016 +0800
----------------------------------------------------------------------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0
.../kylin/engine/mr/steps/MockupMapContext.java | 15 +-
examples/test_case_data/sandbox/core-site.xml | 146 +++---
examples/test_case_data/sandbox/hbase-site.xml | 162 ++----
examples/test_case_data/sandbox/hdfs-site.xml | 259 ++--------
examples/test_case_data/sandbox/mapred-site.xml | 396 ++++++---------
examples/test_case_data/sandbox/yarn-site.xml | 496 ++-----------------
pom.xml | 18 +-
server/pom.xml | 36 ++
.../storage/hbase/steps/MockupMapContext.java | 19 +-
tool/pom.xml | 12 +
11 files changed, 430 insertions(+), 1129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
outKV[0] = key;
outKV[1] = value;
}
+
}
@Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
throw new NotImplementedException();
}
+
@Override
public float getProgress() {
throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
}
@Override
- public RawComparator<?> getSortComparator() {
+ public boolean userClassesTakesPrecedence() {
throw new NotImplementedException();
}
@Override
- public String getJar() {
+ public RawComparator<?> getSortComparator() {
throw new NotImplementedException();
}
@Override
- public RawComparator<?> getGroupingComparator() {
+ public String getJar() {
throw new NotImplementedException();
}
@@ -221,7 +223,7 @@ public class MockupMapContext {
@Override
public boolean getProfileEnabled() {
- throw new NotImplementedException();
+ return false;
}
@Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
public RawComparator<?> getCombinerKeyGroupingComparator() {
throw new NotImplementedException();
}
+
+ @Override
+ public RawComparator<?> getGroupingComparator() {
+ return null;
+ }
});
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
+<!--Autogenerated by Cloudera Manager-->
<configuration>
-
<property>
<name>fs.defaultFS</name>
- <value>hdfs://sandbox.hortonworks.com:8020</value>
- <final>true</final>
+ <value>hdfs://quickstart.cloudera:8020</value>
</property>
-
<property>
<name>fs.trash.interval</name>
- <value>360</value>
+ <value>1</value>
</property>
-
<property>
- <name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
- <value>120</value>
+ <name>io.compression.codecs</name>
+ <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
</property>
-
<property>
- <name>hadoop.http.authentication.simple.anonymous.allowed</name>
- <value>true</value>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
</property>
-
<property>
- <name>hadoop.proxyuser.falcon.groups</name>
- <value>users</value>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value>authentication</value>
+ </property>
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>DEFAULT</value>
</property>
-
<property>
- <name>hadoop.proxyuser.falcon.hosts</name>
+ <name>hadoop.proxyuser.oozie.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hbase.groups</name>
- <value>users</value>
+ <name>hadoop.proxyuser.oozie.groups</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hbase.hosts</name>
+ <name>hadoop.proxyuser.mapred.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hcat.groups</name>
+ <name>hadoop.proxyuser.mapred.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hcat.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <name>hadoop.proxyuser.flume.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hive.groups</name>
- <value>users</value>
+ <name>hadoop.proxyuser.flume.groups</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.HTTP.hosts</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.HTTP.groups</name>
+ <value>*</value>
</property>
-
<property>
<name>hadoop.proxyuser.hive.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hue.groups</name>
+ <name>hadoop.proxyuser.hive.groups</name>
<value>*</value>
</property>
-
<property>
<name>hadoop.proxyuser.hue.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.oozie.groups</name>
+ <name>hadoop.proxyuser.hue.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.oozie.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <name>hadoop.proxyuser.httpfs.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.root.groups</name>
+ <name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.root.hosts</name>
+ <name>hadoop.proxyuser.hdfs.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.security.auth_to_local</name>
- <value>DEFAULT</value>
+ <name>hadoop.proxyuser.hdfs.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.security.authentication</name>
- <value>simple</value>
+ <name>hadoop.proxyuser.yarn.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.security.authorization</name>
- <value>false</value>
+ <name>hadoop.proxyuser.yarn.groups</name>
+ <value>*</value>
</property>
-
<property>
- <name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
+ <name>hadoop.security.group.mapping</name>
+ <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
</property>
-
<property>
- <name>io.file.buffer.size</name>
- <value>131072</value>
+ <name>hadoop.security.instrumentation.requires.admin</name>
+ <value>false</value>
</property>
-
<property>
- <name>io.serializations</name>
- <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
+ <name>net.topology.script.file.name</name>
+ <value>/etc/hadoop/conf.cloudera.yarn/topology.py</value>
</property>
-
<property>
- <name>ipc.client.connect.max.retries</name>
- <value>50</value>
+ <name>io.file.buffer.size</name>
+ <value>65536</value>
</property>
-
<property>
- <name>ipc.client.connection.maxidletime</name>
- <value>30000</value>
+ <name>hadoop.ssl.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.require.client.cert</name>
+ <value>false</value>
+ <final>true</final>
</property>
-
<property>
- <name>ipc.client.idlethreshold</name>
- <value>8000</value>
+ <name>hadoop.ssl.keystores.factory.class</name>
+ <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
+ <final>true</final>
</property>
-
<property>
- <name>ipc.server.tcpnodelay</name>
- <value>true</value>
+ <name>hadoop.ssl.server.conf</name>
+ <value>ssl-server.xml</value>
+ <final>true</final>
</property>
-
<property>
- <name>mapreduce.jobtracker.webinterface.trusted</name>
- <value>false</value>
+ <name>hadoop.ssl.client.conf</name>
+ <value>ssl-client.xml</value>
+ <final>true</final>
</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 734908e..58c6223 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -15,180 +15,104 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
<property>
- <name>hbase.client.keyvalue.maxsize</name>
- <value>10485760</value>
- </property>
-
- <property>
- <name>hbase.client.scanner.caching</name>
- <value>100</value>
+ <name>hbase.rootdir</name>
+ <value>hdfs://quickstart.cloudera:8020/hbase</value>
</property>
-
<property>
- <name>hbase.cluster.distributed</name>
+ <name>hbase.replication</name>
<value>true</value>
</property>
-
- <property>
- <name>hbase.coprocessor.master.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
- </property>
-
<property>
- <name>hbase.coprocessor.region.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
+ <name>hbase.client.write.buffer</name>
+ <value>2097152</value>
</property>
-
- <property>
- <name>hbase.defaults.for.version.skip</name>
- <value>true</value>
- </property>
-
<property>
- <name>hbase.hregion.majorcompaction</name>
- <value>604800000</value>
+ <name>hbase.client.pause</name>
+ <value>100</value>
</property>
-
<property>
- <name>hbase.hregion.majorcompaction.jitter</name>
- <value>0.50</value>
+ <name>hbase.client.retries.number</name>
+ <value>35</value>
</property>
-
<property>
- <name>hbase.hregion.max.filesize</name>
- <value>10737418240</value>
+ <name>hbase.client.scanner.caching</name>
+ <value>100</value>
</property>
-
<property>
- <name>hbase.hregion.memstore.block.multiplier</name>
- <value>4</value>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value>10485760</value>
</property>
-
<property>
- <name>hbase.hregion.memstore.flush.size</name>
- <value>134217728</value>
- </property>
-
- <property>
- <name>hbase.hregion.memstore.mslab.enabled</name>
+ <name>hbase.ipc.client.allowsInterrupt</name>
<value>true</value>
</property>
-
<property>
- <name>hbase.hstore.blockingStoreFiles</name>
+ <name>hbase.client.primaryCallTimeout.get</name>
<value>10</value>
</property>
-
- <property>
- <name>hbase.hstore.compactionThreshold</name>
- <value>3</value>
- </property>
-
<property>
- <name>hbase.local.dir</name>
- <value>${hbase.tmp.dir}/local</value>
+ <name>hbase.client.primaryCallTimeout.multiget</name>
+ <value>10</value>
</property>
-
<property>
- <name>hbase.master.info.bindAddress</name>
- <value>0.0.0.0</value>
+ <name>hbase.regionserver.thrift.http</name>
+ <value>false</value>
</property>
-
<property>
- <name>hbase.master.info.port</name>
- <value>60010</value>
+ <name>hbase.thrift.support.proxyuser</name>
+ <value>false</value>
</property>
-
<property>
- <name>hbase.master.port</name>
+ <name>hbase.rpc.timeout</name>
<value>60000</value>
</property>
-
- <property>
- <name>hbase.regionserver.global.memstore.lowerLimit</name>
- <value>0.38</value>
- </property>
-
<property>
- <name>hbase.regionserver.global.memstore.upperLimit</name>
- <value>0.4</value>
- </property>
-
- <property>
- <name>hbase.regionserver.handler.count</name>
- <value>60</value>
+ <name>hbase.snapshot.enabled</name>
+ <value>true</value>
</property>
-
<property>
- <name>hbase.regionserver.info.port</name>
- <value>60030</value>
+ <name>hbase.snapshot.master.timeoutMillis</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.rootdir</name>
- <value>hdfs://sandbox.hortonworks.com:8020/apps/hbase/data</value>
+ <name>hbase.snapshot.region.timeout</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.rpc.protection</name>
- <value>PRIVACY</value>
+ <name>hbase.snapshot.master.timeout.millis</name>
+ <value>60000</value>
</property>
-
<property>
<name>hbase.security.authentication</name>
<value>simple</value>
</property>
-
<property>
- <name>hbase.security.authorization</name>
- <value>true</value>
+ <name>hbase.rpc.protection</name>
+ <value>authentication</value>
</property>
-
<property>
- <name>hbase.superuser</name>
- <value>hbase</value>
+ <name>zookeeper.session.timeout</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.tmp.dir</name>
- <value>/hadoop/hbase</value>
+ <name>zookeeper.znode.parent</name>
+ <value>/hbase</value>
</property>
-
<property>
- <name>hbase.zookeeper.property.clientPort</name>
- <value>2181</value>
+ <name>zookeeper.znode.rootserver</name>
+ <value>root-region-server</value>
</property>
-
<property>
<name>hbase.zookeeper.quorum</name>
- <value>sandbox.hortonworks.com</value>
+ <value>quickstart.cloudera</value>
</property>
-
<property>
- <name>hbase.zookeeper.useMulti</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hfile.block.cache.size</name>
- <value>0.40</value>
- </property>
-
- <property>
- <name>zookeeper.session.timeout</name>
- <value>30000</value>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>2181</value>
</property>
-
<property>
- <name>zookeeper.znode.parent</name>
- <value>/hbase-unsecure</value>
+ <name>hbase.rest.ssl.enabled</name>
+ <value>false</value>
</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index 1175fff..05854bd 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -15,271 +15,68 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>dfs.block.access.token.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>dfs.block.size</name>
- <value>34217472</value>
- </property>
-
- <property>
- <name>dfs.blockreport.initialDelay</name>
- <value>120</value>
- </property>
-
- <property>
- <name>dfs.blocksize</name>
- <value>134217728</value>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit.streams.cache.size</name>
- <value>4096</value>
- </property>
-
- <property>
- <name>dfs.cluster.administrators</name>
- <value>hdfs</value>
- </property>
-
- <property>
- <name>dfs.datanode.address</name>
- <value>0.0.0.0:50010</value>
- </property>
-
- <property>
- <name>dfs.datanode.balance.bandwidthPerSec</name>
- <value>6250000</value>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir</name>
- <value>/hadoop/hdfs/data</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir.perm</name>
- <value>750</value>
- </property>
-
<property>
- <name>dfs.datanode.du.reserved</name>
- <value>1073741824</value>
- </property>
-
- <property>
- <name>dfs.datanode.failed.volumes.tolerated</name>
- <value>0</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:50075</value>
- </property>
-
- <property>
- <name>dfs.datanode.https.address</name>
- <value>0.0.0.0:50475</value>
- </property>
-
- <property>
- <name>dfs.datanode.ipc.address</name>
- <value>0.0.0.0:8010</value>
- </property>
-
- <property>
- <name>dfs.datanode.max.transfer.threads</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>dfs.datanode.max.xcievers</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
- <property>
- <name>dfs.heartbeat.interval</name>
- <value>3</value>
+ <name>dfs.namenode.name.dir</name>
+ <value>file:///var/lib/hadoop-hdfs/cache/hdfs/dfs/name</value>
</property>
-
<property>
- <name>dfs.hosts.exclude</name>
- <value>/etc/hadoop/conf/dfs.exclude</value>
+ <name>dfs.namenode.servicerpc-address</name>
+ <value>quickstart.cloudera:8022</value>
</property>
-
<property>
- <name>dfs.http.policy</name>
- <value>HTTP_ONLY</value>
+ <name>dfs.https.address</name>
+ <value>quickstart.cloudera:50470</value>
</property>
-
<property>
<name>dfs.https.port</name>
<value>50470</value>
</property>
-
- <property>
- <name>dfs.journalnode.edits.dir</name>
- <value>/hadoop/hdfs/journalnode</value>
- </property>
-
- <property>
- <name>dfs.journalnode.http-address</name>
- <value>0.0.0.0:8480</value>
- </property>
-
- <property>
- <name>dfs.journalnode.https-address</name>
- <value>0.0.0.0:8481</value>
- </property>
-
- <property>
- <name>dfs.namenode.accesstime.precision</name>
- <value>3600000</value>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.read.stale.datanode</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.write.stale.datanode</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.dir</name>
- <value>/hadoop/hdfs/namesecondary</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.edits.dir</name>
- <value>${dfs.namenode.checkpoint.dir}</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.period</name>
- <value>21600</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.txns</name>
- <value>1000000</value>
- </property>
-
- <property>
- <name>dfs.namenode.handler.count</name>
- <value>100</value>
- </property>
-
<property>
<name>dfs.namenode.http-address</name>
- <value>sandbox.hortonworks.com:50070</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.namenode.https-address</name>
- <value>sandbox.hortonworks.com:50470</value>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir</name>
- <value>/hadoop/hdfs/namenode</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir.restore</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.safemode.threshold-pct</name>
- <value>1.0f</value>
+ <value>quickstart.cloudera:50070</value>
</property>
-
<property>
- <name>dfs.namenode.secondary.http-address</name>
- <value>sandbox.hortonworks.com:50090</value>
+ <name>dfs.replication</name>
+ <value>1</value>
</property>
-
<property>
- <name>dfs.namenode.stale.datanode.interval</name>
- <value>30000</value>
+ <name>dfs.blocksize</name>
+ <value>134217728</value>
</property>
-
<property>
- <name>dfs.namenode.startup.delay.block.deletion.sec</name>
- <value>3600</value>
+ <name>dfs.client.use.datanode.hostname</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.namenode.write.stale.datanode.ratio</name>
- <value>1.0f</value>
+ <name>fs.permissions.umask-mode</name>
+ <value>022</value>
</property>
-
<property>
- <name>dfs.nfs.exports.allowed.hosts</name>
- <value>* rw</value>
+ <name>dfs.namenode.acls.enabled</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.nfs3.dump.dir</name>
- <value>/tmp/.hdfs-nfs</value>
+ <name>dfs.client.use.legacy.blockreader</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.permissions.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.permissions.superusergroup</name>
- <value>hdfs</value>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.replication</name>
- <value>1</value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/run/hdfs-sockets/dn</value>
</property>
-
<property>
- <name>dfs.replication.max</name>
- <value>50</value>
+ <name>dfs.client.read.shortcircuit.skip.checksum</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.support.append</name>
- <value>true</value>
- <final>true</final>
+ <name>dfs.client.domain.socket.data.traffic</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.webhdfs.enabled</name>
+ <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
<value>true</value>
- <final>true</final>
</property>
-
- <property>
- <name>fs.permissions.umask-mode</name>
- <value>022</value>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index ff1c7eb..e8abf50 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -14,241 +14,169 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<configuration>
-
- <property>
- <name>io.sort.mb</name>
- <value>64</value>
- </property>
-
- <property>
- <name>mapred.child.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapred.job.map.memory.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>mapred.job.reduce.memory.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>mapreduce.admin.map.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>mapreduce.admin.reduce.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
- </property>
-
- <property>
- <name>mapreduce.am.max-attempts</name>
- <value>2</value>
- </property>
-
- <property>
- <name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
- </property>
-
- <property>
- <name>mapreduce.application.framework.path</name>
- <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
- </property>
-
- <property>
- <name>mapreduce.cluster.administrators</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>mapreduce.framework.name</name>
- <value>yarn</value>
- </property>
-
- <property>
- <name>mapreduce.job.emit-timeline-data</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>0.05</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.address</name>
- <value>sandbox.hortonworks.com:10020</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.done-dir</name>
- <value>/mr-history/done</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.intermediate-done-dir</name>
- <value>/mr-history/tmp</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.webapp.address</name>
- <value>sandbox.hortonworks.com:19888</value>
- </property>
-
- <property>
- <name>mapreduce.map.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapreduce.map.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>mapreduce.map.output.compress</name>
- <value>false</value>
- </property>
- <property>
- <name>mapreduce.map.sort.spill.percent</name>
- <value>0.7</value>
- </property>
- <property>
- <name>mapreduce.map.speculative</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress.type</name>
- <value>BLOCK</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.input.buffer.percent</name>
- <value>0.0</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
- <value>1</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
- <value>0.7</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.merge.percent</name>
- <value>0.66</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.parallelcopies</name>
- <value>30</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.speculative</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.shuffle.port</name>
- <value>13562</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.factor</name>
- <value>100</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.mb</name>
- <value>64</value>
- </property>
-
- <property>
- <name>mapreduce.task.timeout</name>
- <value>300000</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.admin-command-opts</name>
- <value>-Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.resource.mb</name>
- <value>250</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.staging-dir</name>
- <value>/user</value>
- </property>
+<configuration>
+<property>
+ <name>mapreduce.job.split.metainfo.maxsize</name>
+ <value>10000000</value>
+</property>
+<property>
+ <name>mapreduce.job.counters.max</name>
+ <value>120</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.DefaultCodec</value>
+</property>
+<property>
+ <name>mapreduce.map.output.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+</property>
+<property>
+ <name>mapreduce.map.output.compress</name>
+ <value>true</value>
+</property>
+<property>
+ <name>zlib.compress.level</name>
+ <value>DEFAULT_COMPRESSION</value>
+</property>
+<property>
+ <name>mapreduce.task.io.sort.factor</name>
+ <value>64</value>
+</property>
+<property>
+ <name>mapreduce.map.sort.spill.percent</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.reduce.shuffle.parallelcopies</name>
+ <value>10</value>
+</property>
+<property>
+ <name>mapreduce.task.timeout</name>
+ <value>600000</value>
+</property>
+<property>
+ <name>mapreduce.client.submit.file.replication</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.reduces</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.task.io.sort.mb</name>
+ <value>16</value>
+</property>
+<property>
+ <name>mapreduce.map.speculative</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.reduce.speculative</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.job.reduce.slowstart.completedmaps</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.address</name>
+ <value>quickstart.cloudera:10020</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>quickstart.cloudera:19888</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.webapp.https.address</name>
+ <value>quickstart.cloudera:19890</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.admin.address</name>
+ <value>quickstart.cloudera:10033</value>
+</property>
+<property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <value>/user</value>
+</property>
+<property>
+ <name>mapreduce.am.max-attempts</name>
+ <value>2</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.resource.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.ubertask.enable</name>
+ <value>false</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.command-opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>mapreduce.map.java.opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>mapreduce.reduce.java.opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
+</property>
+<property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>mapreduce.map.cpu.vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>mapreduce.reduce.cpu.vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.heap.memory-mb.ratio</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.application.classpath</name>
+ <value>/tmp/kylin/*,/usr/lib/hadoop-mapreduce/lib/*,/etc/hadoop/conf:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-mapreduce/.//*,/usr/lib/hbase/hbase-common.jar,/etc/hive/conf</value>
+</property>
+<property>
+ <name>mapreduce.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
+</property>
+<property>
+ <name>mapreduce.shuffle.max.connections</name>
+ <value>80</value>
+</property>
</configuration>
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index 8256158..8988d4a 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -15,520 +15,128 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>hadoop.registry.rm.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hadoop.registry.zk.quorum</name>
- <value>sandbox.hortonworks.com:2181</value>
- </property>
-
<property>
<name>yarn.acl.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.admin.acl</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.application.classpath</name>
- <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
- </property>
-
- <property>
- <name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.http.policy</name>
- <value>HTTP_ONLY</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation.retain-seconds</name>
- <value>2592000</value>
- </property>
-
- <property>
- <name>yarn.log.server.url</name>
- <value>http://sandbox.hortonworks.com:19888/jobhistory/logs</value>
- </property>
-
- <property>
- <name>yarn.node-labels.fs-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <name>yarn.node-labels.fs-store.root-dir</name>
- <value>/system/yarn/node-labels</value>
- </property>
-
- <property>
- <name>yarn.node-labels.manager-class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.address</name>
- <value>0.0.0.0:45454</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.admin-env</name>
- <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
- <value>org.apache.hadoop.mapred.ShuffleHandler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-executor.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-monitor.interval-ms</name>
- <value>3000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>0</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
- <value>90</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
- <value>0.25</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.interval-ms</name>
- <value>135000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
- <value>hadoop-yarn</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.group</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.local-dirs</name>
- <value>/hadoop/yarn/local</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.compression-type</name>
- <value>gz</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
- <value>30</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
- <value>-1</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-dirs</name>
- <value>/hadoop/yarn/log</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log.retain-second</name>
- <value>604800</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.pmem-check-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.recovery.dir</name>
- <value>/var/log/hadoop-yarn/nodemanager/recovery-state</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.recovery.enabled</name>
<value>true</value>
</property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>/app-logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
- <value>logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.cpu-vcores</name>
- <value>8</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>9216</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- </property>
-
<property>
- <name>yarn.nodemanager.vmem-pmem-ratio</name>
- <value>10</value>
+ <name>yarn.admin.acl</name>
+ <value>*</value>
</property>
-
<property>
<name>yarn.resourcemanager.address</name>
- <value>sandbox.hortonworks.com:8050</value>
+ <value>quickstart.cloudera:8032</value>
</property>
-
<property>
<name>yarn.resourcemanager.admin.address</name>
- <value>sandbox.hortonworks.com:8141</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.am.max-attempts</name>
- <value>2</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.connect.max-wait.ms</name>
- <value>900000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.connect.retry-interval.ms</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.fs.state-store.uri</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.ha.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.hostname</name>
- <value>sandbox.hortonworks.com</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.nodes.exclude-path</name>
- <value>/etc/hadoop/conf/yarn.exclude</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.recovery.enabled</name>
- <value>true</value>
+ <value>quickstart.cloudera:8033</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.resource-tracker.address</name>
- <value>sandbox.hortonworks.com:8025</value>
- </property>
-
<property>
<name>yarn.resourcemanager.scheduler.address</name>
- <value>sandbox.hortonworks.com:8030</value>
+ <value>quickstart.cloudera:8030</value>
</property>
-
<property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.state-store.max-completed-applications</name>
- <value>${yarn.resourcemanager.max-completed-applications}</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.store.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
- <value>10</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
- <value>true</value>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>quickstart.cloudera:8031</value>
</property>
-
<property>
<name>yarn.resourcemanager.webapp.address</name>
- <value>sandbox.hortonworks.com:8088</value>
+ <value>quickstart.cloudera:8088</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
- <value>false</value>
- </property>
-
<property>
<name>yarn.resourcemanager.webapp.https.address</name>
- <value>localhost:8090</value>
+ <value>quickstart.cloudera:8090</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.groups</name>
- <value>*</value>
+ <name>yarn.resourcemanager.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.hosts</name>
- <value>*</value>
+ <name>yarn.resourcemanager.scheduler.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.groups</name>
- <value>*</value>
+ <name>yarn.resourcemanager.admin.client.thread-count</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.hosts</name>
- <value>*</value>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
- <value>true</value>
+ <name>yarn.scheduler.increment-allocation-mb</name>
+ <value>512</value>
</property>
-
<property>
- <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
- <value>10000</value>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>2816</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-acl</name>
- <value>world:anyone:rwcda</value>
+ <name>yarn.scheduler.minimum-allocation-vcores</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-address</name>
- <value>localhost:2181</value>
+ <name>yarn.scheduler.increment-allocation-vcores</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-num-retries</name>
- <value>1000</value>
+ <name>yarn.scheduler.maximum-allocation-vcores</name>
+ <value>2</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-retry-interval-ms</name>
+ <name>yarn.resourcemanager.amliveliness-monitor.interval-ms</name>
<value>1000</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-state-store.parent-path</name>
- <value>/rmstore</value>
+ <name>yarn.am.liveness-monitor.expiry-interval-ms</name>
+ <value>600000</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.zk-timeout-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-mb</name>
- <value>9216</value>
- </property>
-
<property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1536</value>
+ <name>yarn.resourcemanager.am.max-attempts</name>
+ <value>2</value>
</property>
-
<property>
- <name>yarn.timeline-service.address</name>
- <value>sandbox.hortonworks.com:10200</value>
+ <name>yarn.resourcemanager.container.liveness-monitor.interval-ms</name>
+ <value>600000</value>
</property>
-
<property>
- <name>yarn.timeline-service.bind-host</name>
- <value>0.0.0.0</value>
+ <name>yarn.resourcemanager.nm.liveness-monitor.interval-ms</name>
+ <value>1000</value>
</property>
-
<property>
- <name>yarn.timeline-service.client.max-retries</name>
- <value>30</value>
+ <name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
+ <value>600000</value>
</property>
-
<property>
- <name>yarn.timeline-service.client.retry-interval-ms</name>
- <value>1000</value>
+ <name>yarn.resourcemanager.resource-tracker.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.timeline-service.enabled</name>
- <value>true</value>
+ <name>yarn.application.classpath</name>
+ <value>$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*</value>
</property>
-
<property>
- <name>yarn.timeline-service.generic-application-history.store-class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>
-
<property>
- <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+ <name>yarn.scheduler.fair.user-as-default-queue</name>
<value>true</value>
</property>
-
- <property>
- <name>yarn.timeline-service.http-authentication.type</name>
- <value>simple</value>
- </property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.path</name>
- <value>/hadoop/yarn/timeline</value>
+ <name>yarn.scheduler.fair.preemption</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
- <value>104857600</value>
+ <name>yarn.scheduler.fair.sizebasedweight</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
- <value>10000</value>
+ <name>yarn.scheduler.fair.assignmultiple</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
+ <name>yarn.resourcemanager.max-completed-applications</name>
<value>10000</value>
</property>
-
- <property>
- <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
- <value>300000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-ms</name>
- <value>2678400000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.address</name>
- <value>sandbox.hortonworks.com:8188</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.https.address</name>
- <value>sandbox.hortonworks.com:8190</value>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 93b6b46..6fa1124 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,19 +46,19 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.7.1</hadoop2.version>
- <yarn.version>2.7.1</yarn.version>
+ <hadoop2.version>2.6.0-cdh5.7.0</hadoop2.version>
+ <yarn.version>2.6.0-cdh5.7.0</yarn.version>
<!-- Hive versions -->
- <hive.version>1.2.1</hive.version>
- <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
+ <hive.version>1.1.0-cdh5.7.0</hive.version>
+ <hive-hcatalog.version>1.1.0-cdh5.7.0</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
- <kafka.version>0.8.1</kafka.version>
+ <hbase-hadoop2.version>1.2.0-cdh5.7.0</hbase-hadoop2.version>
+ <kafka.version>0.8.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
- <zookeeper.version>3.4.6</zookeeper.version>
+ <zookeeper.version>3.4.5-cdh5.7.0</zookeeper.version>
<curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
@@ -811,6 +811,10 @@
<id>conjars</id>
<url>http://conjars.org/repo/</url>
</repository>
+ <repository>
+ <id>cloudera</id>
+ <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+ </repository>
</repositories>
<build>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index abe10db..370fe27 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -112,6 +112,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
@@ -131,6 +135,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -146,6 +154,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -161,6 +173,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -176,6 +192,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -199,6 +219,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -214,6 +238,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -229,6 +257,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -262,6 +294,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
index d5c3f60..5adf327 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
@@ -100,11 +100,6 @@ public class MockupMapContext {
}
@Override
- public float getProgress() {
- throw new NotImplementedException();
- }
-
- @Override
public Counter getCounter(Enum<?> counterName) {
throw new NotImplementedException();
}
@@ -165,6 +160,11 @@ public class MockupMapContext {
}
@Override
+ public boolean userClassesTakesPrecedence() {
+ return false;
+ }
+
+ @Override
public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
throw new NotImplementedException();
}
@@ -214,10 +214,6 @@ public class MockupMapContext {
throw new NotImplementedException();
}
- @Override
- public boolean getTaskCleanupNeeded() {
- throw new NotImplementedException();
- }
@Override
public boolean getProfileEnabled() {
@@ -230,11 +226,6 @@ public class MockupMapContext {
}
@Override
- public IntegerRanges getProfileTaskRange(boolean isMap) {
- throw new NotImplementedException();
- }
-
- @Override
public String getUser() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7d02166/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index a290d68..ad46cba 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -45,6 +45,18 @@
<!--Env-->
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${yarn.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${yarn.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<scope>provided</scope>
[03/50] [abbrv] kylin git commit: minor, fix bug in kylin.sh refactor
Posted by li...@apache.org.
minor, fix bug in kylin.sh refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ce1cde0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ce1cde0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ce1cde0
Branch: refs/heads/1.5.x-CDH5.7
Commit: 2ce1cde0ab18f1b3541498584543394123f91efb
Parents: 29ba46b
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Aug 22 10:35:09 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Aug 22 10:35:09 2016 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 49 +++++++++++++++++--------------------------------
1 file changed, 17 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2ce1cde0/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 88f9296..365e1fe 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -25,9 +25,17 @@ dir="$KYLIN_HOME/bin"
source ${dir}/check-env.sh
mkdir -p ${KYLIN_HOME}/logs
+mkdir -p ${KYLIN_HOME}/ext
+#retrive $hive_dependency and $hbase_dependency
+source ${dir}/find-hive-dependency.sh
+source ${dir}/find-hbase-dependency.sh
+
+#retrive $KYLIN_EXTRA_START_OPTS
+if [ -f "${dir}/setenv.sh" ]
+ then source ${dir}/setenv.sh
+fi
-mkdir -p ${KYLIN_HOME}/ext
export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
@@ -67,14 +75,6 @@ then
echo "kylin.security.profile is set to $spring_profile"
fi
- #retrive $hive_dependency and $hbase_dependency
- source ${dir}/find-hive-dependency.sh
- source ${dir}/find-hbase-dependency.sh
- #retrive $KYLIN_EXTRA_START_OPTS
- if [ -f "${dir}/setenv.sh" ]
- then source ${dir}/setenv.sh
- fi
-
#additionally add tomcat libs to HBASE_CLASSPATH_PREFIX
export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:${HBASE_CLASSPATH_PREFIX}
@@ -143,16 +143,9 @@ then
fi
if [ "$2" == "start" ]
then
-
- #retrive $hive_dependency and $hbase_dependency
- source ${dir}/find-hive-dependency.sh
- source ${dir}/find-hbase-dependency.sh
+
source ${dir}/find-kafka-dependency.sh
- #retrive $KYLIN_EXTRA_START_OPTS
- if [ -f "${dir}/setenv.sh" ]
- then source ${dir}/setenv.sh
- fi
-
+
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
hbase ${KYLIN_EXTRA_START_OPTS} \
-Dlog4j.configuration=kylin-log4j.properties\
@@ -189,27 +182,20 @@ elif [ "$1" == "monitor" ]
then
echo "monitor job"
- #retrive $hive_dependency and $hbase_dependency
- source ${dir}/find-hive-dependency.sh
- source ${dir}/find-hbase-dependency.sh
source ${dir}/find-kafka-dependency.sh
- #retrive $KYLIN_EXTRA_START_OPTS
- if [ -f "${dir}/setenv.sh" ]
- then source ${dir}/setenv.sh
- fi
-
+
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
hbase ${KYLIN_EXTRA_START_OPTS} \
-Dlog4j.configuration=kylin-log4j.properties\
-Dkylin.hive.dependency=${hive_dependency} \
- -Dkyiln.kafka.dependency=${kafka_dependency} \
+ -Dkylin.kafka.dependency=${kafka_dependency} \
-Dkylin.hbase.dependency=${hbase_dependency} \
org.apache.kylin.engine.streaming.cli.MonitorCLI $@ > ${KYLIN_HOME}/logs/monitor.log 2>&1
exit 0
elif [ "$1" = "version" ]
then
- exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.common.KylinVersion
+ exec hbase -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.common.KylinVersion
exit 0
elif [ "$1" = "diag" ]
@@ -220,10 +206,9 @@ then
# tool command
elif [[ "$1" = org.apache.kylin.* ]]
then
- #retrive $hive_dependency and $hbase_dependency
- source ${dir}/find-hive-dependency.sh
- source ${dir}/find-hbase-dependency.sh
- #retrive $KYLIN_EXTRA_START_OPTS
+
+ #retrive $KYLIN_EXTRA_START_OPTS from a separate file called setenv-tool.sh
+ reset KYLIN_EXTRA_START_OPTS # reset the global server setenv config first
if [ -f "${dir}/setenv-tool.sh" ]
then source ${dir}/setenv-tool.sh
fi
[47/50] [abbrv] kylin git commit: KYLIN-1834 dict offset can be upto
5 bytes, trie dict is 2 GB at most
Posted by li...@apache.org.
KYLIN-1834 dict offset can be upto 5 bytes, trie dict is 2 GB at most
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/67bcec20
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/67bcec20
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/67bcec20
Branch: refs/heads/1.5.x-CDH5.7
Commit: 67bcec20fb6fee502a5267dfa583bd07ca5edc8b
Parents: 7dae977
Author: Yang Li <li...@apache.org>
Authored: Sat Sep 3 13:13:00 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Sep 3 13:13:00 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/BytesUtil.java | 2 +-
.../org/apache/kylin/dict/TrieDictionary.java | 30 +++---
.../kylin/dict/TrieDictionaryBuilder.java | 105 +++++++------------
.../apache/kylin/dict/TrieDictionaryTest.java | 74 ++++++++++---
4 files changed, 121 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/67bcec20/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index bda5c73..759ddbd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -98,7 +98,7 @@ public class BytesUtil {
/**
* No. bytes needed to store a value as big as the given
*/
- public static int sizeForValue(int maxValue) {
+ public static int sizeForValue(long maxValue) {
int size = 0;
while (maxValue > 0) {
size++;
http://git-wip-us.apache.org/repos/asf/kylin/blob/67bcec20/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 03dc76a..aea9551 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -58,8 +58,8 @@ import com.google.common.base.Preconditions;
public class TrieDictionary<T> extends Dictionary<T> {
private static final long serialVersionUID = 1L;
- public static final byte[] HEAD_MAGIC = new byte[] { 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "TrieDict"
- public static final int HEAD_SIZE_I = HEAD_MAGIC.length;
+ public static final byte[] MAGIC = new byte[] { 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74 }; // "TrieDict"
+ public static final int MAGIC_SIZE_I = MAGIC.length;
public static final int BIT_IS_LAST_CHILD = 0x80;
public static final int BIT_IS_END_OF_VALUE = 0x40;
@@ -80,7 +80,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
transient private int nValues;
transient private int sizeOfId;
- transient private int childOffsetMask;
+ transient private long childOffsetMask;
transient private int firstByteOffset;
transient private boolean enableValueCache = true;
@@ -99,12 +99,12 @@ public class TrieDictionary<T> extends Dictionary<T> {
private void init(byte[] trieBytes) {
this.trieBytes = trieBytes;
- if (BytesUtil.compareBytes(HEAD_MAGIC, 0, trieBytes, 0, HEAD_MAGIC.length) != 0)
+ if (BytesUtil.compareBytes(MAGIC, 0, trieBytes, 0, MAGIC.length) != 0)
throw new IllegalArgumentException("Wrong file type (magic does not match)");
try {
DataInputStream headIn = new DataInputStream(//
- new ByteArrayInputStream(trieBytes, HEAD_SIZE_I, trieBytes.length - HEAD_SIZE_I));
+ new ByteArrayInputStream(trieBytes, MAGIC_SIZE_I, trieBytes.length - MAGIC_SIZE_I));
this.headSize = headIn.readShort();
this.bodyLen = headIn.readInt();
this.sizeChildOffset = headIn.read();
@@ -118,7 +118,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
this.nValues = BytesUtil.readUnsigned(trieBytes, headSize + sizeChildOffset, sizeNoValuesBeneath);
this.sizeOfId = BytesUtil.sizeForValue(baseId + nValues + 1); // note baseId could raise 1 byte in ID space, +1 to reserve all 0xFF for NULL case
- this.childOffsetMask = ~((BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
+ this.childOffsetMask = ~((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE) << ((sizeChildOffset - 1) * 8));
this.firstByteOffset = sizeChildOffset + sizeNoValuesBeneath + 1; // the offset from begin of node to its first value byte
} catch (Exception e) {
if (e instanceof RuntimeException)
@@ -229,7 +229,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
seq++;
// find a child to continue
- int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ int c = getChildOffset(n);
if (c == headSize) // has no children
return roundSeqNo(roundingFlag, seq - 1, -1, seq); // input only partially matched
byte inpByte = inp[o];
@@ -253,6 +253,12 @@ public class TrieDictionary<T> extends Dictionary<T> {
}
}
+ private int getChildOffset(int n) {
+ long offset = headSize + (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ assert offset < trieBytes.length;
+ return (int) offset;
+ }
+
private int roundSeqNo(int roundingFlag, int i, int j, int k) {
if (roundingFlag == 0)
return j;
@@ -338,7 +344,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
}
// find a child to continue
- int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ int c = getChildOffset(n);
if (c == headSize) // has no children
return -1; // no child? corrupted dictionary!
int nValuesBeneath;
@@ -401,7 +407,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
}
// find a child to continue
- int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ int c = getChildOffset(n);
if (c == headSize) // has no children
return;
@@ -446,14 +452,14 @@ public class TrieDictionary<T> extends Dictionary<T> {
@Override
public void readFields(DataInput in) throws IOException {
- byte[] headPartial = new byte[HEAD_MAGIC.length + Short.SIZE + Integer.SIZE];
+ byte[] headPartial = new byte[MAGIC.length + Short.SIZE + Integer.SIZE];
in.readFully(headPartial);
- if (BytesUtil.compareBytes(HEAD_MAGIC, 0, headPartial, 0, HEAD_MAGIC.length) != 0)
+ if (BytesUtil.compareBytes(MAGIC, 0, headPartial, 0, MAGIC.length) != 0)
throw new IllegalArgumentException("Wrong file type (magic does not match)");
DataInputStream headIn = new DataInputStream(//
- new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
+ new ByteArrayInputStream(headPartial, MAGIC_SIZE_I, headPartial.length - MAGIC_SIZE_I));
int headSize = headIn.readShort();
int bodyLen = headIn.readInt();
headIn.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/67bcec20/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index 02da741..1271483 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -38,6 +38,8 @@ import org.apache.kylin.common.util.BytesUtil;
* @author yangli9
*/
public class TrieDictionaryBuilder<T> {
+
+ private static final int _2GB = 2000000000;
public static class Node {
public byte[] part;
@@ -112,8 +114,7 @@ public class TrieDictionaryBuilder<T> {
return;
}
- // if partially matched the current, split the current node, add the new
- // value, make a 3-way
+ // if partially matched the current, split the current node, add the new value, make a 3-way
if (i < n) {
Node c1 = new Node(BytesUtil.subarray(node.part, i, n), node.isEndOfValue, node.children);
Node c2 = new Node(BytesUtil.subarray(value, j, nn), true);
@@ -128,8 +129,7 @@ public class TrieDictionaryBuilder<T> {
return;
}
- // out matched the current, binary search the next byte for a child node
- // to continue
+ // out matched the current, binary search the next byte for a child node to continue
byte lookfor = value[j];
int lo = 0;
int hi = node.children.size() - 1;
@@ -188,28 +188,21 @@ public class TrieDictionaryBuilder<T> {
public int mbpn_nNodes; // number of nodes in trie
public int mbpn_trieDepth; // depth of trie
public int mbpn_maxFanOut; // the maximum no. children
- public int mbpn_nChildLookups; // number of child lookups during lookup
- // every value once
- public int mbpn_nTotalFanOut; // the sum of fan outs during lookup every
- // value once
+ public long mbpn_nChildLookups; // number of child lookups during lookup every value once
+ public long mbpn_nTotalFanOut; // the sum of fan outs during lookup every value once
public int mbpn_sizeValueTotal; // the sum of value space in all nodes
public int mbpn_sizeNoValueBytes; // size of field noValueBytes
- public int mbpn_sizeNoValueBeneath; // size of field noValuesBeneath,
- // depends on cardinality
- public int mbpn_sizeChildOffset; // size of field childOffset, points to
- // first child in flattened array
- public int mbpn_footprint; // MBPN footprint in bytes
+ public int mbpn_sizeNoValueBeneath; // size of field noValuesBeneath, depends on cardinality
+ public int mbpn_sizeChildOffset; // size of field childOffset, points to first child in flattened array
+ public long mbpn_footprint; // MBPN footprint in bytes
// stats for one-byte-per-node as well, so there's comparison
public int obpn_sizeValue; // size of value per node, always 1
- public int obpn_sizeNoValuesBeneath; // size of field noValuesBeneath,
- // depends on cardinality
- public int obpn_sizeChildCount; // size of field childCount, enables
- // binary search among children
- public int obpn_sizeChildOffset; // size of field childOffset, points to
- // first child in flattened array
+ public int obpn_sizeNoValuesBeneath; // size of field noValuesBeneath, depends on cardinality
+ public int obpn_sizeChildCount; // size of field childCount, enables binary search among children
+ public int obpn_sizeChildOffset; // size of field childOffset, points to first child in flattened array
public int obpn_nNodes; // no. nodes in OBPN trie
- public int obpn_footprint; // OBPN footprint in bytes
+ public long obpn_footprint; // OBPN footprint in bytes
public void print() {
PrintStream out = System.out;
@@ -289,23 +282,12 @@ public class TrieDictionaryBuilder<T> {
s.obpn_sizeValue = 1;
s.obpn_sizeNoValuesBeneath = BytesUtil.sizeForValue(s.nValues);
s.obpn_sizeChildCount = 1;
- s.obpn_sizeChildOffset = 4; // MSB used as isEndOfValue flag
- s.obpn_nNodes = s.nValueBytesCompressed; // no. nodes is the total
- // number of compressed
- // bytes in OBPN
- s.obpn_footprint = s.obpn_nNodes * (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset);
+ s.obpn_sizeChildOffset = 5; // MSB used as isEndOfValue flag
+ s.obpn_nNodes = s.nValueBytesCompressed; // no. nodes is the total number of compressed bytes in OBPN
+ s.obpn_footprint = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset);
while (true) { // minimize the offset size to match the footprint
- int t = s.obpn_nNodes * (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset - 1);
- if (BytesUtil.sizeForValue(t * 2) <= s.obpn_sizeChildOffset - 1) { // *2
- // because
- // MSB
- // of
- // offset
- // is
- // used
- // for
- // isEndOfValue
- // flag
+ long t = s.obpn_nNodes * (long) (s.obpn_sizeValue + s.obpn_sizeNoValuesBeneath + s.obpn_sizeChildCount + s.obpn_sizeChildOffset - 1);
+ if (BytesUtil.sizeForValue(t * 2) <= s.obpn_sizeChildOffset - 1) { // *2 because MSB of offset is used for isEndOfValue flag
s.obpn_sizeChildOffset--;
s.obpn_footprint = t;
} else
@@ -316,23 +298,11 @@ public class TrieDictionaryBuilder<T> {
s.mbpn_sizeValueTotal = s.nValueBytesCompressed;
s.mbpn_sizeNoValueBytes = 1;
s.mbpn_sizeNoValueBeneath = BytesUtil.sizeForValue(s.nValues);
- s.mbpn_sizeChildOffset = 4;
- s.mbpn_footprint = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset);
+ s.mbpn_sizeChildOffset = 5;
+ s.mbpn_footprint = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset);
while (true) { // minimize the offset size to match the footprint
- int t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset - 1);
- if (BytesUtil.sizeForValue(t * 4) <= s.mbpn_sizeChildOffset - 1) { // *4
- // because
- // 2
- // MSB
- // of
- // offset
- // is
- // used
- // for
- // isEndOfValue
- // &
- // isEndChild
- // flag
+ long t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (long) (s.mbpn_sizeNoValueBytes + s.mbpn_sizeNoValueBeneath + s.mbpn_sizeChildOffset - 1);
+ if (BytesUtil.sizeForValue(t * 4) <= s.mbpn_sizeChildOffset - 1) { // *4 because 2 MSB of offset is used for isEndOfValue & isEndChild flag
s.mbpn_sizeChildOffset--;
s.mbpn_footprint = t;
} else
@@ -415,8 +385,7 @@ public class TrieDictionaryBuilder<T> {
}
}
- completeParts.append(node.part);// by here the node.children may have
- // been changed
+ completeParts.append(node.part); // by here the node.children may have been changed
for (Node child : node.children) {
checkOverflowParts(child);
}
@@ -427,11 +396,13 @@ public class TrieDictionaryBuilder<T> {
* Flatten the trie into a byte array for a minimized memory footprint.
* Lookup remains fast. Cost is inflexibility to modify (becomes immutable).
*
- * Flattened node structure is HEAD + NODEs, for each node: - o byte, offset
- * to child node, o = stats.mbpn_sizeChildOffset - 1 bit, isLastChild flag,
- * the 1st MSB of o - 1 bit, isEndOfValue flag, the 2nd MSB of o - c byte,
- * number of values beneath, c = stats.mbpn_sizeNoValueBeneath - 1 byte,
- * number of value bytes - n byte, value bytes
+ * Flattened node structure is HEAD + NODEs, for each node:
+ * - o byte, offset to child node, o = stats.mbpn_sizeChildOffset
+ * - 1 bit, isLastChild flag, the 1st MSB of o
+ * - 1 bit, isEndOfValue flag, the 2nd MSB of o
+ * - c byte, number of values beneath, c = stats.mbpn_sizeNoValueBeneath
+ * - 1 byte, number of value bytes
+ * - n byte, value bytes
*/
public TrieDictionary<T> build(int baseId) {
byte[] trieBytes = buildTrieBytes(baseId);
@@ -445,15 +416,18 @@ public class TrieDictionaryBuilder<T> {
Stats stats = stats();
int sizeNoValuesBeneath = stats.mbpn_sizeNoValueBeneath;
int sizeChildOffset = stats.mbpn_sizeChildOffset;
+
+ if (stats.mbpn_footprint > _2GB)
+ throw new RuntimeException("Too big dictionary, dictionary cannot be bigger than 2GB");
// write head
byte[] head;
try {
ByteArrayOutputStream byteBuf = new ByteArrayOutputStream();
DataOutputStream headOut = new DataOutputStream(byteBuf);
- headOut.write(TrieDictionary.HEAD_MAGIC);
+ headOut.write(TrieDictionary.MAGIC);
headOut.writeShort(0); // head size, will back fill
- headOut.writeInt(stats.mbpn_footprint); // body size
+ headOut.writeInt((int) stats.mbpn_footprint); // body size
headOut.write(sizeChildOffset);
headOut.write(sizeNoValuesBeneath);
headOut.writeShort(baseId);
@@ -461,13 +435,12 @@ public class TrieDictionaryBuilder<T> {
headOut.writeUTF(bytesConverter == null ? "" : bytesConverter.getClass().getName());
headOut.close();
head = byteBuf.toByteArray();
- BytesUtil.writeUnsigned(head.length, head, TrieDictionary.HEAD_SIZE_I, 2);
+ BytesUtil.writeUnsigned(head.length, head, TrieDictionary.MAGIC_SIZE_I, 2);
} catch (IOException e) {
- throw new RuntimeException(e); // shall not happen, as we are
- // writing in memory
+ throw new RuntimeException(e); // shall not happen, as we are writing in memory
}
- byte[] trieBytes = new byte[stats.mbpn_footprint + head.length];
+ byte[] trieBytes = new byte[(int) stats.mbpn_footprint + head.length];
System.arraycopy(head, 0, trieBytes, 0, head.length);
LinkedList<Node> open = new LinkedList<Node>();
@@ -506,6 +479,8 @@ public class TrieDictionaryBuilder<T> {
private int build_writeNode(Node n, int offset, boolean isLastChild, int sizeNoValuesBeneath, int sizeChildOffset, byte[] trieBytes) {
int o = offset;
+ if (o > _2GB)
+ throw new IllegalStateException();
// childOffset
if (isLastChild)
http://git-wip-us.apache.org/repos/asf/kylin/blob/67bcec20/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
index 90283b8..a87d7cb 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
@@ -32,10 +32,10 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.NoSuchElementException;
import java.util.Random;
import java.util.TreeSet;
@@ -44,12 +44,61 @@ import org.junit.Test;
public class TrieDictionaryTest {
public static void main(String[] args) throws Exception {
- InputStream is = new FileInputStream("src/test/resources/dict/dw_category_grouping_names.dat");
- // InputStream is =
- // Util.getPackageResourceAsStream(TrieDictionaryTest.class,
- // "eng_com.dic");
- ArrayList<String> str = loadStrings(is);
- benchmarkStringDictionary(str);
+ int count = (int) (Integer.MAX_VALUE * 0.8 / 64);
+ benchmarkStringDictionary(new RandomStrings(count));
+ }
+
+ private static class RandomStrings implements Iterable<String> {
+ final private int size;
+
+ public RandomStrings(int size) {
+ this.size = size;
+ System.out.println("size = " + size);
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return new Iterator<String>() {
+ Random rand = new Random(1000);
+ int i = 0;
+
+ @Override
+ public boolean hasNext() {
+ return i < size;
+ }
+
+ @Override
+ public String next() {
+ if (hasNext() == false)
+ throw new NoSuchElementException();
+
+ i++;
+ if (i % 1000000 == 0)
+ System.out.println(i);
+
+ return nextString();
+ }
+
+ private String nextString() {
+ StringBuffer buf = new StringBuffer();
+ for (int i = 0; i < 64; i++) {
+ int v = rand.nextInt(16);
+ char c;
+ if (v >= 0 && v <= 9)
+ c = (char) ('0' + v);
+ else
+ c = (char) ('a' + v - 10);
+ buf.append(c);
+ }
+ return buf.toString();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
}
@Test
@@ -172,11 +221,11 @@ public class TrieDictionaryTest {
testStringDictionary(str, null);
}
- private static void benchmarkStringDictionary(ArrayList<String> str) throws UnsupportedEncodingException {
+ private static void benchmarkStringDictionary(Iterable<String> str) throws IOException {
TrieDictionaryBuilder<String> b = newDictBuilder(str);
b.stats().print();
TrieDictionary<String> dict = b.build(0);
-
+
TreeSet<String> set = new TreeSet<String>();
for (String s : str) {
set.add(s);
@@ -205,13 +254,14 @@ public class TrieDictionaryTest {
// following jvm options may help
// -XX:CompileThreshold=1500
// -XX:+PrintCompilation
+ System.out.println("Benchmark awaitig...");
benchmark("Warm up", dict, set, map, strArray, array);
benchmark("Benchmark", dict, set, map, strArray, array);
}
private static int benchmark(String msg, TrieDictionary<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) {
int n = set.size();
- int times = 10 * 1000 * 1000 / n; // run 10 million lookups
+ int times = Math.max(10 * 1000 * 1000 / n, 1); // run 10 million lookups
int keep = 0; // make sure JIT don't OPT OUT function calls under test
byte[] valueBytes = new byte[dict.getSizeOfValue()];
long start;
@@ -259,7 +309,7 @@ public class TrieDictionaryTest {
}
long timeIdToValueByDict = System.currentTimeMillis() - start;
System.out.println(timeIdToValueByDict);
-
+
return keep;
}
@@ -322,7 +372,7 @@ public class TrieDictionaryTest {
}
}
- private static TrieDictionaryBuilder<String> newDictBuilder(ArrayList<String> str) {
+ private static TrieDictionaryBuilder<String> newDictBuilder(Iterable<String> str) {
TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
for (String s : str)
b.addValue(s);
[44/50] [abbrv] kylin git commit: KYLIN-1698 fix UT
Posted by li...@apache.org.
KYLIN-1698 fix UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/111e984f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/111e984f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/111e984f
Branch: refs/heads/1.5.x-CDH5.7
Commit: 111e984f9c3a47a8a90cc622ef368cf68b0f84fe
Parents: a20d709
Author: Li Yang <li...@apache.org>
Authored: Fri Sep 2 16:11:14 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Sep 2 16:11:14 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/metadata/model/PartitionDesc.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/111e984f/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 8d3979e..598e6d0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -92,6 +92,9 @@ public class PartitionDesc {
}
public boolean partitionColumnIsTimeMillis() {
+ if (partitionDateColumnRef == null)
+ return false;
+
DataType type = partitionDateColumnRef.getType();
return type.isBigInt();
}