You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/10 09:03:35 UTC
[01/17] kylin git commit: minor, import correct Lists [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2428 51db9d2a8 -> 7c2632d96 (forced update)
minor, import correct Lists
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0dcce15f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0dcce15f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0dcce15f
Branch: refs/heads/KYLIN-2428
Commit: 0dcce15f7fc01f6d8c388661025e5dab99ef05ad
Parents: 39afa51
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Feb 8 15:18:10 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Feb 8 15:18:10 2017 +0800
----------------------------------------------------------------------
kylin-it/src/test/java/org/apache/kylin/query/H2Database.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0dcce15f/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
index 8b60c49..78ed1b6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import com.clearspring.analytics.util.Lists;
+import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.MetadataManager;
[09/17] kylin git commit: minor,filter inDimNotInMea add return
Posted by bi...@apache.org.
minor,filter inDimNotInMea add return
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/abf53322
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/abf53322
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/abf53322
Branch: refs/heads/KYLIN-2428
Commit: abf53322527148eaa49020bc2a327a210308e62d
Parents: 1e9f090
Author: luguosheng <55...@qq.com>
Authored: Thu Feb 9 16:14:56 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Feb 9 16:17:49 2017 +0800
----------------------------------------------------------------------
webapp/app/js/filters/filter.js | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/abf53322/webapp/app/js/filters/filter.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index d097075..01c060a 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -202,6 +202,7 @@ KylinApp
});
}
});
+ return out;
}
}).filter('assignedMeasureNames', function ($filter) {
//return the measures that haven't assign to column family
[14/17] kylin git commit: KYLIN-2441 protocol for REST API result
format
Posted by bi...@apache.org.
KYLIN-2441 protocol for REST API result format
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/41332f3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/41332f3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/41332f3e
Branch: refs/heads/KYLIN-2428
Commit: 41332f3ed430b095efe5edefc831f0ec27f4b2ee
Parents: f2e8b69
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Feb 10 12:34:49 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Feb 10 12:35:14 2017 +0800
----------------------------------------------------------------------
.../rest/controller/EncodingController.java | 3 +-
.../rest/exception/BadRequestException.java | 37 +++++++++-----------
.../kylin/rest/response/EnvelopeResponse.java | 34 ++++--------------
.../kylin/rest/response/ErrorResponse.java | 32 ++++++++++++-----
.../kylin/rest/response/ResponseCode.java | 30 ++++++++++++++++
5 files changed, 79 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/41332f3e/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
index b95394c..4a8b122 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.response.ResponseCode;
import org.apache.kylin.rest.service.EncodingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,6 +66,6 @@ public class EncodingController extends BasicController {
datatypeValidEncodings.put(dataTypeStr, encodingService.getValidEncodings(DataType.getType(dataTypeStr)));
}
- return new EnvelopeResponse(EnvelopeResponse.CODE_SUCCESS, datatypeValidEncodings, "");
+ return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, datatypeValidEncodings, "");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/41332f3e/server-base/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java b/server-base/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java
index 42b671b..af1995b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/exception/BadRequestException.java
@@ -18,6 +18,7 @@
package org.apache.kylin.rest.exception;
+import org.apache.kylin.rest.response.ResponseCode;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@@ -27,35 +28,31 @@ import org.springframework.web.bind.annotation.ResponseStatus;
*/
@ResponseStatus(value = HttpStatus.BAD_REQUEST)
public class BadRequestException extends RuntimeException {
- /**
- *
- */
+
private static final long serialVersionUID = -6798154278095441848L;
- public BadRequestException(String s) {
- super(s);
- }
+ private String code;
/**
- *
+ * legacy support, new APIs should not call this. Instead, new APIs should provide return code
*/
- public BadRequestException() {
- super();
+ public BadRequestException(String msg) {
+ super(msg);
+ this.code = ResponseCode.CODE_UNDEFINED;
}
- /**
- * @param arg0
- * @param arg1
- */
- public BadRequestException(String arg0, Throwable arg1) {
- super(arg0, arg1);
+ public BadRequestException(String msg, String code) {
+ super(msg);
+ this.code = code;
}
- /**
- * @param arg0
- */
- public BadRequestException(Throwable arg0) {
- super(arg0);
+ public BadRequestException(String msg, String code, Throwable cause) {
+ super(msg, cause);
+ this.code = code;
+ }
+
+ public String getCode() {
+ return code;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/41332f3e/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
index 564db70..7855dee 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
@@ -20,39 +20,17 @@ package org.apache.kylin.rest.response;
public class EnvelopeResponse {
- public final static String CODE_SUCCESS = "000";
+ public String code;
+ public Object data;
+ public String msg;
- private String code;
- private Object data;
- private String msg;
-
- public EnvelopeResponse(String code, Object data, String msg) {
- this.code = code;
- this.data = data;
- this.msg = msg;
- }
-
- public String getCode() {
- return code;
+ //only for child
+ protected EnvelopeResponse() {
}
- public void setCode(String code) {
+ public EnvelopeResponse(String code, Object data, String msg) {
this.code = code;
- }
-
- public Object getData() {
- return data;
- }
-
- public void setData(Object data) {
this.data = data;
- }
-
- public String getMsg() {
- return msg;
- }
-
- public void setMsg(String msg) {
this.msg = msg;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/41332f3e/server-base/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java
index 3327cc0..508a35f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/ErrorResponse.java
@@ -18,21 +18,37 @@
package org.apache.kylin.rest.response;
+import org.apache.kylin.rest.exception.BadRequestException;
+
+import com.google.common.base.Throwables;
+
/**
- * @author xduo
- *
+ * response to client when the return HTTP code is not 200
*/
-public class ErrorResponse {
+public class ErrorResponse extends EnvelopeResponse {
- public String url;
+ //stacktrace of the exception
+ public String stacktrace;
+
+ //same as EnvelopeResponse.msg, kept for legacy reasons
public String exception;
- /**
- * @param exception
- */
+ //request URL, kept from legacy codes
+ public String url;
+
public ErrorResponse(String url, Exception exception) {
+ super();
+
this.url = url;
this.exception = exception.getLocalizedMessage();
- }
+ this.msg = exception.getLocalizedMessage();
+ this.stacktrace = Throwables.getStackTraceAsString(exception);
+ this.data = null;
+ if (exception instanceof BadRequestException) {
+ this.code = ((BadRequestException) exception).getCode();
+ } else {
+ this.code = ResponseCode.CODE_UNDEFINED;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/41332f3e/server-base/src/main/java/org/apache/kylin/rest/response/ResponseCode.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/ResponseCode.java b/server-base/src/main/java/org/apache/kylin/rest/response/ResponseCode.java
new file mode 100644
index 0000000..8c3860a
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/ResponseCode.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.response;
+
+/**
+ * It's not HTTP return code!
+ * It represents code for business states
+ * each API can specify different semantics to each code
+ * 000 and 999 are reserved
+ */
+public class ResponseCode {
+ public final static String CODE_SUCCESS = "000";
+ public final static String CODE_UNDEFINED = "999";
+}
[06/17] kylin git commit: KYLIN-2222 web ui uses rest api to decide
which dim encoding is valid for different typed columns
Posted by bi...@apache.org.
KYLIN-2222 web ui uses rest api to decide which dim encoding is valid for different typed columns
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/722efb82
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/722efb82
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/722efb82
Branch: refs/heads/KYLIN-2428
Commit: 722efb82357e0ebcf7853a813272bd960044dd52
Parents: 570ab42
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Feb 8 21:41:41 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Feb 8 21:41:57 2017 +0800
----------------------------------------------------------------------
.../rest/controller/EncodingController.java | 73 ++++++++++++++++++++
.../kylin/rest/service/EncodingService.java | 54 +++++++++++++++
2 files changed, 127 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/722efb82/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
new file mode 100644
index 0000000..2f532e2
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.metadata.datatype.DataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import org.apache.kylin.rest.service.EncodingService;
+
+@Controller
+@RequestMapping(value = "/encodings")
+public class EncodingController extends BasicController {
+
+ private static final Logger logger = LoggerFactory.getLogger(EncodingController.class);
+
+ @Autowired
+ private EncodingService encodingService;
+
+ /**
+ * Get valid encodings for the datatype, if no datatype parameter, return all encodings.
+ *
+ * @return suggestion map
+ */
+ @RequestMapping(value = "valid_encodings", method = { RequestMethod.GET })
+ @ResponseBody
+ public Map<String, Object> getValidEncodings() {
+
+ Set<String> allDatatypes = Sets.newHashSet();
+ allDatatypes.addAll(DataType.DATETIME_FAMILY);
+ allDatatypes.addAll(DataType.INTEGER_FAMILY);
+ allDatatypes.addAll(DataType.NUMBER_FAMILY);
+ allDatatypes.addAll(DataType.STRING_FAMILY);
+
+ Map<String, List<String>> datatypeValidEncodings = Maps.newHashMap();
+ for (String dataTypeStr : allDatatypes) {
+ datatypeValidEncodings.put(dataTypeStr, encodingService.getValidEncodings(DataType.getType(dataTypeStr)));
+ }
+
+ Map<String, Object> ret = Maps.newHashMap();
+ ret.put("code", "000");
+ ret.put("data", datatypeValidEncodings);
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/722efb82/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
new file mode 100644
index 0000000..7d7d016
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/EncodingService.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.kylin.rest.service;
+
+import java.util.List;
+
+import org.apache.kylin.dimension.BooleanDimEnc;
+import org.apache.kylin.dimension.DateDimEnc;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.FixedLenDimEnc;
+import org.apache.kylin.dimension.FixedLenHexDimEnc;
+import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.TimeDimEnc;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.springframework.stereotype.Component;
+
+import com.google.common.collect.Lists;
+
+@Component("encodingService")
+public class EncodingService extends BasicService {
+
+ public List<String> getValidEncodings(DataType dataType) {
+ if (dataType.isIntegerFamily()) {
+ return Lists.newArrayList(BooleanDimEnc.ENCODING_NAME, DateDimEnc.ENCODING_NAME, TimeDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME, IntegerDimEnc.ENCODING_NAME);
+ } else if (dataType.isNumberFamily()) { //numbers include integers
+ return Lists.newArrayList(DictionaryDimEnc.ENCODING_NAME);
+ } else if (dataType.isDateTimeFamily()) {
+ return Lists.newArrayList(DateDimEnc.ENCODING_NAME, TimeDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME);
+ } else if (dataType.isStringFamily()) {
+ return Lists.newArrayList(BooleanDimEnc.ENCODING_NAME, DateDimEnc.ENCODING_NAME, TimeDimEnc.ENCODING_NAME, DictionaryDimEnc.ENCODING_NAME, FixedLenDimEnc.ENCODING_NAME, //
+ FixedLenHexDimEnc.ENCODING_NAME, IntegerDimEnc.ENCODING_NAME);
+ } else {
+ throw new IllegalArgumentException("can't provide valid encodings for datatype:" + dataType);
+ }
+ }
+
+}
[11/17] kylin git commit: KYLIN-2440 Make PK/FK always exposed
Posted by bi...@apache.org.
KYLIN-2440 Make PK/FK always exposed
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7fa25e0f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7fa25e0f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7fa25e0f
Branch: refs/heads/KYLIN-2428
Commit: 7fa25e0fafe25686450587407f2ed6f1e71cf4d3
Parents: f456961
Author: Li Yang <li...@apache.org>
Authored: Thu Feb 9 18:28:38 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Feb 9 18:28:53 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeDesc.java | 35 +++++++++++++++++---
1 file changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7fa25e0f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 5e970bf..9acb90c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -64,6 +64,7 @@ import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
@@ -570,10 +571,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
checkState(rowkey.getRowKeyColumns().length == dimCols.size(), "RowKey columns count (%d) doesn't match dimensions columns count (%d)", rowkey.getRowKeyColumns().length, dimCols.size());
initDictionaryDesc();
-
- for (TblColRef col : allColumns) {
- allColumnDescs.add(col.getColumnDesc());
- }
+ amendAllColumns();
}
public void validateAggregationGroups() {
@@ -958,6 +956,35 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return false;
}
+ private void amendAllColumns() {
+ // make sure all PF/FK are included, thus become exposed to calcite later
+ Set<TableRef> tables = collectTablesOnJoinChain(allColumns);
+ for (TableRef t : tables) {
+ JoinDesc join = model.getJoinByPKSide(t);
+ if (join != null) {
+ allColumns.addAll(Arrays.asList(join.getForeignKeyColumns()));
+ allColumns.addAll(Arrays.asList(join.getPrimaryKeyColumns()));
+ }
+ }
+
+ for (TblColRef col : allColumns) {
+ allColumnDescs.add(col.getColumnDesc());
+ }
+ }
+
+ private Set<TableRef> collectTablesOnJoinChain(Set<TblColRef> columns) {
+ Set<TableRef> result = new HashSet<>();
+ for (TblColRef col : columns) {
+ TableRef t = col.getTableRef();
+ while (t != null) {
+ result.add(t);
+ JoinDesc join = model.getJoinByPKSide(t);
+ t = join == null ? null : join.getFKSide();
+ }
+ }
+ return result;
+ }
+
public long getRetentionRange() {
return retentionRange;
}
[05/17] kylin git commit: Fix bug in static init of DataType to pass
ExtendedColumnSerializerTest UT
Posted by bi...@apache.org.
Fix bug in static init of DataType to pass ExtendedColumnSerializerTest UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/570ab42a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/570ab42a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/570ab42a
Branch: refs/heads/KYLIN-2428
Commit: 570ab42a9b7597ce85b17202b67cd0ea5403cba4
Parents: df3ecd3
Author: kangkaisen <ka...@163.com>
Authored: Wed Feb 8 21:25:37 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Wed Feb 8 21:38:00 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/metadata/datatype/DataType.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/570ab42a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index b726c5f..d3756b8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -114,7 +114,8 @@ public class DataType implements Serializable {
public static final DataType ANY = DataType.getType("any");
static {
- MeasureTypeFactory.init();
+ //to ensure the MeasureTypeFactory class has initialized
+ MeasureTypeFactory.getUDAFs();
}
public static DataType getType(String type) {
[13/17] kylin git commit: fix some testcases which can not be run
concurrently
Posted by bi...@apache.org.
fix some testcases which can not be run concurrently
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f2e8b690
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f2e8b690
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f2e8b690
Branch: refs/heads/KYLIN-2428
Commit: f2e8b690f186abe5048dee8a6b0339c2a28c0594
Parents: 3ee4946
Author: etherge <et...@163.com>
Authored: Wed Feb 8 23:50:35 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Feb 9 22:05:11 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 16 +++---
.../apache/kylin/common/KylinConfigBase.java | 7 ++-
.../apache/kylin/common/KylinConfigTest.java | 17 ++++++-
.../apache/kylin/dict/CachedTreeMapTest.java | 7 ++-
.../kylin/job/BaseTestDistributedScheduler.java | 29 ++---------
.../kylin/provision/BuildCubeWithStream.java | 32 ++++++++++--
.../org/apache/kylin/provision/MockKafka.java | 11 ++---
.../hbase/util/ZookeeperDistributedJobLock.java | 24 +--------
.../kylin/storage/hbase/util/ZookeeperUtil.java | 52 ++++++++++++++++++++
9 files changed, 124 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index e8c7fae..fdcd52c 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -56,6 +56,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
+import com.google.common.io.Files;
public class DeployUtil {
private static final Logger logger = LoggerFactory.getLogger(DeployUtil.class);
@@ -139,7 +140,7 @@ public class DeployUtil {
boolean buildCubeUsingProvidedData = Boolean.parseBoolean(System.getProperty("buildCubeUsingProvidedData"));
if (!buildCubeUsingProvidedData) {
System.out.println("build cube with random dataset");
-
+
// data is generated according to cube descriptor and saved in resource store
MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
DataModelDesc model = mgr.getDataModelDesc(modelName);
@@ -208,12 +209,12 @@ public class DeployUtil {
MetadataManager metaMgr = MetadataManager.getInstance(config());
// scp data files, use the data from hbase, instead of local files
- File temp = File.createTempFile("temp", ".csv");
- temp.createNewFile();
+ File tempDir = Files.createTempDir();
+ String tempDirAbsPath = tempDir.getAbsolutePath();
for (String tablename : TABLE_NAMES) {
tablename = tablename.toUpperCase();
- File localBufferFile = new File(temp.getParent() + "/" + tablename + ".csv");
+ File localBufferFile = new File(tempDirAbsPath + "/" + tablename + ".csv");
localBufferFile.createNewFile();
InputStream hbaseDataStream = metaMgr.getStore().getResource("/data/" + tablename + ".csv").inputStream;
@@ -225,8 +226,7 @@ public class DeployUtil {
localBufferFile.deleteOnExit();
}
- String tableFileDir = temp.getParent();
- temp.delete();
+ tempDir.deleteOnExit();
IHiveClient hiveClient = HiveClientFactory.getHiveClient();
// create hive tables
@@ -238,7 +238,7 @@ public class DeployUtil {
// load data to hive tables
// LOAD DATA LOCAL INPATH 'filepath' [OVERWRITE] INTO TABLE tablename
for (String tablename : TABLE_NAMES) {
- hiveClient.executeHQL(generateLoadDataHql(tablename.toUpperCase(), tableFileDir));
+ hiveClient.executeHQL(generateLoadDataHql(tablename.toUpperCase(), tempDirAbsPath));
}
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
@@ -255,7 +255,7 @@ public class DeployUtil {
String dropsql = "DROP TABLE IF EXISTS " + tableDesc.getIdentity();
String dropsql2 = "DROP VIEW IF EXISTS " + tableDesc.getIdentity();
-
+
StringBuilder ddl = new StringBuilder();
ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n");
ddl.append("(" + "\n");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ebd9dfc..dce4149 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -216,12 +216,11 @@ abstract public class KylinConfigBase implements Serializable {
String metadataUrl = getMetadataUrl();
String defaultPrefix = "kylin_metadata";
- if (metadataUrl.endsWith("@hbase")) {
- int cut = metadataUrl.lastIndexOf('@');
+ int cut = metadataUrl.lastIndexOf('@');
+ if (cut > 0) {
return metadataUrl.substring(0, cut);
- } else {
- return defaultPrefix;
}
+ return defaultPrefix;
}
public String[] getRealizationProviders() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
index 4d5f130..7e4b444 100644
--- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java
@@ -31,7 +31,7 @@ import org.junit.Test;
import com.google.common.collect.Maps;
-public class KylinConfigTest extends HotLoadKylinPropertiesTestCase{
+public class KylinConfigTest extends HotLoadKylinPropertiesTestCase {
@Test
public void testMRConfigOverride() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -81,4 +81,19 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase{
assertEquals("kylin@kylin.apache.org", config.getKylinOwner());
}
+
+ @Test
+ public void testGetMetadataUrlPrefix() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ final String default_metadata_prefix = "kylin_metadata";
+
+ config.setMetadataUrl("testMetaPrefix@hbase");
+ assertEquals("testMetaPrefix", config.getMetadataUrlPrefix());
+
+ config.setMetadataUrl("testMetaPrefix@hdfs");
+ assertEquals("testMetaPrefix", config.getMetadataUrlPrefix());
+
+ config.setMetadataUrl("/kylin/temp");
+ assertEquals(default_metadata_prefix, config.getMetadataUrlPrefix());
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
index ccf6e24..3c29d9c 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/CachedTreeMapTest.java
@@ -30,6 +30,7 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
+import java.util.UUID;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -114,8 +115,10 @@ public class CachedTreeMapTest {
}
}
- public static final String baseDir = "/tmp/kylin_cachedtreemap_test/";
- public static final String workingDir = "/tmp/kylin_cachedtreemap_test/working";
+
+ static final UUID uuid = UUID.randomUUID();
+ static final String baseDir = "/tmp/kylin_cachedtreemap_test/" + uuid;
+ static final String workingDir = baseDir + "/working";
private static void cleanup() {
Path basePath = new Path(baseDir);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
index 2f37a50..2d79970 100644
--- a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
+++ b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
@@ -20,9 +20,7 @@ package org.apache.kylin.job;
import java.io.File;
import java.nio.charset.Charset;
-import java.util.Arrays;
-
-import javax.annotation.Nullable;
+import java.util.UUID;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
@@ -31,8 +29,6 @@ import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.imps.CuratorFrameworkState;
import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.job.engine.JobEngineConfig;
@@ -40,15 +36,13 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.impl.threadpool.DistributedScheduler;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.ZookeeperDistributedJobLock;
+import org.apache.kylin.storage.hbase.util.ZookeeperUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
import com.google.common.io.Files;
public class BaseTestDistributedScheduler extends HBaseMetadataTestCase {
@@ -62,8 +56,8 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase {
static File localMetaDir;
static final String SEGMENT_ID = "segmentId";
- static final String segmentId1 = "segmentId1";
- static final String segmentId2 = "segmentId2";
+ static final String segmentId1 = "seg1" + UUID.randomUUID();
+ static final String segmentId2 = "seg2" + UUID.randomUUID();
static final String serverName1 = "serverName1";
static final String serverName2 = "serverName2";
static final String confDstPath1 = "target/kylin_metadata_dist_lock_test1/kylin.properties";
@@ -177,7 +171,7 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase {
}
private static void initZk() {
- String zkConnectString = getZKConnectString();
+ String zkConnectString = ZookeeperUtil.getZKConnectString();
if (StringUtils.isEmpty(zkConnectString)) {
throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
}
@@ -186,19 +180,6 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase {
zkClient.start();
}
- private static String getZKConnectString() {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- final String serverList = conf.get(HConstants.ZOOKEEPER_QUORUM);
- final String port = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
- return StringUtils.join(Iterables.transform(Arrays.asList(serverList.split(",")), new Function<String, String>() {
- @Nullable
- @Override
- public String apply(String input) {
- return input + ":" + port;
- }
- }), ",");
- }
-
String getServerName(String cubeName) {
String lockPath = getLockPath(cubeName);
String serverName = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 53c89cf..f3b1ec9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -36,6 +36,10 @@ import java.util.concurrent.TimeUnit;
import org.I0Itec.zkclient.ZkConnection;
import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.kafka.common.requests.MetadataResponse;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
@@ -63,6 +67,7 @@ import org.apache.kylin.source.kafka.KafkaConfigManager;
import org.apache.kylin.source.kafka.config.BrokerConfig;
import org.apache.kylin.source.kafka.config.KafkaConfig;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
+import org.apache.kylin.storage.hbase.util.ZookeeperUtil;
import org.apache.kylin.tool.StorageCleanupJob;
import org.junit.Assert;
import org.slf4j.Logger;
@@ -84,6 +89,9 @@ public class BuildCubeWithStream {
private KafkaConfig kafkaConfig;
private MockKafka kafkaServer;
+ private ZkConnection zkConnection;
+ private final String kafkaZkPath = "/" + UUID.randomUUID().toString();
+
protected static boolean fastBuildMode = false;
private boolean generateData = true;
@@ -128,8 +136,9 @@ public class BuildCubeWithStream {
private void startEmbeddedKafka(String topicName, BrokerConfig brokerConfig) {
//Start mock Kakfa
- String zkConnectionStr = "sandbox:2181";
- ZkConnection zkConnection = new ZkConnection(zkConnectionStr);
+ String zkConnectionStr = ZookeeperUtil.getZKConnectString() + kafkaZkPath;
+ System.out.println("zkConnectionStr" + zkConnectionStr);
+ zkConnection = new ZkConnection(zkConnectionStr);
// Assert.assertEquals(ZooKeeper.States.CONNECTED, zkConnection.getZookeeperState());
kafkaServer = new MockKafka(zkConnection, brokerConfig.getPort(), brokerConfig.getId());
kafkaServer.start();
@@ -287,9 +296,24 @@ public class BuildCubeWithStream {
public void after() {
kafkaServer.stop();
+ cleanKafkaZkPath(kafkaZkPath);
DefaultScheduler.destroyInstance();
}
+ private void cleanKafkaZkPath(String path) {
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ CuratorFramework zkClient = CuratorFrameworkFactory.newClient(ZookeeperUtil.getZKConnectString(), retryPolicy);
+ zkClient.start();
+
+ try {
+ zkClient.delete().deletingChildrenIfNeeded().forPath(kafkaZkPath);
+ } catch (Exception e) {
+ logger.warn("Failed to delete zookeeper path: " + path, e);
+ } finally {
+ zkClient.close();
+ }
+ }
+
protected void waitForJob(String jobId) {
while (true) {
AbstractExecutable job = jobService.getJob(jobId);
@@ -327,6 +351,8 @@ public class BuildCubeWithStream {
buildCubeWithStream.before();
buildCubeWithStream.build();
logger.info("Build is done");
+
+ buildCubeWithStream.after();
buildCubeWithStream.cleanup();
logger.info("Going to exit");
} catch (Throwable e) {
@@ -336,7 +362,7 @@ public class BuildCubeWithStream {
long millis = System.currentTimeMillis() - start;
System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " + BuildCubeWithStream.class.getName());
-
+
System.exit(exitCode);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/kylin-it/src/test/java/org/apache/kylin/provision/MockKafka.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/MockKafka.java b/kylin-it/src/test/java/org/apache/kylin/provision/MockKafka.java
index 3f47923..fce422a 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/MockKafka.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/MockKafka.java
@@ -29,6 +29,8 @@ import org.I0Itec.zkclient.ZkConnection;
import org.I0Itec.zkclient.exception.ZkMarshallingError;
import org.I0Itec.zkclient.serialize.ZkSerializer;
import org.apache.kafka.common.requests.MetadataResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import kafka.admin.AdminUtils;
import kafka.server.KafkaConfig;
@@ -52,6 +54,7 @@ public class MockKafka {
}
private KafkaServerStartable kafkaServer;
+ private static final Logger logger = LoggerFactory.getLogger(MockKafka.class);
private ZkConnection zkConnection;
@@ -67,7 +70,7 @@ public class MockKafka {
public MockKafka(ZkConnection zkServerConnection, int port, int brokerId) {
this(zkServerConnection, System.getProperty("java.io.tmpdir") + "/" + UUID.randomUUID().toString(), String.valueOf(port), String.valueOf(brokerId));
- start();
+ //start();
}
private MockKafka(ZkConnection zkServerConnection, String logDir, String port, String brokerId) {
@@ -110,13 +113,9 @@ public class MockKafka {
zkClient.close();
}
- public String getConnectionString() {
- return String.format("%s:%d", kafkaServer.serverConfig().hostName(), kafkaServer.serverConfig().port());
- }
-
public void start() {
kafkaServer.startup();
- System.out.println("embedded kafka is up");
+ System.out.println("--embedded kafka is up");
}
public void stop() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
index ee7cd50..983bfd9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java
@@ -19,11 +19,8 @@
package org.apache.kylin.storage.hbase.util;
import java.nio.charset.Charset;
-import java.util.Arrays;
import java.util.concurrent.ExecutorService;
-import javax.annotation.Nullable;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
@@ -33,18 +30,12 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.job.lock.DistributedJobLock;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.zookeeper.CreateMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-
/**
* the jobLock is specially used to support distributed scheduler.
*/
@@ -65,7 +56,7 @@ public class ZookeeperDistributedJobLock implements DistributedJobLock {
public ZookeeperDistributedJobLock(KylinConfig config) {
this.config = config;
- String zkConnectString = getZKConnectString();
+ String zkConnectString = ZookeeperUtil.getZKConnectString();
logger.info("zk connection string:" + zkConnectString);
if (StringUtils.isEmpty(zkConnectString)) {
throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!");
@@ -243,19 +234,6 @@ public class ZookeeperDistributedJobLock implements DistributedJobLock {
}
}
- private static String getZKConnectString() {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- final String serverList = conf.get(HConstants.ZOOKEEPER_QUORUM);
- final String port = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
- return StringUtils.join(Iterables.transform(Arrays.asList(serverList.split(",")), new Function<String, String>() {
- @Nullable
- @Override
- public String apply(String input) {
- return input + ":" + port;
- }
- }), ",");
- }
-
private String getLockPath(String pathName) {
return ZOOKEEPER_LOCK_PATH + "/" + config.getMetadataUrlPrefix() + "/" + pathName;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2e8b690/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperUtil.java
new file mode 100644
index 0000000..b5ebe89
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperUtil.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.util;
+
+import java.util.Arrays;
+
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+
+public class ZookeeperUtil {
+
+ /**
+ * Get zookeeper connection string from HBase Configuration
+ *
+ * @return Zookeeper Connection string
+ */
+ public static String getZKConnectString() {
+ Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
+ final String serverList = conf.get(HConstants.ZOOKEEPER_QUORUM);
+ final String port = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);
+ return StringUtils.join(Iterables.transform(Arrays.asList(serverList.split(",")), new Function<String, String>() {
+ @Nullable
+ @Override
+ public String apply(String input) {
+ return input + ":" + port;
+ }
+ }), ",");
+ }
+}
[02/17] kylin git commit: KYLIN-2377 Add kylin client query timeout
Posted by bi...@apache.org.
KYLIN-2377 Add kylin client query timeout
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8263752a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8263752a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8263752a
Branch: refs/heads/KYLIN-2428
Commit: 8263752a499158342e0588fda851a4006e8b1669
Parents: 0dcce15
Author: kangkaisen <ka...@live.com>
Authored: Tue Jan 10 14:02:18 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Wed Feb 8 15:51:30 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../java/org/apache/kylin/cube/CubeInstance.java | 1 +
.../kylin/metadata/realization/IRealization.java | 5 ++++-
.../org/apache/kylin/storage/StorageContext.java | 16 +++++++++++++++-
.../storage/gtrecord/GTCubeStorageQueryBase.java | 2 ++
.../gtrecord/SequentialCubeTupleIterator.java | 5 +++++
.../apache/kylin/storage/hybrid/HybridInstance.java | 1 +
7 files changed, 32 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index fe15b1e..ebd9dfc 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -877,6 +877,10 @@ abstract public class KylinConfigBase implements Serializable {
return udfMap;
}
+ public int getQueryTimeoutSeconds() {
+ return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
+ }
+
// ============================================================================
// SERVER
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 1d60575..fb9a7a7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -178,6 +178,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return sizeRecordSize;
}
+ @Override
public KylinConfig getConfig() {
return config;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
index a0243f4..aafc0f0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IStorageAware;
@@ -42,7 +43,7 @@ public interface IRealization extends IStorageAware {
public DataModelDesc getModel();
public Set<TblColRef> getAllColumns();
-
+
public Set<ColumnDesc> getAllColumnDescs();
public List<TblColRef> getAllDimensions();
@@ -60,4 +61,6 @@ public interface IRealization extends IStorageAware {
public long getDateRangeEnd();
public boolean supportsLimitPushDown();
+
+ public KylinConfig getConfig();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index ec46f83..ab0ea73 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -41,6 +41,7 @@ public class StorageContext {
private int finalPushDownLimit = Integer.MAX_VALUE;
private boolean hasSort = false;
private boolean acceptPartialResult = false;
+ private long deadline;
private boolean exactAggregation = false;
private boolean needStorageAggregation = false;
@@ -123,6 +124,19 @@ public class StorageContext {
}
}
+ public long getDeadline() {
+ return this.deadline;
+ }
+
+ public void setDeadline(IRealization realization) {
+ int timeout = realization.getConfig().getQueryTimeoutSeconds() * 1000;
+ if (timeout == 0) {
+ this.deadline = Long.MAX_VALUE;
+ } else {
+ this.deadline = timeout + System.currentTimeMillis();
+ }
+ }
+
public void markSort() {
this.hasSort = true;
}
@@ -202,5 +216,5 @@ public class StorageContext {
public void setStorageQuery(IStorageQuery storageQuery) {
this.storageQuery = storageQuery;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 4fcfad1..4dbdf94 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -120,6 +120,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// set limit push down
enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context);
+ // set query deadline
+ context.setDeadline(cubeInstance);
// set cautious threshold to prevent out of memory
setThresholdIfNecessary(dimensionsD, metrics, context);
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index c621215..c6b2c6c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -28,6 +28,7 @@ import javax.annotation.Nullable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.gridtable.GTScanTimeoutException;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.ITuple;
@@ -141,6 +142,10 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
@Override
public ITuple next() {
+ if (scanCount % 100 == 1 && System.currentTimeMillis() > context.getDeadline()) {
+ throw new GTScanTimeoutException("Query Timeout!");
+ }
+
// prevent the big query to make the Query Server OOM
if (scanCount++ > SCAN_THRESHOLD) {
throw new ScanOutOfLimitException("Scan count exceed the scan threshold: " + SCAN_THRESHOLD);
http://git-wip-us.apache.org/repos/asf/kylin/blob/8263752a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
index 4f0e446..1b113ee 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java
@@ -251,6 +251,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization
return getType() + "[name=" + name + "]";
}
+ @Override
public KylinConfig getConfig() {
return config;
}
[12/17] kylin git commit: KYLIN-2316 Global dict couldn't be used for
dimension column
Posted by bi...@apache.org.
KYLIN-2316 Global dict couldn't be used for dimension column
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ee49467
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ee49467
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ee49467
Branch: refs/heads/KYLIN-2428
Commit: 3ee49467ce686d11afcecbc636bb5213b0927427
Parents: 7fa25e0
Author: kangkaisen <ka...@163.com>
Authored: Thu Feb 9 19:54:26 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Thu Feb 9 20:28:17 2017 +0800
----------------------------------------------------------------------
.../kylin/cube/model/validation/rule/DictionaryRule.java | 9 +++++++++
.../cube/model/validation/rule/DictionaryRuleTest.java | 11 +++++++++--
2 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ee49467/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
index 37889c2..8da3ca0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
@@ -29,6 +29,7 @@ import org.apache.kylin.cube.model.DictionaryDesc;
import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
+import org.apache.kylin.dict.GlobalDictionaryBuilder;
import org.apache.kylin.metadata.model.TblColRef;
/**
@@ -45,10 +46,13 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
static final String ERROR_REUSE_BUILDER_BOTH_SET = "REUSE and BUILDER both set on dictionary for column: ";
static final String ERROR_REUSE_BUILDER_BOTH_EMPTY = "REUSE and BUILDER both empty on dictionary for column: ";
static final String ERROR_TRANSITIVE_REUSE = "Transitive REUSE is not allowed for dictionary: ";
+ static final String ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE = "Global dictionary couldn't be used for dimension column: ";
@Override
public void validate(CubeDesc cubeDesc, ValidateContext context) {
List<DictionaryDesc> dictDescs = cubeDesc.getDictionaries();
+ Set<TblColRef> dimensionColumns = cubeDesc.listDimensionColumnsIncludingDerived();
+
if (dictDescs == null || dictDescs.isEmpty()) {
return;
}
@@ -78,6 +82,11 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
return;
}
+ if (StringUtils.isNotEmpty(builderClass) && builderClass.equalsIgnoreCase(GlobalDictionaryBuilder.class.getName()) && dimensionColumns.contains(dictCol)) {
+ context.addResult(ResultLevel.ERROR, ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE + dictCol);
+ return;
+ }
+
if (reuseCol != null) {
reuseDictionaries.add(dictDesc);
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ee49467/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index 452768f..fcb723e 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.cube.model.validation.rule;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_DUPLICATE_DICTIONARY_COLUMN;
+import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_EMPTY;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_SET;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_TRANSITIVE_REUSE;
@@ -91,10 +92,16 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase {
DictionaryDesc.create("lstg_site_id", "SELLER_ID", null),
DictionaryDesc.create("price", "lstg_site_id", null));
}
-
+
+ @Test
+ public void testBadDesc5() throws IOException {
+ testDictionaryDesc(ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE,
+ DictionaryDesc.create("CATEG_LVL2_NAME", null, GlobalDictionaryBuilder.class.getName()));
+ }
+
@Test
public void testGoodDesc2() throws IOException {
- testDictionaryDesc(null, DictionaryDesc.create("lstg_site_id", null, GlobalDictionaryBuilder.class.getName()));
+ testDictionaryDesc(null, DictionaryDesc.create("SELLER_ID", null, GlobalDictionaryBuilder.class.getName()));
}
private void testDictionaryDesc(String expectMessage, DictionaryDesc... descs) throws IOException {
[04/17] kylin git commit: minor, clean up kylin.sh a little
Posted by bi...@apache.org.
minor, clean up kylin.sh a little
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/df3ecd32
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/df3ecd32
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/df3ecd32
Branch: refs/heads/KYLIN-2428
Commit: df3ecd324a6442fee52f146d930cc5f84dafd945
Parents: d135bdb
Author: Yang Li <li...@apache.org>
Authored: Wed Feb 8 20:42:02 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Feb 8 20:42:02 2017 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/df3ecd32/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 7813b79..cc571e5 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -60,8 +60,7 @@ then
PID=`cat $KYLIN_HOME/pid`
if ps -p $PID > /dev/null
then
- echo "Kylin is running, stop it first"
- exit 1
+ quit "Kylin is running, stop it first"
fi
fi
@@ -81,7 +80,7 @@ then
spring_profile=`bash ${dir}/get-properties.sh kylin.security.profile`
if [ -z "$spring_profile" ]
then
- quit 'please set kylin.security.profile in kylin.properties, options are: testing, ldap, saml.'
+ quit 'Please set kylin.security.profile in kylin.properties, options are: testing, ldap, saml.'
else
verbose "kylin.security.profile is set to $spring_profile"
fi
@@ -95,10 +94,7 @@ then
kylin_rest_address_arr=(${kylin_rest_address//;/ })
nc -z -w 5 ${kylin_rest_address_arr[0]} ${kylin_rest_address_arr[1]} 1>/dev/null 2>&1; nc_result=$?
if [ $nc_result -eq 0 ]; then
- echo "port ${kylin_rest_address} is not available, could not start Kylin"
- exit 1
- else
- echo "port ${kylin_rest_address} is available"
+ quit "Port ${kylin_rest_address} is not available, could not start Kylin."
fi
#debug if encounter NoClassDefError
@@ -155,7 +151,7 @@ then
elif [ "$1" = "diag" ]
then
- echo "kylin.sh diag no longer supported, use diag.sh instead"
+ echo "'kylin.sh diag' no longer supported, use diag.sh instead"
exit 0
# tool command
[17/17] kylin git commit: KYLIN-2428 refactor library shade and
cleanup redundency in job/coprocessor/jdbc/tool
Posted by bi...@apache.org.
KYLIN-2428 refactor library shade and cleanup redundency in job/coprocessor/jdbc/tool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7c2632d9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7c2632d9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7c2632d9
Branch: refs/heads/KYLIN-2428
Commit: 7c2632d9605db8823169a107d59ddd97a34511d2
Parents: e09338b
Author: Billy Liu <bi...@apache.org>
Authored: Fri Feb 10 17:03:13 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Feb 10 17:03:13 2017 +0800
----------------------------------------------------------------------
assembly/pom.xml | 49 +++-
atopcalcite/pom.xml | 1 +
build/bin/get-properties.sh | 3 +-
build/bin/kylin.sh | 8 +-
build/conf/kylin-tools-log4j.properties | 9 +-
build/script/prepare_libs.sh | 2 +-
core-common/pom.xml | 22 +-
core-cube/pom.xml | 18 +-
.../apache/kylin/gridtable/GTInvertedIndex.java | 223 -------------------
.../gridtable/GTInvertedIndexOfColumn.java | 133 -----------
.../gridtable/SimpleInvertedIndexTest.java | 196 ----------------
core-job/pom.xml | 9 +
core-metadata/pom.xml | 23 ++
core-storage/pom.xml | 20 ++
engine-spark/pom.xml | 1 +
jdbc/pom.xml | 14 +-
pom.xml | 220 ++++++++++--------
query/pom.xml | 6 +
server/pom.xml | 4 +
source-kafka/pom.xml | 16 +-
storage-hbase/pom.xml | 19 +-
tool-assembly/pom.xml | 135 +++++++++++
tool/pom.xml | 39 ----
23 files changed, 428 insertions(+), 742 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 873abff..ba69583 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -19,16 +19,20 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-assembly</artifactId>
+ <name>Apache Kylin - Assembly</name>
+
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
<version>2.0.0-SNAPSHOT</version>
-
</parent>
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-assembly</artifactId>
- <name>Apache Kylin - Assembly</name>
+ <properties>
+ <shadeBase>org.apache.kylin.job.shaded</shadeBase>
+ </properties>
<dependencies>
<dependency>
@@ -180,8 +184,45 @@
<exclude>org.apache.zookeeper:*</exclude>
<exclude>net.sf.ehcache:*</exclude>
<exclude>org.apache.httpcomponents:*</exclude>
+ <exclude>org.apache.commons:commons-email:*</exclude>
+ <exclude>com.sun.mail:javax.mail:*</exclude>
+ <exclude>javax.activation:activation:*</exclude>
</excludes>
</artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.tdunning</pattern>
+ <shadedPattern>${shadeBase}.com.tdunning</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.codec</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.codec</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.collections</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.collections</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.lang3</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.lang3</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.logging</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.logging</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.roaringbitmap</pattern>
+ <shadedPattern>${shadeBase}.org.roaringbitmap</shadedPattern>
+ </relocation>
+ </relocations>
<filters>
<filter>
<artifact>*:*</artifact>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 1b327fe..c65c0d1 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -43,6 +43,7 @@
</exclusion>
</exclusions>
</dependency>
+ <!-- It should be avatica(the shaded one), not avatica-core, since the inconsistency protobuf dependency with Hadoop -->
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/build/bin/get-properties.sh
----------------------------------------------------------------------
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 170442d..57721d8 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -25,7 +25,6 @@ then
exit -1
fi
-job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
-result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
+result=`java -cp $tool_jar -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
echo "$result"
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index cc571e5..15e1aed 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -38,7 +38,7 @@ function retrieveDependency() {
source ${dir}/setenv.sh
fi
- export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+ export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
if [ -n "$KAFKA_HOME" ]
then
@@ -103,7 +103,7 @@ then
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
hbase ${KYLIN_EXTRA_START_OPTS} \
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
- -Dlog4j.configuration=kylin-server-log4j.properties \
+ -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-server-log4j.properties \
-Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
-Djava.endorsed.dirs=${tomcat_root}/endorsed \
@@ -165,8 +165,10 @@ then
then source ${dir}/setenv-tool.sh
fi
+ hbase_original=${HBASE_CLASSPATH}
+ export HBASE_CLASSPATH=${hbase_original}:${KYLIN_HOME}/tool/*
exec hbase ${KYLIN_EXTRA_START_OPTS} -Dkylin.hive.dependency=${hive_dependency} -Dkylin.hbase.dependency=${hbase_dependency} -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties "$@"
-
+ export HBASE_CLASSPATH=${hbase_original}
else
quit "Usage: 'kylin.sh [-v] start' or 'kylin.sh [-v] stop'"
fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/build/conf/kylin-tools-log4j.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin-tools-log4j.properties b/build/conf/kylin-tools-log4j.properties
index 0ba7d2e..e975d18 100644
--- a/build/conf/kylin-tools-log4j.properties
+++ b/build/conf/kylin-tools-log4j.properties
@@ -25,11 +25,12 @@
# It's called kylin-tools-log4j.properties so that it won't distract users from the other more important log4j config file: kylin-server-log4j.properties
# enable this by -Dlog4j.configuration=kylin-tools-log4j.properties
-log4j.rootLogger=INFO,stdout
+log4j.rootLogger=INFO,stderr
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %m%n
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %m%n
#log4j.logger.org.apache.hadoop=ERROR
log4j.logger.org.apache.kylin=DEBUG
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/build/script/prepare_libs.sh
----------------------------------------------------------------------
diff --git a/build/script/prepare_libs.sh b/build/script/prepare_libs.sh
index aec9923..3dbdf21 100755
--- a/build/script/prepare_libs.sh
+++ b/build/script/prepare_libs.sh
@@ -33,7 +33,7 @@ mkdir build/lib build/tool
cp assembly/target/kylin-assembly-${version}-job.jar build/lib/kylin-job-${version}.jar
cp storage-hbase/target/kylin-storage-hbase-${version}-coprocessor.jar build/lib/kylin-coprocessor-${version}.jar
cp jdbc/target/kylin-jdbc-${version}.jar build/lib/kylin-jdbc-${version}.jar
-cp tool/target/kylin-tool-${version}-assembly.jar build/tool/kylin-tool-${version}.jar
+cp tool-assembly/target/kylin-tool-assembly-${version}-assembly.jar build/tool/kylin-tool-${version}.jar
# Copied file becomes 000 for some env (e.g. my Cygwin)
chmod 644 build/lib/kylin-job-${version}.jar
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 016d470..95d3c29 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -35,26 +35,10 @@
<dependencies>
<!-- Basic Utilities -->
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-email</artifactId>
</dependency>
@@ -63,9 +47,11 @@
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
</dependency>
+
+ <!-- Provided -->
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 0a30432..fe80783 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -48,9 +48,23 @@
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</dependency>
+
+ <!-- Provided -->
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
<dependency>
- <groupId>com.n3twork.druid</groupId>
- <artifactId>extendedset</artifactId>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
deleted file mode 100644
index a2c713a..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-/**
- * A thread-safe inverted index of row blocks in memory.
- *
- * Note function not() must return all blocks, because index only know what block contains a value,
- * but not sure what block does not contain a value.
- *
- * @author yangli9
- */
-public class GTInvertedIndex {
-
- private final GTInfo info;
- private final ImmutableBitSet colPreferIndex;
- private final ImmutableBitSet colBlocks;
- private final GTInvertedIndexOfColumn[] index; // for each column
-
- private volatile int nIndexedBlocks;
-
- public GTInvertedIndex(GTInfo info) {
- this.info = info;
- this.colPreferIndex = info.colPreferIndex;
- this.colBlocks = info.selectColumnBlocks(colPreferIndex);
-
- index = new GTInvertedIndexOfColumn[info.getColumnCount()];
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- index[c] = new GTInvertedIndexOfColumn(info.codeSystem.getComparator());
- }
- }
-
- public void add(GTRowBlock block) {
-
- @SuppressWarnings("unchecked")
- Set<ByteArray>[] distinctValues = new Set[info.getColumnCount()];
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- distinctValues[c] = new HashSet<ByteArray>();
- }
-
- GTRowBlock.Reader reader = block.getReader(colBlocks);
- GTRecord record = new GTRecord(info);
- while (reader.hasNext()) {
- reader.fetchNext(record);
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- distinctValues[c].add(record.get(c));
- }
- }
-
- for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
- int c = colPreferIndex.trueBitAt(i);
- index[c].add(distinctValues[c], block.getSequenceId());
- }
-
- nIndexedBlocks = Math.max(nIndexedBlocks, block.seqId + 1);
- }
-
- public ConciseSet filter(TupleFilter filter) {
- return filter(filter, nIndexedBlocks);
- }
-
- public ConciseSet filter(TupleFilter filter, int totalBlocks) {
- // number of indexed blocks may increase as we do evaluation
- int indexedBlocks = nIndexedBlocks;
-
- Evaluator evaluator = new Evaluator(indexedBlocks);
- ConciseSet r = evaluator.evaluate(filter);
-
- // add blocks that have not been indexed
- for (int i = indexedBlocks; i < totalBlocks; i++) {
- r.add(i);
- }
-
- return r;
- }
-
- private class Evaluator {
- private int indexedBlocks;
-
- Evaluator(int indexedBlocks) {
- this.indexedBlocks = indexedBlocks;
- }
-
- public ConciseSet evaluate(TupleFilter filter) {
- if (filter == null) {
- return all();
- }
-
- if (filter instanceof LogicalTupleFilter)
- return evalLogical((LogicalTupleFilter) filter);
-
- if (filter instanceof CompareTupleFilter)
- return evalCompare((CompareTupleFilter) filter);
-
- // unable to evaluate
- return all();
- }
-
- @SuppressWarnings("unchecked")
- private ConciseSet evalCompare(CompareTupleFilter filter) {
- int col = col(filter);
- if (index[col] == null)
- return all();
-
- switch (filter.getOperator()) {
- case ISNULL:
- return index[col].getNull();
- case ISNOTNULL:
- return all();
- case EQ:
- return index[col].getEquals((ByteArray) filter.getFirstValue());
- case NEQ:
- return all();
- case IN:
- return index[col].getIn((Iterable<ByteArray>) filter.getValues());
- case NOTIN:
- return all();
- case LT:
- return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), false);
- case LTE:
- return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), true);
- case GT:
- return index[col].getRange((ByteArray) filter.getFirstValue(), false, null, false);
- case GTE:
- return index[col].getRange((ByteArray) filter.getFirstValue(), true, null, false);
- default:
- throw new IllegalStateException("Unsupported operator " + filter.getOperator());
- }
- }
-
- private ConciseSet evalLogical(LogicalTupleFilter filter) {
- List<? extends TupleFilter> children = filter.getChildren();
-
- switch (filter.getOperator()) {
- case AND:
- return evalLogicalAnd(children);
- case OR:
- return evalLogicalOr(children);
- case NOT:
- return evalLogicalNot(children);
- default:
- throw new IllegalStateException("Unsupported operator " + filter.getOperator());
- }
- }
-
- private ConciseSet evalLogicalAnd(List<? extends TupleFilter> children) {
- ConciseSet set = all();
-
- for (TupleFilter c : children) {
- ConciseSet t = evaluate(c);
- if (t == null)
- continue; // because it's AND
-
- set.retainAll(t);
- }
- return set;
- }
-
- private ConciseSet evalLogicalOr(List<? extends TupleFilter> children) {
- ConciseSet set = new ConciseSet();
-
- for (TupleFilter c : children) {
- ConciseSet t = evaluate(c);
- if (t == null)
- return null; // because it's OR
-
- set.addAll(t);
- }
- return set;
- }
-
- private ConciseSet evalLogicalNot(List<? extends TupleFilter> children) {
- return all();
- }
-
- private ConciseSet all() {
- return not(new ConciseSet());
- }
-
- private ConciseSet not(ConciseSet set) {
- set.add(indexedBlocks);
- set.complement();
- return set;
- }
-
- private int col(CompareTupleFilter filter) {
- return filter.getColumn().getColumnDesc().getZeroBasedIndex();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
deleted file mode 100644
index bfacc0f..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.NavigableMap;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.kylin.common.util.ByteArray;
-
-import com.google.common.collect.Maps;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-public class GTInvertedIndexOfColumn {
-
- final private IGTComparator comparator;
- final private ReentrantReadWriteLock rwLock;
-
- private int nBlocks;
- private NavigableMap<ByteArray, ConciseSet> rangeIndex;
- private ConciseSet nullIndex;
-
- public GTInvertedIndexOfColumn(IGTComparator comparator) {
- this.comparator = comparator;
- this.rwLock = new ReentrantReadWriteLock();
- this.rangeIndex = Maps.newTreeMap(comparator);
- this.nullIndex = new ConciseSet();
- }
-
- public void add(Iterable<ByteArray> codes, int blockId) {
- rwLock.writeLock().lock();
- try {
- for (ByteArray code : codes) {
- if (comparator.isNull(code)) {
- nullIndex.add(blockId);
- continue;
- }
- ConciseSet set = rangeIndex.get(code);
- if (set == null) {
- set = new ConciseSet();
- rangeIndex.put(code.copy(), set);
- }
- set.add(blockId);
- }
-
- if (blockId >= nBlocks) {
- nBlocks = blockId + 1;
- }
-
- } finally {
- rwLock.writeLock().unlock();
- }
- }
-
- public ConciseSet getNull() {
- rwLock.readLock().lock();
- try {
- return nullIndex.clone();
- } finally {
- rwLock.readLock().unlock();
- }
- }
-
- public ConciseSet getEquals(ByteArray code) {
- rwLock.readLock().lock();
- try {
- ConciseSet set = rangeIndex.get(code);
- if (set == null)
- return new ConciseSet();
- else
- return set.clone();
- } finally {
- rwLock.readLock().unlock();
- }
- }
-
- public ConciseSet getIn(Iterable<ByteArray> codes) {
- rwLock.readLock().lock();
- try {
- ConciseSet r = new ConciseSet();
- for (ByteArray code : codes) {
- ConciseSet set = rangeIndex.get(code);
- if (set != null)
- r.addAll(set);
- }
- return r;
- } finally {
- rwLock.readLock().unlock();
- }
- }
-
- public ConciseSet getRange(ByteArray from, boolean fromInclusive, ByteArray to, boolean toInclusive) {
- rwLock.readLock().lock();
- try {
- ConciseSet r = new ConciseSet();
- if (from == null && to == null) {
- r.add(nBlocks);
- r.complement();
- return r;
- }
- NavigableMap<ByteArray, ConciseSet> subMap;
- if (from == null) {
- subMap = rangeIndex.headMap(to, toInclusive);
- } else if (to == null) {
- subMap = rangeIndex.tailMap(from, fromInclusive);
- } else {
- subMap = rangeIndex.subMap(from, fromInclusive, to, toInclusive);
- }
- for (ConciseSet set : subMap.values()) {
- r.addAll(set);
- }
- return r;
- } finally {
- rwLock.readLock().unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
deleted file mode 100644
index 5e3e771..0000000
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.gridtable;
-
-import static org.junit.Assert.assertEquals;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase {
-
- GTInfo info;
- GTInvertedIndex index;
- ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList();
- ArrayList<ConciseSet> basicResults = Lists.newArrayList();
-
- @BeforeClass
- public static void setUp() throws Exception {
- staticCreateTestMetadata();
- }
-
- @AfterClass
- public static void after() throws Exception {
- cleanAfterClass();
- }
-
- public SimpleInvertedIndexTest() {
-
- info = UnitTestSupport.advancedInfo();
- TblColRef colA = info.colRef(0);
-
- // block i contains value "i", the last is NULL
- index = new GTInvertedIndex(info);
- GTRowBlock mockBlock = GTRowBlock.allocate(info);
- GTRowBlock.Writer writer = mockBlock.getWriter();
- GTRecord record = new GTRecord(info);
- for (int i = 0; i < 10; i++) {
- record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0));
- for (int j = 0; j < info.getRowBlockSize(); j++) {
- writer.append(record);
- }
- writer.readyForFlush();
- index.add(mockBlock);
-
- writer.clearForNext();
- }
-
- basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL));
- basicResults.add(set(9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0));
- basicResults.add(set(0));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5));
- basicResults.add(set(0, 5));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5));
- basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3));
- basicResults.add(set(0, 1, 2));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3));
- basicResults.add(set(0, 1, 2, 3));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3));
- basicResults.add(set(4, 5, 6, 7, 8));
-
- basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3));
- basicResults.add(set(3, 4, 5, 6, 7, 8));
- }
-
- @Test
- public void testBasics() {
- for (int i = 0; i < basicFilters.size(); i++) {
- assertEquals(basicResults.get(i), index.filter(basicFilters.get(i)));
- }
- }
-
- @Test
- public void testLogicalAnd() {
- for (int i = 0; i < basicFilters.size(); i++) {
- for (int j = 0; j < basicFilters.size(); j++) {
- LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
- ConciseSet r = basicResults.get(i).clone();
- r.retainAll(basicResults.get(j));
- assertEquals(r, index.filter(f));
- }
- }
- }
-
- @Test
- public void testLogicalOr() {
- for (int i = 0; i < basicFilters.size(); i++) {
- for (int j = 0; j < basicFilters.size(); j++) {
- LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
- ConciseSet r = basicResults.get(i).clone();
- r.addAll(basicResults.get(j));
- assertEquals(r, index.filter(f));
- }
- }
- }
-
- @Test
- public void testNotEvaluable() {
- ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-
- CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0);
- assertEquals(all, index.filter(notEvaluable));
-
- LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable);
- assertEquals(all, index.filter(or));
-
- LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable);
- assertEquals(basicResults.get(0), index.filter(and));
- }
-
- public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
- CompareTupleFilter filter = new CompareTupleFilter(op);
- filter.addChild(columnFilter(col));
- for (int i : ids) {
- filter.addChild(constFilter(i));
- }
- return filter;
- }
-
- public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) {
- LogicalTupleFilter filter = new LogicalTupleFilter(op);
- for (TupleFilter f : filters)
- filter.addChild(f);
- return filter;
- }
-
- public static ColumnTupleFilter columnFilter(TblColRef col) {
- return new ColumnTupleFilter(col);
- }
-
- public static ConstantTupleFilter constFilter(int id) {
- byte[] space = new byte[10];
- ByteBuffer buf = ByteBuffer.wrap(space);
- StringSerializer stringSerializer = new StringSerializer(DataType.getType("string"));
- stringSerializer.serialize("" + id, buf);
- ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
- return new ConstantTupleFilter(data);
- }
-
- public static ConciseSet set(int... ints) {
- ConciseSet set = new ConciseSet();
- for (int i : ints)
- set.add(i);
- return set;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 36d34c8..9478555 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -39,6 +39,15 @@
<artifactId>kylin-core-cube</artifactId>
</dependency>
+ <!-- Provided -->
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 87c4438..0832e80 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -39,6 +39,29 @@
<artifactId>kylin-core-common</artifactId>
</dependency>
+ <!-- Provided -->
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
+ <!-- Compiled -->
<dependency>
<groupId>net.sf.ehcache</groupId>
<artifactId>ehcache</artifactId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 9bd3f04..8c3da5f 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -39,8 +39,28 @@
<artifactId>kylin-core-cube</artifactId>
</dependency>
+ <!-- Provided -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+
<!-- Env & Test -->
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index be2cdf6..fe6d998 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -70,6 +70,7 @@
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
+ <scope>provided</scope>
</dependency>
<!-- Hadoop dependency -->
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 874ead6..02890a6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -43,7 +43,7 @@
</dependency>
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
- <artifactId>avatica</artifactId>
+ <artifactId>avatica-core</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -69,9 +69,7 @@
<artifactSet>
<excludes>
- <exclude>com.google.protobuf:*</exclude>
- <exclude>commons-logging:*</exclude>
- <exclude>commons-codec:*</exclude>
+ <exclude>org.slf4j:jcl-over-slf4j:*</exclude>
</excludes>
</artifactSet>
@@ -85,9 +83,17 @@
<shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
</relocation>
<relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>${shadeBase}.com.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
<pattern>org.apache.http</pattern>
<shadedPattern>${shadeBase}.org.apache.http</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>org.apache.commons</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons</shadedPattern>
+ </relocation>
</relocations>
<filters>
<filter>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b82eee2..68a5372 100644
--- a/pom.xml
+++ b/pom.xml
@@ -55,15 +55,32 @@
<!-- HBase versions -->
<hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+
+ <!-- Kafka versions -->
<kafka.version>0.10.1.0</kafka.version>
- <!-- Hadoop deps, keep compatible with hadoop2.version -->
+ <!-- Spark versions -->
+ <spark.version>1.6.3</spark.version>
+ <kryo.version>4.0.0</kryo.version>
+ <reflections.version>0.9.10</reflections.version>
+
+ <!-- Calcite Version -->
+ <calcite.version>1.11.0</calcite.version>
+ <avatica.version>1.9.0</avatica.version>
+
+ <!-- Hadoop Common deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
<curator.version>2.7.1</curator.version>
- <jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
+ <jsch.version>0.1.53</jsch.version>
<commons-cli.version>1.2</commons-cli.version>
+ <commons-lang.version>2.6</commons-lang.version>
+ <commons-io.version>2.5</commons-io.version>
+ <commons-math3.version>3.6.1</commons-math3.version>
+
+ <!-- Calcite deps, keep compatible with calcite.version -->
+ <jackson.version>2.6.3</jackson.version>
<!-- Test Dependency versions -->
<antlr.version>3.4</antlr.version>
@@ -75,26 +92,16 @@
<jamm.version>0.3.1</jamm.version>
<!-- Commons -->
- <commons-lang.version>2.6</commons-lang.version>
<commons-lang3.version>3.4</commons-lang3.version>
<commons-collections.version>3.2.2</commons-collections.version>
- <commons-io.version>2.5</commons-io.version>
<commons-daemon.version>1.0.15</commons-daemon.version>
<commons-email.version>1.4</commons-email.version>
- <commons-math3.version>3.6.1</commons-math3.version>
-
- <!-- Spark -->
- <spark.version>1.6.3</spark.version>
<!-- Utility -->
<log4j.version>1.2.17</log4j.version>
<slf4j.version>1.7.21</slf4j.version>
- <reflections.version>0.9.10</reflections.version>
- <jsch.version>0.1.53</jsch.version>
<xerces.version>2.11.0</xerces.version>
<xalan.version>2.7.2</xalan.version>
- <extendedset.version>1.3.4</extendedset.version>
- <kryo.version>4.0.0</kryo.version>
<ehcache.version>2.10.2.2.21</ehcache.version>
<apache-httpclient.version>4.2.5</apache-httpclient.version>
<roaring.version>0.6.18</roaring.version>
@@ -111,10 +118,6 @@
<opensaml.version>2.6.4</opensaml.version>
<aspectj.version>1.8.9</aspectj.version>
- <!-- Calcite Version -->
- <calcite.version>1.11.0</calcite.version>
- <avatica.version>1.9.0</avatica.version>
-
<!-- Sonar -->
<sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
<sonar.dynamicAnalysis>reuseReports</sonar.dynamicAnalysis>
@@ -279,6 +282,11 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-tool-assembly</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
<artifactId>kylin-it</artifactId>
<version>${project.version}</version>
</dependency>
@@ -390,6 +398,66 @@
<version>${hadoop2.version}</version>
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>${jsr305.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <version>${jsch.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>${commons-cli.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons-lang.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>${commons-math3.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons-io.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- HBase2 dependencies -->
<dependency>
@@ -461,6 +529,11 @@
</dependency>
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica-core</artifactId>
+ <version>${avatica.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
<version>${avatica.version}</version>
</dependency>
@@ -476,22 +549,44 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+
<!-- Spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<version>${spark.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
<version>${spark.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.10</artifactId>
<version>${spark.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.esotericsoftware</groupId>
+ <artifactId>kryo-shaded</artifactId>
+ <version>${kryo.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>${reflections.version}</version>
+ <scope>provided</scope>
</dependency>
+ <!-- Kafka dependency -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
@@ -501,26 +596,6 @@
<!-- Other dependencies -->
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>${junit.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <version>${zookeeper.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- <version>${commons-cli.version}</version>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>${commons-lang.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
@@ -531,21 +606,15 @@
<version>${commons-collections.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <version>${commons-math3.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>${commons-io.version}</version>
- </dependency>
- <dependency>
<groupId>commons-daemon</groupId>
<artifactId>commons-daemon</artifactId>
<version>${commons-daemon.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-email</artifactId>
+ <version>${commons-email.version}</version>
+ </dependency>
<!-- Logging -->
<dependency>
@@ -571,40 +640,18 @@
<version>${slf4j.version}</version>
</dependency>
+ <!-- Test -->
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-email</artifactId>
- <version>${commons-email.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <version>${jsr305.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
- <dependency>
- <groupId>org.reflections</groupId>
- <artifactId>reflections</artifactId>
- <version>${reflections.version}</version>
- </dependency>
- <dependency>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- <version>${jsch.version}</version>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.dbunit</groupId>
<artifactId>dbunit</artifactId>
<version>${dbunit.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
@@ -615,6 +662,7 @@
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>xerces</groupId>
@@ -627,31 +675,10 @@
<version>${xalan.version}</version>
</dependency>
<dependency>
- <groupId>com.n3twork.druid</groupId>
- <artifactId>extendedset</artifactId>
- <version>${extendedset.version}</version>
- </dependency>
- <dependency>
- <groupId>com.esotericsoftware</groupId>
- <artifactId>kryo-shaded</artifactId>
- <version>${kryo.version}</version>
- </dependency>
- <dependency>
<groupId>com.github.jbellis</groupId>
<artifactId>jamm</artifactId>
<version>${jamm.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>${curator.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
@@ -1036,6 +1063,7 @@
<module>jdbc</module>
<module>assembly</module>
<module>tool</module>
+ <module>tool-assembly</module>
<module>kylin-it</module>
<module>tomcat-ext</module>
</modules>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 6ab74a7..d35c74c 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -47,10 +47,16 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-storage</artifactId>
</dependency>
+
+ <!-- Provided -->
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
<!-- Env & Test -->
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 454ed6b..3c96e60 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -233,6 +233,10 @@
<artifactId>jetty-all</artifactId>
<groupId>org.eclipse.jetty.aggregate</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>calcite-core</artifactId>
+ <groupId>org.apache.calcite</groupId>
+ </exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index e2fe448..da5e033 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -35,23 +35,19 @@
<dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-engine-mr</artifactId>
+ <artifactId>kylin-core-common</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
+ <artifactId>kylin-engine-mr</artifactId>
</dependency>
+ <!-- Provided -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
</dependency>
+
<!-- Env & Test -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -76,9 +72,5 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
-
-
-
</dependencies>
-
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 3aea531..29ca7e5 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -32,8 +32,11 @@
<version>2.0.0-SNAPSHOT</version>
</parent>
- <dependencies>
+ <properties>
+ <shadeBase>org.apache.kylin.coprocessor.shaded</shadeBase>
+ </properties>
+ <dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-engine-mr</artifactId>
@@ -119,14 +122,20 @@
<include>org.apache.kylin:kylin-core-metadata</include>
<include>org.apache.kylin:kylin-core-dictionary</include>
<include>org.apache.kylin:kylin-core-cube</include>
- <include>com.ning:compress-lzf</include>
<include>org.roaringbitmap:RoaringBitmap</include>
<include>com.tdunning:t-digest</include>
- <!-- below for inverted index only -->
- <include>com.n3twork.druid:extendedset</include>
- <include>org.apache.commons:commons-lang3</include>
</includes>
</artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.tdunning</pattern>
+ <shadedPattern>${shadeBase}.com.tdunning</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.roaringbitmap</pattern>
+ <shadedPattern>${shadeBase}.org.roaringbitmap</shadedPattern>
+ </relocation>
+ </relocations>
<filters>
<filter>
<artifact>*:*</artifact>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/tool-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml
new file mode 100644
index 0000000..67d92d4
--- /dev/null
+++ b/tool-assembly/pom.xml
@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-tool-assembly</artifactId>
+ <name>Apache Kylin - Tool Assembly</name>
+ <packaging>jar</packaging>
+ <description>Apache Kylin - Tool Assembly</description>
+
+ <parent>
+ <artifactId>kylin</artifactId>
+ <groupId>org.apache.kylin</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+
+ <properties>
+ <shadeBase>org.apache.kylin.tool.shaded</shadeBase>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-tool</artifactId>
+ </dependency>
+
+ <!-- As KylinConfigCLI will be called before Hbase/Hive/Hadoop dependency loaded, has to define the following as compile in tool -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>assembly</shadedClassifierName>
+ <artifactSet>
+ <includes>
+ <!-- shade the httpcore to avoid the lower version conflict with HBase one -->
+ <include>org.apache.httpcomponents:httpcore</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.slf4j:slf4j-log4j12</include>
+ <include>log4j:log4j</include>
+ <include>commons-io:commons-io</include>
+ <include>commons-lang:commons-lang</include>
+ <include>com.google.guava:guava</include>
+ <include>org.apache.kylin:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>${shadeBase}.com.google.common</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.io</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.io</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons.lang</pattern>
+ <shadedPattern>${shadeBase}.org.apache.commons.lang</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.http</pattern>
+ <shadedPattern>${shadeBase}.org.apache.http</shadedPattern>
+ </relocation>
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7c2632d9/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 91040d4..278c2b8 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -81,43 +81,4 @@
</dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>assembly</shadedClassifierName>
- <artifactSet>
- <includes>
- <!-- shade the httpcore to avoid the lower version conflict with HBase one -->
- <include>org.apache.httpcomponents:httpcore</include>
- <include>org.apache.kylin:kylin-tool</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
</project>
[03/17] kylin git commit: minor,
use local metadata to avoid conflict when running concurrently
Posted by bi...@apache.org.
minor, use local metadata to avoid conflict when running concurrently
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d135bdb0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d135bdb0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d135bdb0
Branch: refs/heads/KYLIN-2428
Commit: d135bdb0eed6118245a606fb0bca996420dee70f
Parents: 8263752
Author: etherge <et...@163.com>
Authored: Wed Feb 8 18:40:07 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Feb 8 18:54:06 2017 +0800
----------------------------------------------------------------------
kylin-it/pom.xml | 3 +++
.../kylin/job/BaseTestDistributedScheduler.java | 14 ++++++++++----
2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d135bdb0/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index 91104ba..d58a895 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -36,6 +36,7 @@
<properties>
<hdp.version/>
<fastBuildMode/>
+ <buildCubeUsingProvidedData/>
<engineType/>
</properties>
@@ -316,6 +317,7 @@
<arguments>
<argument>-Dhdp.version=${hdp.version}</argument>
<argument>-DfastBuildMode=${fastBuildMode}</argument>
+ <argument>-DbuildCubeUsingProvidedData=${buildCubeUsingProvidedData}</argument>
<argument>-DengineType=${engineType}</argument>
<argument>-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties</argument>
<argument>-classpath</argument>
@@ -338,6 +340,7 @@
<arguments>
<argument>-Dhdp.version=${hdp.version}</argument>
<argument>-DfastBuildMode=${fastBuildMode}</argument>
+ <argument>-DbuildCubeUsingProvidedData=${buildCubeUsingProvidedData}</argument>
<argument>-Dlog4j.configuration=file:${project.basedir}/..//build/conf/kylin-tools-log4j.properties</argument>
<argument>-classpath</argument>
<classpath/>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d135bdb0/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
index 1a0a39d..2f37a50 100644
--- a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
+++ b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
import javax.annotation.Nullable;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
@@ -48,6 +49,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
+import com.google.common.io.Files;
public class BaseTestDistributedScheduler extends HBaseMetadataTestCase {
static ExecutableManager execMgr;
@@ -57,13 +59,13 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase {
static KylinConfig kylinConfig1;
static KylinConfig kylinConfig2;
static CuratorFramework zkClient;
+ static File localMetaDir;
static final String SEGMENT_ID = "segmentId";
static final String segmentId1 = "segmentId1";
static final String segmentId2 = "segmentId2";
static final String serverName1 = "serverName1";
static final String serverName2 = "serverName2";
- static final String confSrcPath = "../examples/test_case_data/sandbox/kylin.properties";
static final String confDstPath1 = "target/kylin_metadata_dist_lock_test1/kylin.properties";
static final String confDstPath2 = "target/kylin_metadata_dist_lock_test2/kylin.properties";
@@ -77,14 +79,17 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase {
new File(confDstPath1).getParentFile().mkdirs();
new File(confDstPath2).getParentFile().mkdirs();
KylinConfig srcConfig = KylinConfig.getInstanceFromEnv();
+
+ localMetaDir = Files.createTempDir();
String backup = srcConfig.getMetadataUrl();
- srcConfig.setProperty("kylin.metadata.url", "kylin_metadata_dist_lock_test@hbase");
+ srcConfig.setProperty("kylin.metadata.url", localMetaDir.getAbsolutePath());
srcConfig.writeProperties(new File(confDstPath1));
srcConfig.writeProperties(new File(confDstPath2));
srcConfig.setProperty("kylin.metadata.url", backup);
+
kylinConfig1 = KylinConfig.createInstanceFromUri(new File(confDstPath1).getAbsolutePath());
kylinConfig2 = KylinConfig.createInstanceFromUri(new File(confDstPath2).getAbsolutePath());
-
+
initZk();
if (jobLock == null)
@@ -130,7 +135,8 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase {
zkClient.close();
zkClient = null;
}
-
+
+ FileUtils.deleteDirectory(localMetaDir);
System.clearProperty("kylin.job.lock");
staticCleanupTestMetadata();
}
[08/17] kylin git commit: KYLIN-2308 bug fix and improve user
experience
Posted by bi...@apache.org.
KYLIN-2308 bug fix and improve user experience
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e9f0908
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e9f0908
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e9f0908
Branch: refs/heads/KYLIN-2428
Commit: 1e9f0908b9942113bbd251dc5eeba71973cc77e1
Parents: 7742b31
Author: kangkaisen <ka...@163.com>
Authored: Thu Feb 9 12:06:56 2017 +0800
Committer: kangkaisen <ka...@163.com>
Committed: Thu Feb 9 15:54:01 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 36 ++++++++++++++++++--
webapp/app/js/controllers/cubeSchema.js | 8 +++++
.../cubeDesigner/advanced_settings.html | 2 +-
3 files changed, 43 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 39d36b0..7cdb1cd 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -342,12 +342,44 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
return assignedMeasures;
};
- if ($scope.getAllMeasureNames().length != $scope.getAssignedMeasureNames().length) {
+ $scope.rmDeprecatedMeasureNames = function () {
+ var allMeasureNames = $scope.getAllMeasureNames();
+ var tmpColumnFamily = $scope.cubeMetaFrame.hbase_mapping.column_family;
+
+ angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function (colFamily,index1) {
+ angular.forEach(colFamily.columns[0].measure_refs, function (measureName, index2) {
+ var allIndex = allMeasureNames.indexOf(measureName);
+ if (allIndex == -1) {
+ tmpColumnFamily[index1].columns[0].measure_refs.splice(index2, 1);
+ }
+
+ if (tmpColumnFamily[index1].columns[0].measure_refs == 0) {
+ tmpColumnFamily.splice(index1, 1);
+ }
+ });
+ });
+
+ $scope.cubeMetaFrame.hbase_mapping.column_family = tmpColumnFamily;
+ };
+
+ if ($scope.getAssignedMeasureNames().length == 0) {
$scope.initColumnFamily();
+ } else {
+ $scope.rmDeprecatedMeasureNames();
+ if ($scope.getAllMeasureNames().length > $scope.getAssignedMeasureNames().length) {
+ $scope.initColumnFamily();
+ }
}
-
$scope.addColumnFamily = function () {
+ var isCFEmpty = _.some($scope.cubeMetaFrame.hbase_mapping.column_family, function(colFamily) {
+ return colFamily.columns[0].measure_refs.length == 0;
+ });
+
+ if (isCFEmpty === true) {
+ return;
+ }
+
var colFamily = $scope.newColFamily($scope.cubeMetaFrame.hbase_mapping.column_family.length + 1);
$scope.cubeMetaFrame.hbase_mapping.column_family.push(colFamily);
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/js/controllers/cubeSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js
index 17371f2..4beeebe 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -287,6 +287,14 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic
errors.push("All measures need to be assigned to column family");
}
+ var isCFEmpty = _.some($scope.cubeMetaFrame.hbase_mapping.column_family, function(colFamily) {
+ return colFamily.columns[0].measure_refs.length == 0;
+ });
+
+ if (isCFEmpty == true) {
+ errors.push("Each column family can't not be empty");
+ }
+
var errorInfo = "";
angular.forEach(errors,function(item){
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e9f0908/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index bf95256..f26722b 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -453,7 +453,7 @@
<td class="col-xs-1">
<!--Remove Button -->
- <button class="btn btn-xs btn-info" ng-click="removeElement(cubeMetaFrame.hbase_mapping.column_family, colFamily)">
+ <button ng-if="colFamily.columns[0].measure_refs.length == 0" class="btn btn-xs btn-info" ng-click="removeElement(cubeMetaFrame.hbase_mapping.column_family, colFamily)">
<i class="fa fa-minus"></i>
</button>
</td>
[15/17] kylin git commit: KYLIN-2436 add a configuration knob to
disable spilling of aggregation cache
Posted by bi...@apache.org.
KYLIN-2436 add a configuration knob to disable spilling of aggregation cache
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ecf6a69f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ecf6a69f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ecf6a69f
Branch: refs/heads/KYLIN-2428
Commit: ecf6a69fece7cbda3a9bd8d678c928224ce677aa
Parents: 41332f3
Author: gaodayue <ga...@meituan.com>
Authored: Mon Feb 6 14:59:06 2017 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Fri Feb 10 12:43:34 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +
.../cube/inmemcubing/InMemCubeBuilder.java | 2 +-
.../kylin/gridtable/GTAggregateScanner.java | 54 ++++----
.../apache/kylin/gridtable/GTScanRequest.java | 6 +-
.../GTScanSelfTerminatedException.java | 4 +
.../gridtable/AggregationCacheSpillTest.java | 4 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 1 +
.../coprocessor/endpoint/CubeVisitService.java | 12 +-
.../endpoint/generated/CubeVisitProtos.java | 123 ++++++++++++++++---
.../endpoint/protobuf/CubeVisit.proto | 1 +
10 files changed, 162 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecf6a69f/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index dce4149..81f0187 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -609,6 +609,10 @@ abstract public class KylinConfigBase implements Serializable {
return Double.parseDouble(this.getOptional("kylin.storage.hbase.coprocessor-mem-gb", "3.0"));
}
+ public boolean getQueryCoprocessorSpillEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.storage.hbase.coprocessor-spill-enabled", "true"));
+ }
+
public int getQueryCoprocessorTimeoutSeconds() {
return Integer.parseInt(this.getOptional("kylin.storage.hbase.coprocessor-timeout-seconds", "0"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecf6a69f/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
index 398c594..a74f0c0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/inmemcubing/InMemCubeBuilder.java
@@ -335,7 +335,7 @@ public class InMemCubeBuilder extends AbstractInMemCubeBuilder {
Pair<ImmutableBitSet, ImmutableBitSet> dimensionMetricsBitSet = InMemCubeBuilderUtils.getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
GTScanRequest req = new GTScanRequestBuilder().setInfo(baseCuboid.getInfo()).setRanges(null).setDimensions(null).setAggrGroupBy(dimensionMetricsBitSet.getFirst()).setAggrMetrics(dimensionMetricsBitSet.getSecond()).setAggrMetricsFuncs(metricsAggrFuncs).setFilterPushDown(null).createGTScanRequest();
- GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req, Long.MAX_VALUE);
+ GTAggregateScanner aggregationScanner = new GTAggregateScanner(baseInput, req);
aggregationScanner.trackMemoryLevel(baseCuboidMemTracker);
int count = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecf6a69f/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 9158aa3..147dbc1 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -63,15 +63,20 @@ public class GTAggregateScanner implements IGTScanner {
final String[] metricsAggrFuncs;
final IGTScanner inputScanner;
final AggregationCache aggrCache;
- final long spillThreshold;
+ final long spillThreshold; // 0 means no memory control && no spill
final int storagePushDownLimit;//default to be Int.MAX
final long deadline;
+ final boolean spillEnabled;
private int aggregatedRowCount = 0;
private MemoryWaterLevel memTracker;
private boolean[] aggrMask;
- public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req, long deadline) {
+ public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req) {
+ this(inputScanner, req, Long.MAX_VALUE, true);
+ }
+
+ public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req, long deadline, boolean spillEnabled) {
if (!req.hasAggregation())
throw new IllegalStateException();
@@ -86,6 +91,7 @@ public class GTAggregateScanner implements IGTScanner {
this.aggrMask = new boolean[metricsAggrFuncs.length];
this.storagePushDownLimit = req.getStoragePushDownLimit();
this.deadline = deadline;
+ this.spillEnabled = spillEnabled;
Arrays.fill(aggrMask, true);
}
@@ -276,10 +282,16 @@ public class GTAggregateScanner implements IGTScanner {
if (memTracker != null) {
memTracker.markHigh();
}
- if (spillThreshold > 0) {
+
+ final long estMemSize = estimatedMemSize();
+ if (spillThreshold > 0 && estMemSize > spillThreshold) {
// spill to disk when aggBufMap used too large memory
- if (estimatedMemSize() > spillThreshold) {
- spillBuffMap();
+ if (spillEnabled) {
+ spillBuffMap(estMemSize);
+ aggBufMap = createBuffMap();
+
+ } else {
+ throw new GTScanSelfTerminatedException("Aggregation using more than " + spillThreshold + " memory and spill is disabled");
}
}
}
@@ -306,17 +318,13 @@ public class GTAggregateScanner implements IGTScanner {
return true;
}
- private void spillBuffMap() throws RuntimeException {
- if (aggBufMap.isEmpty())
- return;
-
+ private void spillBuffMap(long estMemSize) throws RuntimeException {
try {
- Dump dump = new Dump(aggBufMap);
+ Dump dump = new Dump(aggBufMap, estMemSize);
dump.flush();
dumps.add(dump);
- aggBufMap = createBuffMap();
} catch (Exception e) {
- throw new RuntimeException("AggregationCache spill failed: " + e.getMessage());
+ throw new RuntimeException("AggregationCache failed to spill", e);
}
}
@@ -372,9 +380,9 @@ public class GTAggregateScanner implements IGTScanner {
};
} else {
// the spill case
-
- logger.info("Last spill, current AggregationCache memory estimated size is: " + getEstimateSizeOfAggrCache());
- this.spillBuffMap();
+ if (!aggBufMap.isEmpty()) {
+ this.spillBuffMap(getEstimateSizeOfAggrCache()); // TODO allow merge in-mem map with spilled dumps
+ }
return new Iterator<GTRecord>() {
final DumpMerger merger = new DumpMerger(dumps);
@@ -430,12 +438,16 @@ public class GTAggregateScanner implements IGTScanner {
}
class Dump implements Iterable<Pair<byte[], byte[]>> {
- File dumpedFile;
- DataInputStream dis;
+ final File dumpedFile;
SortedMap<byte[], MeasureAggregator[]> buffMap;
+ final long estMemSize;
- public Dump(SortedMap<byte[], MeasureAggregator[]> buffMap) throws IOException {
+ DataInputStream dis;
+
+ public Dump(SortedMap<byte[], MeasureAggregator[]> buffMap, long estMemSize) throws IOException {
+ this.dumpedFile = File.createTempFile("KYLIN_SPILL_", ".tmp");
this.buffMap = buffMap;
+ this.estMemSize = estMemSize;
}
@Override
@@ -482,13 +494,13 @@ public class GTAggregateScanner implements IGTScanner {
}
public void flush() throws IOException {
+ logger.info("AggregationCache(size={} est_mem_size={} threshold={}) will spill to {}",
+ buffMap.size(), estMemSize, spillThreshold, dumpedFile.getAbsolutePath());
+
if (buffMap != null) {
DataOutputStream dos = null;
Object[] aggrResult = null;
try {
- dumpedFile = File.createTempFile("KYLIN_AGGR_", ".tmp");
-
- logger.info("AggregationCache will dump to file: " + dumpedFile.getAbsolutePath());
dos = new DataOutputStream(new FileOutputStream(dumpedFile));
dos.writeInt(buffMap.size());
for (Entry<byte[], MeasureAggregator[]> entry : buffMap.entrySet()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecf6a69f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 7976e3f..7c94f5a 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -166,13 +166,13 @@ public class GTScanRequest {
* Refer to CoprocessorBehavior for explanation
*/
public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, long deadline) throws IOException {
- return decorateScanner(scanner, filterToggledOn, aggrToggledOn, false, deadline);
+ return decorateScanner(scanner, filterToggledOn, aggrToggledOn, false, deadline, true);
}
/**
* hasPreFiltered indicate the data has been filtered before scanning
*/
- public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, boolean hasPreFiltered, long deadline) throws IOException {
+ public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, boolean hasPreFiltered, long deadline, boolean spillEnabled) throws IOException {
IGTScanner result = scanner;
if (!filterToggledOn) { //Skip reading this section if you're not profiling!
int scanned = lookAndForget(result);
@@ -194,7 +194,7 @@ public class GTScanRequest {
} else if (this.hasAggregation()) {
logger.info("pre aggregating results before returning");
this.doingStorageAggregation = true;
- result = new GTAggregateScanner(result, this, deadline);
+ result = new GTAggregateScanner(result, this, deadline, spillEnabled);
} else {
logger.info("has no aggregation, skip it");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecf6a69f/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java
index 4775ac6..30d3aaa 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanSelfTerminatedException.java
@@ -18,6 +18,10 @@
package org.apache.kylin.gridtable;
+/**
+ * Implementations of {@link IGTScanner} should throw {@link GTScanSelfTerminatedException} or its subclasses
+ * in cases where the scan runs out of resources (time, memory, etc) and can not be continued.
+ */
public class GTScanSelfTerminatedException extends RuntimeException {
public GTScanSelfTerminatedException(String s) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecf6a69f/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
index 74c2659..7abb069 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheSpillTest.java
@@ -84,7 +84,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(0, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest();
- GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest, Long.MAX_VALUE);
+ GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest);
int count = 0;
for (GTRecord record : scanner) {
@@ -126,7 +126,7 @@ public class AggregationCacheSpillTest extends LocalFileMetadataTestCase {
// all-in-mem testcase
GTScanRequest scanRequest = new GTScanRequestBuilder().setInfo(INFO).setRanges(null).setDimensions(new ImmutableBitSet(0, 3)).setAggrGroupBy(new ImmutableBitSet(1, 3)).setAggrMetrics(new ImmutableBitSet(3, 6)).setAggrMetricsFuncs(new String[] { "SUM", "SUM", "COUNT_DISTINCT" }).setFilterPushDown(null).setAggCacheMemThreshold(0.5).createGTScanRequest();
- GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest, Long.MAX_VALUE);
+ GTAggregateScanner scanner = new GTAggregateScanner(inputScanner, scanRequest);
int count = 0;
for (GTRecord record : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecf6a69f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index cad5a3f..dd9f74c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -163,6 +163,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
if (queryId != null) {
builder.setQueryId(queryId);
}
+ builder.setSpillEnabled(cubeSeg.getConfig().getQueryCoprocessorSpillEnabled());
for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) {
executorService.submit(new Runnable() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecf6a69f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 810747f..3e0a065 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -51,6 +51,7 @@ import org.apache.kylin.cube.kv.RowConstants;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanExceedThresholdException;
import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.gridtable.GTScanTimeoutException;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
@@ -280,7 +281,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, request.getRowkeyPreambleSize(), behavior.delayToggledOn());
IGTScanner rawScanner = store.scan(scanReq);
- IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, behavior.filterToggledOn(), behavior.aggrToggledOn(), deadline);
+ IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, behavior.filterToggledOn(), behavior.aggrToggledOn(), false, deadline, request.getSpillEnabled());
ByteBuffer buffer = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
@@ -315,12 +316,11 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
break;
}
}
- } catch (GTScanTimeoutException e) {
+ } catch (GTScanSelfTerminatedException e) {
+ // the query is using too much resource, we mark it as abnormal finish instead of
+ // throwing RuntimeException to avoid client retrying RPC.
scanNormalComplete.setValue(false);
- logger.info("The cube visit did not finish normally because scan timeout", e);
- } catch (GTScanExceedThresholdException e) {
- scanNormalComplete.setValue(false);
- logger.info("The cube visit did not finish normally because scan num exceeds threshold", e);
+ logger.warn("Abort scan: {}", e.getMessage());
} finally {
finalScanner.close();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecf6a69f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index 628d453..def0182 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -107,6 +107,16 @@ public final class CubeVisitProtos {
*/
com.google.protobuf.ByteString
getQueryIdBytes();
+
+ // optional bool spillEnabled = 7 [default = true];
+ /**
+ * <code>optional bool spillEnabled = 7 [default = true];</code>
+ */
+ boolean hasSpillEnabled();
+ /**
+ * <code>optional bool spillEnabled = 7 [default = true];</code>
+ */
+ boolean getSpillEnabled();
}
/**
* Protobuf type {@code CubeVisitRequest}
@@ -192,6 +202,11 @@ public final class CubeVisitProtos {
queryId_ = input.readBytes();
break;
}
+ case 56: {
+ bitField0_ |= 0x00000020;
+ spillEnabled_ = input.readBool();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -921,6 +936,22 @@ public final class CubeVisitProtos {
}
}
+ // optional bool spillEnabled = 7 [default = true];
+ public static final int SPILLENABLED_FIELD_NUMBER = 7;
+ private boolean spillEnabled_;
+ /**
+ * <code>optional bool spillEnabled = 7 [default = true];</code>
+ */
+ public boolean hasSpillEnabled() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>optional bool spillEnabled = 7 [default = true];</code>
+ */
+ public boolean getSpillEnabled() {
+ return spillEnabled_;
+ }
+
private void initFields() {
gtScanRequest_ = com.google.protobuf.ByteString.EMPTY;
hbaseRawScan_ = com.google.protobuf.ByteString.EMPTY;
@@ -928,6 +959,7 @@ public final class CubeVisitProtos {
hbaseColumnsToGT_ = java.util.Collections.emptyList();
kylinProperties_ = "";
queryId_ = "";
+ spillEnabled_ = true;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -975,6 +1007,9 @@ public final class CubeVisitProtos {
if (((bitField0_ & 0x00000010) == 0x00000010)) {
output.writeBytes(6, getQueryIdBytes());
}
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ output.writeBool(7, spillEnabled_);
+ }
getUnknownFields().writeTo(output);
}
@@ -1008,6 +1043,10 @@ public final class CubeVisitProtos {
size += com.google.protobuf.CodedOutputStream
.computeBytesSize(6, getQueryIdBytes());
}
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(7, spillEnabled_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -1058,6 +1097,11 @@ public final class CubeVisitProtos {
result = result && getQueryId()
.equals(other.getQueryId());
}
+ result = result && (hasSpillEnabled() == other.hasSpillEnabled());
+ if (hasSpillEnabled()) {
+ result = result && (getSpillEnabled()
+ == other.getSpillEnabled());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -1095,6 +1139,10 @@ public final class CubeVisitProtos {
hash = (37 * hash) + QUERYID_FIELD_NUMBER;
hash = (53 * hash) + getQueryId().hashCode();
}
+ if (hasSpillEnabled()) {
+ hash = (37 * hash) + SPILLENABLED_FIELD_NUMBER;
+ hash = (53 * hash) + hashBoolean(getSpillEnabled());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -1221,6 +1269,8 @@ public final class CubeVisitProtos {
bitField0_ = (bitField0_ & ~0x00000010);
queryId_ = "";
bitField0_ = (bitField0_ & ~0x00000020);
+ spillEnabled_ = true;
+ bitField0_ = (bitField0_ & ~0x00000040);
return this;
}
@@ -1278,6 +1328,10 @@ public final class CubeVisitProtos {
to_bitField0_ |= 0x00000010;
}
result.queryId_ = queryId_;
+ if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+ to_bitField0_ |= 0x00000020;
+ }
+ result.spillEnabled_ = spillEnabled_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -1339,6 +1393,9 @@ public final class CubeVisitProtos {
queryId_ = other.queryId_;
onChanged();
}
+ if (other.hasSpillEnabled()) {
+ setSpillEnabled(other.getSpillEnabled());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -1899,6 +1956,39 @@ public final class CubeVisitProtos {
return this;
}
+ // optional bool spillEnabled = 7 [default = true];
+ private boolean spillEnabled_ = true;
+ /**
+ * <code>optional bool spillEnabled = 7 [default = true];</code>
+ */
+ public boolean hasSpillEnabled() {
+ return ((bitField0_ & 0x00000040) == 0x00000040);
+ }
+ /**
+ * <code>optional bool spillEnabled = 7 [default = true];</code>
+ */
+ public boolean getSpillEnabled() {
+ return spillEnabled_;
+ }
+ /**
+ * <code>optional bool spillEnabled = 7 [default = true];</code>
+ */
+ public Builder setSpillEnabled(boolean value) {
+ bitField0_ |= 0x00000040;
+ spillEnabled_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bool spillEnabled = 7 [default = true];</code>
+ */
+ public Builder clearSpillEnabled() {
+ bitField0_ = (bitField0_ & ~0x00000040);
+ spillEnabled_ = true;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:CubeVisitRequest)
}
@@ -4253,25 +4343,26 @@ public final class CubeVisitProtos {
java.lang.String[] descriptorData = {
"\npstorage-hbase/src/main/java/org/apache" +
"/kylin/storage/hbase/cube/v2/coprocessor" +
- "/endpoint/protobuf/CubeVisit.proto\"\323\001\n\020C" +
+ "/endpoint/protobuf/CubeVisit.proto\"\357\001\n\020C" +
"ubeVisitRequest\022\025\n\rgtScanRequest\030\001 \002(\014\022\024" +
"\n\014hbaseRawScan\030\002 \002(\014\022\032\n\022rowkeyPreambleSi" +
"ze\030\003 \002(\005\0223\n\020hbaseColumnsToGT\030\004 \003(\0132\031.Cub" +
"eVisitRequest.IntList\022\027\n\017kylinProperties" +
- "\030\005 \002(\t\022\017\n\007queryId\030\006 \001(\t\032\027\n\007IntList\022\014\n\004in" +
- "ts\030\001 \003(\005\"\321\002\n\021CubeVisitResponse\022\026\n\016compre" +
- "ssedRows\030\001 \002(\014\022\'\n\005stats\030\002 \002(\0132\030.CubeVisi",
- "tResponse.Stats\032\372\001\n\005Stats\022\030\n\020serviceStar" +
- "tTime\030\001 \001(\003\022\026\n\016serviceEndTime\030\002 \001(\003\022\027\n\017s" +
- "cannedRowCount\030\003 \001(\003\022\032\n\022aggregatedRowCou" +
- "nt\030\004 \001(\003\022\025\n\rsystemCpuLoad\030\005 \001(\001\022\036\n\026freeP" +
- "hysicalMemorySize\030\006 \001(\001\022\031\n\021freeSwapSpace" +
- "Size\030\007 \001(\001\022\020\n\010hostname\030\010 \001(\t\022\016\n\006etcMsg\030\t" +
- " \001(\t\022\026\n\016normalComplete\030\n \001(\0052F\n\020CubeVisi" +
- "tService\0222\n\tvisitCube\022\021.CubeVisitRequest" +
- "\032\022.CubeVisitResponseB`\nEorg.apache.kylin" +
- ".storage.hbase.cube.v2.coprocessor.endpo",
- "int.generatedB\017CubeVisitProtosH\001\210\001\001\240\001\001"
+ "\030\005 \002(\t\022\017\n\007queryId\030\006 \001(\t\022\032\n\014spillEnabled\030" +
+ "\007 \001(\010:\004true\032\027\n\007IntList\022\014\n\004ints\030\001 \003(\005\"\321\002\n" +
+ "\021CubeVisitResponse\022\026\n\016compressedRows\030\001 \002",
+ "(\014\022\'\n\005stats\030\002 \002(\0132\030.CubeVisitResponse.St" +
+ "ats\032\372\001\n\005Stats\022\030\n\020serviceStartTime\030\001 \001(\003\022" +
+ "\026\n\016serviceEndTime\030\002 \001(\003\022\027\n\017scannedRowCou" +
+ "nt\030\003 \001(\003\022\032\n\022aggregatedRowCount\030\004 \001(\003\022\025\n\r" +
+ "systemCpuLoad\030\005 \001(\001\022\036\n\026freePhysicalMemor" +
+ "ySize\030\006 \001(\001\022\031\n\021freeSwapSpaceSize\030\007 \001(\001\022\020" +
+ "\n\010hostname\030\010 \001(\t\022\016\n\006etcMsg\030\t \001(\t\022\026\n\016norm" +
+ "alComplete\030\n \001(\0052F\n\020CubeVisitService\0222\n\t" +
+ "visitCube\022\021.CubeVisitRequest\032\022.CubeVisit" +
+ "ResponseB`\nEorg.apache.kylin.storage.hba",
+ "se.cube.v2.coprocessor.endpoint.generate" +
+ "dB\017CubeVisitProtosH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -4283,7 +4374,7 @@ public final class CubeVisitProtos {
internal_static_CubeVisitRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_CubeVisitRequest_descriptor,
- new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "RowkeyPreambleSize", "HbaseColumnsToGT", "KylinProperties", "QueryId", });
+ new java.lang.String[] { "GtScanRequest", "HbaseRawScan", "RowkeyPreambleSize", "HbaseColumnsToGT", "KylinProperties", "QueryId", "SpillEnabled", });
internal_static_CubeVisitRequest_IntList_descriptor =
internal_static_CubeVisitRequest_descriptor.getNestedTypes().get(0);
internal_static_CubeVisitRequest_IntList_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/kylin/blob/ecf6a69f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
index 1a56719..c7c2954 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -36,6 +36,7 @@ message CubeVisitRequest {
repeated IntList hbaseColumnsToGT = 4;
required string kylinProperties = 5; // kylin properties
optional string queryId = 6;
+ optional bool spillEnabled = 7 [default = true];
message IntList {
repeated int32 ints = 1;
}
[10/17] kylin git commit: minor, decorateScanner can skip filter only
Posted by bi...@apache.org.
minor, decorateScanner can skip filter only
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f4569619
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f4569619
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f4569619
Branch: refs/heads/KYLIN-2428
Commit: f456961986e5c6ab94080a0fa74fa86c1a33c273
Parents: abf5332
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Feb 9 17:14:57 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Feb 9 17:35:05 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/gridtable/GTScanRequest.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f4569619/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index c45f90c..7976e3f 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -166,13 +166,20 @@ public class GTScanRequest {
* Refer to CoprocessorBehavior for explanation
*/
public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, long deadline) throws IOException {
+ return decorateScanner(scanner, filterToggledOn, aggrToggledOn, false, deadline);
+ }
+
+ /**
+ * hasPreFiltered indicate the data has been filtered before scanning
+ */
+ public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, boolean hasPreFiltered, long deadline) throws IOException {
IGTScanner result = scanner;
if (!filterToggledOn) { //Skip reading this section if you're not profiling!
int scanned = lookAndForget(result);
return new EmptyGTScanner(scanned);
} else {
- if (this.hasFilterPushDown()) {
+ if (this.hasFilterPushDown() && !hasPreFiltered) {
result = new GTFilterScanner(result, this);
}
[16/17] kylin git commit: KYLIN-2437 collect number of bytes scanned
to query metrics
Posted by bi...@apache.org.
KYLIN-2437 collect number of bytes scanned to query metrics
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e09338b3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e09338b3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e09338b3
Branch: refs/heads/KYLIN-2428
Commit: e09338b34c0b07a7167096e45bf9185aa0d0cbd5
Parents: ecf6a69
Author: gaodayue <ga...@meituan.com>
Authored: Wed Feb 8 13:59:31 2017 +0800
Committer: gaodayue <ga...@meituan.com>
Committed: Fri Feb 10 15:39:11 2017 +0800
----------------------------------------------------------------------
.../kylin/gridtable/GTAggregateScanner.java | 11 +-
.../apache/kylin/gridtable/GTScanRequest.java | 10 +-
.../apache/kylin/gridtable/ScannerWorker.java | 64 ---------
.../apache/kylin/storage/StorageContext.java | 9 ++
.../storage/gtrecord/CubeSegmentScanner.java | 3 +-
.../kylin/storage/gtrecord/ScannerWorker.java | 71 ++++++++++
.../apache/kylin/rest/response/SQLResponse.java | 10 ++
.../apache/kylin/rest/service/QueryService.java | 9 +-
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 7 +-
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 6 +-
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 17 ++-
.../coprocessor/endpoint/CubeVisitService.java | 142 +++++++++++--------
.../endpoint/generated/CubeVisitProtos.java | 107 ++++++++++++--
.../endpoint/protobuf/CubeVisit.proto | 1 +
14 files changed, 309 insertions(+), 158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
index 147dbc1..dd359f8 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTAggregateScanner.java
@@ -65,7 +65,6 @@ public class GTAggregateScanner implements IGTScanner {
final AggregationCache aggrCache;
final long spillThreshold; // 0 means no memory control && no spill
final int storagePushDownLimit;//default to be Int.MAX
- final long deadline;
final boolean spillEnabled;
private int aggregatedRowCount = 0;
@@ -73,10 +72,10 @@ public class GTAggregateScanner implements IGTScanner {
private boolean[] aggrMask;
public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req) {
- this(inputScanner, req, Long.MAX_VALUE, true);
+ this(inputScanner, req, true);
}
- public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req, long deadline, boolean spillEnabled) {
+ public GTAggregateScanner(IGTScanner inputScanner, GTScanRequest req, boolean spillEnabled) {
if (!req.hasAggregation())
throw new IllegalStateException();
@@ -90,7 +89,6 @@ public class GTAggregateScanner implements IGTScanner {
this.spillThreshold = (long) (req.getAggCacheMemThreshold() * MemoryBudgetController.ONE_GB);
this.aggrMask = new boolean[metricsAggrFuncs.length];
this.storagePushDownLimit = req.getStoragePushDownLimit();
- this.deadline = deadline;
this.spillEnabled = spillEnabled;
Arrays.fill(aggrMask, true);
@@ -145,11 +143,6 @@ public class GTAggregateScanner implements IGTScanner {
long count = 0;
for (GTRecord r : inputScanner) {
- //check deadline
- if (count % GTScanRequest.terminateCheckInterval == 1 && System.currentTimeMillis() > deadline) {
- throw new GTScanTimeoutException("Timeout in GTAggregateScanner with scanned count " + count);
- }
-
if (getNumOfSpills() == 0) {
//check limit
boolean ret = aggrCache.aggregate(r, storagePushDownLimit);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
index 7c94f5a..651e5c4 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java
@@ -156,7 +156,7 @@ public class GTScanRequest {
}
public IGTScanner decorateScanner(IGTScanner scanner) throws IOException {
- return decorateScanner(scanner, true, true, Long.MAX_VALUE);
+ return decorateScanner(scanner, true, true);
}
/**
@@ -165,14 +165,14 @@ public class GTScanRequest {
*
* Refer to CoprocessorBehavior for explanation
*/
- public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, long deadline) throws IOException {
- return decorateScanner(scanner, filterToggledOn, aggrToggledOn, false, deadline, true);
+ public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn) throws IOException {
+ return decorateScanner(scanner, filterToggledOn, aggrToggledOn, false, true);
}
/**
* hasPreFiltered indicate the data has been filtered before scanning
*/
- public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, boolean hasPreFiltered, long deadline, boolean spillEnabled) throws IOException {
+ public IGTScanner decorateScanner(IGTScanner scanner, boolean filterToggledOn, boolean aggrToggledOn, boolean hasPreFiltered, boolean spillEnabled) throws IOException {
IGTScanner result = scanner;
if (!filterToggledOn) { //Skip reading this section if you're not profiling!
int scanned = lookAndForget(result);
@@ -194,7 +194,7 @@ public class GTScanRequest {
} else if (this.hasAggregation()) {
logger.info("pre aggregating results before returning");
this.doingStorageAggregation = true;
- result = new GTAggregateScanner(result, this, deadline, spillEnabled);
+ result = new GTAggregateScanner(result, this, spillEnabled);
} else {
logger.info("has no aggregation, skip it");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java b/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
deleted file mode 100644
index f26d993..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/ScannerWorker.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.gridtable;
-
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Iterator;
-
-import org.apache.kylin.metadata.model.ISegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ScannerWorker {
-
- private static final Logger logger = LoggerFactory.getLogger(ScannerWorker.class);
- private IGTScanner internal = null;
-
- public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage) {
- if (scanRequest == null) {
- logger.info("Segment {} will be skipped", segment);
- internal = new EmptyGTScanner(0);
- return;
- }
-
- final GTInfo info = scanRequest.getInfo();
-
- try {
- IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(ISegment.class, Cuboid.class, GTInfo.class).newInstance(segment, cuboid, info); // default behavior
- internal = rpc.getGTScanner(scanRequest);
- } catch (IOException | InstantiationException | InvocationTargetException | IllegalAccessException | ClassNotFoundException | NoSuchMethodException e) {
- throw new RuntimeException(e);
- }
- }
-
- public Iterator<GTRecord> iterator() {
- return internal.iterator();
- }
-
- public void close() throws IOException {
- internal.close();
- }
-
- public long getScannedRowCount() {
- return internal.getScannedRowCount();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index ab0ea73..708dfde 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -50,6 +50,7 @@ public class StorageContext {
private IStorageQuery storageQuery;
private AtomicLong totalScanCount = new AtomicLong();
+ private AtomicLong totalScanBytes = new AtomicLong();
private Cuboid cuboid;
private boolean partialResultReturned = false;
@@ -161,6 +162,14 @@ public class StorageContext {
return this.totalScanCount.addAndGet(count);
}
+ public long getTotalScanBytes() {
+ return totalScanBytes.get();
+ }
+
+ public long increaseTotalScanBytes(long bytes) {
+ return totalScanBytes.addAndGet(bytes);
+ }
+
public boolean isAcceptPartialResult() {
return acceptPartialResult;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
index 9d6f946..974b8ea 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeSegmentScanner.java
@@ -30,7 +30,6 @@ import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
-import org.apache.kylin.gridtable.ScannerWorker;
import org.apache.kylin.metadata.filter.ITupleFilterTransformer;
import org.apache.kylin.metadata.filter.StringCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -79,7 +78,7 @@ public class CubeSegmentScanner implements IGTScanner {
}
scanRequest = scanRangePlanner.planScanRequest();
String gtStorage = ((GTCubeStorageQueryBase) context.getStorageQuery()).getGTStorage();
- scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest, gtStorage);
+ scanner = new ScannerWorker(cubeSeg, cuboid, scanRequest, gtStorage, context);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
new file mode 100644
index 0000000..2a2a86a
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/ScannerWorker.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.gtrecord;
+
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.gridtable.EmptyGTScanner;
+import org.apache.kylin.gridtable.GTInfo;
+import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.gridtable.GTScanRequest;
+import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.gridtable.IGTStorage;
+import org.apache.kylin.metadata.model.ISegment;
+import org.apache.kylin.storage.StorageContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Iterator;
+
+public class ScannerWorker {
+
+ private static final Logger logger = LoggerFactory.getLogger(ScannerWorker.class);
+ private IGTScanner internal = null;
+
+ public ScannerWorker(ISegment segment, Cuboid cuboid, GTScanRequest scanRequest, String gtStorage, StorageContext context) {
+ if (scanRequest == null) {
+ logger.info("Segment {} will be skipped", segment);
+ internal = new EmptyGTScanner(0);
+ return;
+ }
+
+ final GTInfo info = scanRequest.getInfo();
+
+ try {
+ IGTStorage rpc = (IGTStorage) Class.forName(gtStorage).getConstructor(ISegment.class, Cuboid.class, GTInfo.class, StorageContext.class).newInstance(segment, cuboid, info, context); // default behavior
+ internal = rpc.getGTScanner(scanRequest);
+ } catch (IOException | InstantiationException | InvocationTargetException | IllegalAccessException | ClassNotFoundException | NoSuchMethodException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public Iterator<GTRecord> iterator() {
+ return internal.iterator();
+ }
+
+ public void close() throws IOException {
+ internal.close();
+ }
+
+ public long getScannedRowCount() {
+ return internal.getScannedRowCount();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index 9c4e9da..387e6c9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@ -55,6 +55,8 @@ public class SQLResponse implements Serializable {
protected long totalScanCount;
+ protected long totalScanBytes;
+
protected boolean hitExceptionCache = false;
protected boolean storageCacheUsed = false;
@@ -150,6 +152,14 @@ public class SQLResponse implements Serializable {
this.totalScanCount = totalScanCount;
}
+ public long getTotalScanBytes() {
+ return totalScanBytes;
+ }
+
+ public void setTotalScanBytes(long totalScanBytes) {
+ this.totalScanBytes = totalScanBytes;
+ }
+
public boolean isHitExceptionCache() {
return hitExceptionCache;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 7ce38ea..9ccda03 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -276,6 +276,7 @@ public class QueryService extends BasicService {
stringBuilder.append("Realization Names: ").append(realizationNames).append(newLine);
stringBuilder.append("Cuboid Ids: ").append(cuboidIds).append(newLine);
stringBuilder.append("Total scan count: ").append(response.getTotalScanCount()).append(newLine);
+ stringBuilder.append("Total scan bytes: ").append(response.getTotalScanBytes()).append(newLine);
stringBuilder.append("Result row count: ").append(resultRowCount).append(newLine);
stringBuilder.append("Accept Partial: ").append(request.isAcceptPartial()).append(newLine);
stringBuilder.append("Is Partial Result: ").append(response.isPartial()).append(newLine);
@@ -580,15 +581,18 @@ public class QueryService extends BasicService {
boolean isPartialResult = false;
String cube = "";
- StringBuilder sb = new StringBuilder("Scan count for each storageContext: ");
+ StringBuilder sb = new StringBuilder("Scan stats for each storageContext: ");
long totalScanCount = 0;
+ long totalScanBytes = 0;
if (OLAPContext.getThreadLocalContexts() != null) { // contexts can be null in case of 'explain plan for'
for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) {
if (ctx.realization != null) {
isPartialResult |= ctx.storageContext.isPartialResultReturned();
cube = ctx.realization.getName();
totalScanCount += ctx.storageContext.getTotalScanCount();
- sb.append(ctx.storageContext.getTotalScanCount() + ",");
+ totalScanBytes += ctx.storageContext.getTotalScanBytes();
+ sb.append("{rows=").append(ctx.storageContext.getTotalScanCount()).
+ append(" bytes=").append(ctx.storageContext.getTotalScanBytes()).append("} ");
}
}
}
@@ -596,6 +600,7 @@ public class QueryService extends BasicService {
SQLResponse response = new SQLResponse(columnMetas, results, cube, 0, false, null, isPartialResult);
response.setTotalScanCount(totalScanCount);
+ response.setTotalScanBytes(totalScanBytes);
return response;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index dd9f74c..a2b2611 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -47,6 +47,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanSelfTerminatedException;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.metadata.model.ISegment;
+import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.gtrecord.DummyPartitionStreamer;
import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -69,8 +70,8 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
private static ExecutorService executorService = new LoggableCachedThreadPool();
- public CubeHBaseEndpointRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo) {
- super(segment, cuboid, fullGTInfo);
+ public CubeHBaseEndpointRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo, StorageContext context) {
+ super(segment, cuboid, fullGTInfo, context);
}
private byte[] getByteArrayForShort(short v) {
@@ -198,6 +199,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
if (region == null)
return;
+ context.increaseTotalScanBytes(result.getStats().getScannedBytes());
totalScannedCount.addAndGet(result.getStats().getScannedRowCount());
logger.info(logHeader + getStatsString(region, result));
@@ -280,6 +282,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
Stats stats = result.getStats();
sb.append("Endpoint RPC returned from HTable ").append(cubeSeg.getStorageLocationIdentifier()).append(" Shard ").append(BytesUtil.toHex(region)).append(" on host: ").append(stats.getHostname()).append(".");
sb.append("Total scanned row: ").append(stats.getScannedRowCount()).append(". ");
+ sb.append("Total scanned bytes: ").append(stats.getScannedBytes()).append(". ");
sb.append("Total filtered/aggred row: ").append(stats.getAggregatedRowCount()).append(". ");
sb.append("Time elapsed in EP: ").append(stats.getServiceEndTime() - stats.getServiceStartTime()).append("(ms). ");
sb.append("Server CPU usage: ").append(stats.getSystemCpuLoad()).append(", server physical mem left: ").append(stats.getFreePhysicalMemorySize()).append(", server swap mem left:").append(stats.getFreeSwapSpaceSize()).append(".");
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 05b34c7..11fbb19 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -48,6 +48,7 @@ import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRange;
import org.apache.kylin.gridtable.IGTStorage;
+import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,15 +63,18 @@ public abstract class CubeHBaseRPC implements IGTStorage {
final protected CubeSegment cubeSeg;
final protected Cuboid cuboid;
final protected GTInfo fullGTInfo;
+ final protected StorageContext context;
+
final private RowKeyEncoder fuzzyKeyEncoder;
final private RowKeyEncoder fuzzyMaskEncoder;
- public CubeHBaseRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo) {
+ public CubeHBaseRPC(ISegment segment, Cuboid cuboid, GTInfo fullGTInfo, StorageContext context) {
Preconditions.checkArgument(segment instanceof CubeSegment, "segment must be CubeSegment");
this.cubeSeg = (CubeSegment) segment;
this.cuboid = cuboid;
this.fullGTInfo = fullGTInfo;
+ this.context = context;
this.fuzzyKeyEncoder = new FuzzyKeyEncoder(cubeSeg, cuboid);
this.fuzzyMaskEncoder = new FuzzyMaskEncoder(cubeSeg, cuboid);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index a52af90..b94346c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
@@ -41,6 +42,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.IGTStore;
import org.apache.kylin.metadata.model.ISegment;
+import org.apache.kylin.storage.StorageContext;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -86,8 +88,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
}
- public CubeHBaseScanRPC(ISegment segment, Cuboid cuboid, final GTInfo fullGTInfo) {
- super(segment, cuboid, fullGTInfo);
+ public CubeHBaseScanRPC(ISegment segment, Cuboid cuboid, final GTInfo fullGTInfo, StorageContext context) {
+ super(segment, cuboid, fullGTInfo, context);
}
@Override
@@ -180,12 +182,15 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
final Iterator<Result> allResultsIterator = Iterators.concat(resultIterators.iterator());
CellListIterator cellListIterator = new CellListIterator() {
+ long scanBytes = 0;
+
@Override
public void close() throws IOException {
for (ResultScanner scanner : scanners) {
scanner.close();
}
hbaseTable.close();
+ context.increaseTotalScanBytes(scanBytes);
}
@Override
@@ -195,7 +200,11 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
@Override
public List<Cell> next() {
- return allResultsIterator.next().listCells();
+ List<Cell> result = allResultsIterator.next().listCells();
+ for (Cell cell : result) {
+ scanBytes += CellUtil.estimatedSizeOf(cell);
+ }
+ return result;
}
@Override
@@ -232,4 +241,4 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
}
};
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 3e0a065..1f6425f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -33,6 +33,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.client.Scan;
@@ -87,7 +88,19 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
private long serviceStartTime;
- static class InnerScannerAsIterator implements CellListIterator {
+ abstract static class BaseCellListIterator implements CellListIterator {
+ @Override
+ public final void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public final void close() throws IOException {
+ // no op. we close all region scanners at final block.
+ }
+ }
+
+ static class InnerScannerAsIterator extends BaseCellListIterator {
private RegionScanner regionScanner;
private List<Cell> nextOne = Lists.newArrayList();
private List<Cell> ret = Lists.newArrayList();
@@ -127,15 +140,58 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
return ret;
}
+ }
+
+ // TODO move this logic to HBaseReadonlyStore once it's been refactored
+ static class ResourceTrackingCellListIterator extends BaseCellListIterator {
+ private final Iterator<List<Cell>> delegate;
+ private final long rowCountLimit;
+ private final long bytesLimit;
+ private final long timeout;
+ private final long deadline;
+
+ private long rowCount;
+ private long rowBytes;
+
+ ResourceTrackingCellListIterator(Iterator<List<Cell>> delegate,
+ long rowCountLimit, long bytesLimit, long timeout) {
+ this.delegate = delegate;
+ this.rowCountLimit = rowCountLimit;
+ this.bytesLimit = bytesLimit;
+ this.timeout = timeout;
+ this.deadline = System.currentTimeMillis() + timeout;
+ }
@Override
- public void remove() {
- throw new UnsupportedOperationException();
+ public boolean hasNext() {
+ if (rowCount > rowCountLimit) {
+ throw new GTScanExceedThresholdException("Number of rows scanned exceeds threshold " + rowCountLimit);
+ }
+ if (rowBytes > bytesLimit) {
+ throw new GTScanExceedThresholdException("Scanned " + rowBytes + " bytes exceeds threshold " + bytesLimit);
+ }
+ if ((rowCount % GTScanRequest.terminateCheckInterval == 1) && System.currentTimeMillis() > deadline) {
+ throw new GTScanTimeoutException("Scan timeout after " + timeout + " ms");
+ }
+ return delegate.hasNext();
}
@Override
- public void close() throws IOException {
- //does not need to close as regionScanner will be closed in finally block
+ public List<Cell> next() {
+ List<Cell> result = delegate.next();
+ rowCount++;
+ for (Cell cell : result) {
+ rowBytes += CellUtil.estimatedSizeOf(cell);
+ }
+ return result;
+ }
+
+ public long getTotalScannedRowCount() {
+ return rowCount;
+ }
+
+ public long getTotalScannedRowBytes() {
+ return rowBytes;
}
}
@@ -237,51 +293,18 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
final MutableBoolean scanNormalComplete = new MutableBoolean(true);
- final long deadline = serviceStartTime + scanReq.getTimeout();
- logger.info("deadline(local) is " + deadline);
final long storagePushDownLimit = scanReq.getStoragePushDownLimit();
- final CellListIterator cellListIterator = new CellListIterator() {
-
- int counter = 0;
-
- @Override
- public void close() throws IOException {
- for (CellListIterator closeable : cellListsForeachRawScan) {
- closeable.close();
- }
- }
-
- @Override
- public boolean hasNext() {
-
- counter++;
-
- if (counter > scanReq.getStorageScanRowNumThreshold()) {
- throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter + ", consider increasing kylin.query.memory-budget-bytes and kylin.query.scan-threshold");
- }
-
- if (counter % (10 * GTScanRequest.terminateCheckInterval) == 1) {
- logger.info("scanning " + counter + "th row from HBase.");
- }
- return allCellLists.hasNext();
- }
-
- @Override
- public List<Cell> next() {
- return allCellLists.next();
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
+ ResourceTrackingCellListIterator cellListIterator = new ResourceTrackingCellListIterator(
+ allCellLists,
+ scanReq.getStorageScanRowNumThreshold(),
+ Long.MAX_VALUE,
+ scanReq.getTimeout());
IGTStore store = new HBaseReadonlyStore(cellListIterator, scanReq, hbaseRawScans.get(0).hbaseColumns, hbaseColumnsToGT, request.getRowkeyPreambleSize(), behavior.delayToggledOn());
IGTScanner rawScanner = store.scan(scanReq);
- IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, behavior.filterToggledOn(), behavior.aggrToggledOn(), false, deadline, request.getSpillEnabled());
+ IGTScanner finalScanner = scanReq.decorateScanner(rawScanner, behavior.filterToggledOn(), behavior.aggrToggledOn(), false, request.getSpillEnabled());
ByteBuffer buffer = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
@@ -290,13 +313,6 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try {
for (GTRecord oneRecord : finalScanner) {
-
- if (finalRowCount % GTScanRequest.terminateCheckInterval == 1) {
- if (System.currentTimeMillis() > deadline) {
- throw new GTScanTimeoutException("finalScanner timeouts after contributed " + finalRowCount);
- }
- }
-
buffer.clear();
try {
oneRecord.exportColumns(scanReq.getColumns(), buffer);
@@ -326,6 +342,8 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
appendProfileInfo(sb, "agg done");
+ logger.info("Total scanned {} rows and {} bytes",
+ cellListIterator.getTotalScannedRowCount(), cellListIterator.getTotalScannedRowBytes());
//outputStream.close() is not necessary
byte[] compressedAllRows;
@@ -341,6 +359,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
}
appendProfileInfo(sb, "compress done");
+ logger.info("Size of final result = {} ({} before compressing)", compressedAllRows.length, allRows.length);
OperatingSystemMXBean operatingSystemMXBean = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
double systemCpuLoad = operatingSystemMXBean.getSystemCpuLoad();
@@ -353,16 +372,17 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
CubeVisitProtos.CubeVisitResponse.Builder responseBuilder = CubeVisitProtos.CubeVisitResponse.newBuilder();
done.run(responseBuilder.//
setCompressedRows(HBaseZeroCopyByteString.wrap(compressedAllRows)).//too many array copies
- setStats(CubeVisitProtos.CubeVisitResponse.Stats.newBuilder().//
- setAggregatedRowCount(finalScanner.getScannedRowCount() - finalRowCount).//
- setScannedRowCount(finalScanner.getScannedRowCount()).//
- setServiceStartTime(serviceStartTime).//
- setServiceEndTime(System.currentTimeMillis()).//
- setSystemCpuLoad(systemCpuLoad).//
- setFreePhysicalMemorySize(freePhysicalMemorySize).//
- setFreeSwapSpaceSize(freeSwapSpaceSize).//
- setHostname(InetAddress.getLocalHost().getHostName()).//
- setEtcMsg(sb.toString()).//
+ setStats(CubeVisitProtos.CubeVisitResponse.Stats.newBuilder().
+ setAggregatedRowCount(cellListIterator.getTotalScannedRowCount() - finalRowCount).
+ setScannedRowCount(cellListIterator.getTotalScannedRowCount()).
+ setScannedBytes(cellListIterator.getTotalScannedRowBytes()).
+ setServiceStartTime(serviceStartTime).
+ setServiceEndTime(System.currentTimeMillis()).
+ setSystemCpuLoad(systemCpuLoad).
+ setFreePhysicalMemorySize(freePhysicalMemorySize).
+ setFreeSwapSpaceSize(freeSwapSpaceSize).
+ setHostname(InetAddress.getLocalHost().getHostName()).
+ setEtcMsg(sb.toString()).
setNormalComplete(scanNormalComplete.booleanValue() ? 1 : 0).build())
.//
build());
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
index def0182..5a3aa5a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/generated/CubeVisitProtos.java
@@ -2255,6 +2255,16 @@ public final class CubeVisitProtos {
* </pre>
*/
int getNormalComplete();
+
+ // optional int64 scannedBytes = 11;
+ /**
+ * <code>optional int64 scannedBytes = 11;</code>
+ */
+ boolean hasScannedBytes();
+ /**
+ * <code>optional int64 scannedBytes = 11;</code>
+ */
+ long getScannedBytes();
}
/**
* Protobuf type {@code CubeVisitResponse.Stats}
@@ -2357,6 +2367,11 @@ public final class CubeVisitProtos {
normalComplete_ = input.readInt32();
break;
}
+ case 88: {
+ bitField0_ |= 0x00000400;
+ scannedBytes_ = input.readInt64();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -2619,6 +2634,22 @@ public final class CubeVisitProtos {
return normalComplete_;
}
+ // optional int64 scannedBytes = 11;
+ public static final int SCANNEDBYTES_FIELD_NUMBER = 11;
+ private long scannedBytes_;
+ /**
+ * <code>optional int64 scannedBytes = 11;</code>
+ */
+ public boolean hasScannedBytes() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>optional int64 scannedBytes = 11;</code>
+ */
+ public long getScannedBytes() {
+ return scannedBytes_;
+ }
+
private void initFields() {
serviceStartTime_ = 0L;
serviceEndTime_ = 0L;
@@ -2630,6 +2661,7 @@ public final class CubeVisitProtos {
hostname_ = "";
etcMsg_ = "";
normalComplete_ = 0;
+ scannedBytes_ = 0L;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -2673,6 +2705,9 @@ public final class CubeVisitProtos {
if (((bitField0_ & 0x00000200) == 0x00000200)) {
output.writeInt32(10, normalComplete_);
}
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ output.writeInt64(11, scannedBytes_);
+ }
getUnknownFields().writeTo(output);
}
@@ -2722,6 +2757,10 @@ public final class CubeVisitProtos {
size += com.google.protobuf.CodedOutputStream
.computeInt32Size(10, normalComplete_);
}
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(11, scannedBytes_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -2792,6 +2831,11 @@ public final class CubeVisitProtos {
result = result && (getNormalComplete()
== other.getNormalComplete());
}
+ result = result && (hasScannedBytes() == other.hasScannedBytes());
+ if (hasScannedBytes()) {
+ result = result && (getScannedBytes()
+ == other.getScannedBytes());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -2848,6 +2892,10 @@ public final class CubeVisitProtos {
hash = (37 * hash) + NORMALCOMPLETE_FIELD_NUMBER;
hash = (53 * hash) + getNormalComplete();
}
+ if (hasScannedBytes()) {
+ hash = (37 * hash) + SCANNEDBYTES_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getScannedBytes());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -2977,6 +3025,8 @@ public final class CubeVisitProtos {
bitField0_ = (bitField0_ & ~0x00000100);
normalComplete_ = 0;
bitField0_ = (bitField0_ & ~0x00000200);
+ scannedBytes_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000400);
return this;
}
@@ -3045,6 +3095,10 @@ public final class CubeVisitProtos {
to_bitField0_ |= 0x00000200;
}
result.normalComplete_ = normalComplete_;
+ if (((from_bitField0_ & 0x00000400) == 0x00000400)) {
+ to_bitField0_ |= 0x00000400;
+ }
+ result.scannedBytes_ = scannedBytes_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -3095,6 +3149,9 @@ public final class CubeVisitProtos {
if (other.hasNormalComplete()) {
setNormalComplete(other.getNormalComplete());
}
+ if (other.hasScannedBytes()) {
+ setScannedBytes(other.getScannedBytes());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -3550,6 +3607,39 @@ public final class CubeVisitProtos {
return this;
}
+ // optional int64 scannedBytes = 11;
+ private long scannedBytes_ ;
+ /**
+ * <code>optional int64 scannedBytes = 11;</code>
+ */
+ public boolean hasScannedBytes() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>optional int64 scannedBytes = 11;</code>
+ */
+ public long getScannedBytes() {
+ return scannedBytes_;
+ }
+ /**
+ * <code>optional int64 scannedBytes = 11;</code>
+ */
+ public Builder setScannedBytes(long value) {
+ bitField0_ |= 0x00000400;
+ scannedBytes_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional int64 scannedBytes = 11;</code>
+ */
+ public Builder clearScannedBytes() {
+ bitField0_ = (bitField0_ & ~0x00000400);
+ scannedBytes_ = 0L;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:CubeVisitResponse.Stats)
}
@@ -4349,20 +4439,21 @@ public final class CubeVisitProtos {
"ze\030\003 \002(\005\0223\n\020hbaseColumnsToGT\030\004 \003(\0132\031.Cub" +
"eVisitRequest.IntList\022\027\n\017kylinProperties" +
"\030\005 \002(\t\022\017\n\007queryId\030\006 \001(\t\022\032\n\014spillEnabled\030" +
- "\007 \001(\010:\004true\032\027\n\007IntList\022\014\n\004ints\030\001 \003(\005\"\321\002\n" +
+ "\007 \001(\010:\004true\032\027\n\007IntList\022\014\n\004ints\030\001 \003(\005\"\347\002\n" +
"\021CubeVisitResponse\022\026\n\016compressedRows\030\001 \002",
"(\014\022\'\n\005stats\030\002 \002(\0132\030.CubeVisitResponse.St" +
- "ats\032\372\001\n\005Stats\022\030\n\020serviceStartTime\030\001 \001(\003\022" +
+ "ats\032\220\002\n\005Stats\022\030\n\020serviceStartTime\030\001 \001(\003\022" +
"\026\n\016serviceEndTime\030\002 \001(\003\022\027\n\017scannedRowCou" +
"nt\030\003 \001(\003\022\032\n\022aggregatedRowCount\030\004 \001(\003\022\025\n\r" +
"systemCpuLoad\030\005 \001(\001\022\036\n\026freePhysicalMemor" +
"ySize\030\006 \001(\001\022\031\n\021freeSwapSpaceSize\030\007 \001(\001\022\020" +
"\n\010hostname\030\010 \001(\t\022\016\n\006etcMsg\030\t \001(\t\022\026\n\016norm" +
- "alComplete\030\n \001(\0052F\n\020CubeVisitService\0222\n\t" +
- "visitCube\022\021.CubeVisitRequest\032\022.CubeVisit" +
- "ResponseB`\nEorg.apache.kylin.storage.hba",
- "se.cube.v2.coprocessor.endpoint.generate" +
- "dB\017CubeVisitProtosH\001\210\001\001\240\001\001"
+ "alComplete\030\n \001(\005\022\024\n\014scannedBytes\030\013 \001(\0032F" +
+ "\n\020CubeVisitService\0222\n\tvisitCube\022\021.CubeVi" +
+ "sitRequest\032\022.CubeVisitResponseB`\nEorg.ap",
+ "ache.kylin.storage.hbase.cube.v2.coproce" +
+ "ssor.endpoint.generatedB\017CubeVisitProtos" +
+ "H\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -4392,7 +4483,7 @@ public final class CubeVisitProtos {
internal_static_CubeVisitResponse_Stats_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_CubeVisitResponse_Stats_descriptor,
- new java.lang.String[] { "ServiceStartTime", "ServiceEndTime", "ScannedRowCount", "AggregatedRowCount", "SystemCpuLoad", "FreePhysicalMemorySize", "FreeSwapSpaceSize", "Hostname", "EtcMsg", "NormalComplete", });
+ new java.lang.String[] { "ServiceStartTime", "ServiceEndTime", "ScannedRowCount", "AggregatedRowCount", "SystemCpuLoad", "FreePhysicalMemorySize", "FreeSwapSpaceSize", "Hostname", "EtcMsg", "NormalComplete", "ScannedBytes", });
return null;
}
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/e09338b3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
index c7c2954..f416669 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/protobuf/CubeVisit.proto
@@ -54,6 +54,7 @@ message CubeVisitResponse {
optional string hostname = 8;
optional string etcMsg = 9;
optional int32 normalComplete =10;//when time outs, normalComplete will be false
+ optional int64 scannedBytes = 11;
}
required bytes compressedRows = 1;
required Stats stats = 2;
[07/17] kylin git commit: minor, use envelope for rest API reponse
Posted by bi...@apache.org.
minor, use envelope for rest API reponse
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7742b31d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7742b31d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7742b31d
Branch: refs/heads/KYLIN-2428
Commit: 7742b31d85e4256556c52d3bf0b485ef20151508
Parents: 722efb8
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Feb 9 10:47:21 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Feb 9 10:47:21 2017 +0800
----------------------------------------------------------------------
.../rest/controller/EncodingController.java | 11 ++--
.../kylin/rest/response/EnvelopeResponse.java | 58 ++++++++++++++++++++
2 files changed, 62 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7742b31d/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
index 2f532e2..b95394c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/EncodingController.java
@@ -23,6 +23,8 @@ import java.util.Map;
import java.util.Set;
import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.service.EncodingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -34,8 +36,6 @@ import org.springframework.web.bind.annotation.ResponseBody;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import org.apache.kylin.rest.service.EncodingService;
-
@Controller
@RequestMapping(value = "/encodings")
public class EncodingController extends BasicController {
@@ -52,7 +52,7 @@ public class EncodingController extends BasicController {
*/
@RequestMapping(value = "valid_encodings", method = { RequestMethod.GET })
@ResponseBody
- public Map<String, Object> getValidEncodings() {
+ public EnvelopeResponse getValidEncodings() {
Set<String> allDatatypes = Sets.newHashSet();
allDatatypes.addAll(DataType.DATETIME_FAMILY);
@@ -65,9 +65,6 @@ public class EncodingController extends BasicController {
datatypeValidEncodings.put(dataTypeStr, encodingService.getValidEncodings(DataType.getType(dataTypeStr)));
}
- Map<String, Object> ret = Maps.newHashMap();
- ret.put("code", "000");
- ret.put("data", datatypeValidEncodings);
- return ret;
+ return new EnvelopeResponse(EnvelopeResponse.CODE_SUCCESS, datatypeValidEncodings, "");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7742b31d/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
new file mode 100644
index 0000000..564db70
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.response;
+
+public class EnvelopeResponse {
+
+ public final static String CODE_SUCCESS = "000";
+
+ private String code;
+ private Object data;
+ private String msg;
+
+ public EnvelopeResponse(String code, Object data, String msg) {
+ this.code = code;
+ this.data = data;
+ this.msg = msg;
+ }
+
+ public String getCode() {
+ return code;
+ }
+
+ public void setCode(String code) {
+ this.code = code;
+ }
+
+ public Object getData() {
+ return data;
+ }
+
+ public void setData(Object data) {
+ this.data = data;
+ }
+
+ public String getMsg() {
+ return msg;
+ }
+
+ public void setMsg(String msg) {
+ this.msg = msg;
+ }
+}