You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/20 11:27:26 UTC
[01/50] [abbrv] kylin git commit: minor,
fix NumberDictionaryForestTest [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/master-cdh5.7 0b71be997 -> 980b8ceff (forced update)
minor, fix NumberDictionaryForestTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d6bfbaa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d6bfbaa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d6bfbaa
Branch: refs/heads/master-cdh5.7
Commit: 5d6bfbaa46dfd4b0ea899a7aa05bbc357732edc0
Parents: 6957fd7
Author: Li Yang <li...@apache.org>
Authored: Sun Dec 11 20:03:22 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Dec 11 20:04:08 2016 +0800
----------------------------------------------------------------------
.../mr/steps/NumberDictionaryForestTest.java | 35 +++++++-------------
.../mr/steps/SelfDefineSortableKeyTest.java | 8 ++---
2 files changed, 16 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d6bfbaa/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
index 12105c9..bb10b4a 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
@@ -14,7 +14,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Random;
-import java.util.UUID;
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.util.Bytes;
@@ -35,14 +34,13 @@ import org.junit.Test;
public class NumberDictionaryForestTest {
@Test
public void testNumberDictionaryForestLong() {
- List<String> list = randomLongData(10);
+ List<String> list = randomLongData(100);
testData(list, TypeFlag.INTEGER_FAMILY_TYPE);
}
@Test
public void testNumberDictionaryForestDouble() {
- List<String> list = randomDoubleData(10);
-
+ List<String> list = randomDoubleData(100);
testData(list, TypeFlag.DOUBLE_FAMILY_TYPE);
}
@@ -199,23 +197,15 @@ public class NumberDictionaryForestTest {
Random rand = new Random(System.currentTimeMillis());
ArrayList<String> list = new ArrayList<>();
for (int i = 0; i < count; i++) {
- list.add(rand.nextDouble() + "");
+ String str = rand.nextDouble() + "";
+ if (str.contains("E"))
+ continue;
+ list.add(str);
}
list.add("-1");
return list;
}
- private List<String> randomStringData(int count) {
- Random rand = new Random(System.currentTimeMillis());
- ArrayList<String> list = new ArrayList<>();
- for (int i = 0; i < count; i++) {
- list.add(UUID.randomUUID().toString());
- }
- list.add("123");
- list.add("123");
- return list;
- }
-
private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) {
int partationId = 0;
ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>();
@@ -237,16 +227,15 @@ public class NumberDictionaryForestTest {
}
private String printKey(SelfDefineSortableKey key) {
- byte[] data = key.getText().getBytes();
- byte[] fieldValue = Bytes.copy(data, 1, data.length - 1);
- System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + new String(fieldValue));
- return new String(fieldValue);
+ Text data = key.getText();
+ String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
+ System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue);
+ return fieldValue;
}
private String getFieldValue(SelfDefineSortableKey key) {
- byte[] data = key.getText().getBytes();
- byte[] fieldValue = Bytes.copy(data, 1, data.length - 1);
- return new String(fieldValue);
+ Text data = key.getText();
+ return Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
}
private <T> boolean isIncreasedOrder(List<T> list, Comparator<T> comp) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/5d6bfbaa/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
index 283a0f5..d71e366 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
@@ -182,10 +182,10 @@ public class SelfDefineSortableKeyTest {
}
private String printKey(SelfDefineSortableKey key) {
- byte[] data = key.getText().getBytes();
- byte[] fieldValue = Bytes.copy(data, 1, data.length - 1);
- System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + new String(fieldValue));
- return new String(fieldValue);
+ Text data = key.getText();
+ String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
+ System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue);
+ return fieldValue;
}
private <T> boolean isIncreasedOrder(List<T> list, Comparator<T> comp) {
[03/50] [abbrv] kylin git commit: KYLIN-2271 Purge cube should check
whether there is building segment
Posted by li...@apache.org.
KYLIN-2271 Purge cube should check whether there is building segment
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8265c3b0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8265c3b0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8265c3b0
Branch: refs/heads/master-cdh5.7
Commit: 8265c3b0e567e0c9ffd9d4bcb04d7ef2bbfecdbd
Parents: 65c5899
Author: shaofengshi <sh...@apache.org>
Authored: Mon Dec 12 11:31:06 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 12 11:31:06 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 5 -----
.../org/apache/kylin/rest/controller/CubeController.java | 9 +++++++--
2 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8265c3b0/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d4272f9..2b35c70 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -281,11 +281,6 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.cube.ignore-signature-inconsistency", "false"));
}
- @Deprecated
- public int getCubeAggrGroupMaxSize() {
- return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-size", "12"));
- }
-
public int getCubeAggrGroupMaxCombination() {
return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max-combination", "4096"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8265c3b0/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 416fc88..4cf38e9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -289,7 +289,7 @@ public class CubeController extends BasicController {
CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
if (cube == null) {
- throw new IllegalArgumentException("Cube with name '" + cubeName + "' was not found.");
+ throw new InternalErrorException("Cannot find cube " + cubeName);
}
return jobService.submitJob(cube, startTime, endTime, startOffset, endOffset, //
sourcePartitionOffsetStart, sourcePartitionOffsetEnd, CubeBuildTypeEnum.valueOf(buildType), force, submitter);
@@ -324,7 +324,12 @@ public class CubeController extends BasicController {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
if (cube == null) {
- throw new InternalErrorException("Cannot find cube " + cubeName);
+ throw new IllegalArgumentException("Cannot find cube '" + cubeName + "'");
+ }
+
+ if (cube.getSegments() != null && cube.getBuildingSegments().size() > 0) {
+ int num = cube.getBuildingSegments().size();
+ throw new IllegalStateException("Cannot purge cube '" + cubeName + "' as there is " + num + " building " + (num > 1 ? "segment(s)." : "segment."));
}
return cubeService.purgeCube(cube);
[39/50] [abbrv] kylin git commit: KYLIN-2298,
timer component get wrong seconds
Posted by li...@apache.org.
KYLIN-2298,timer component get wrong seconds
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/699a88eb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/699a88eb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/699a88eb
Branch: refs/heads/master-cdh5.7
Commit: 699a88eba6833449defa166cc3397a96ff765961
Parents: 5fd3c56
Author: luguosheng <55...@qq.com>
Authored: Mon Dec 19 13:55:30 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Mon Dec 19 13:57:22 2016 +0800
----------------------------------------------------------------------
webapp/app/js/directives/directives.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/699a88eb/webapp/app/js/directives/directives.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/directives/directives.js b/webapp/app/js/directives/directives.js
index bf1690c..3d18451 100644
--- a/webapp/app/js/directives/directives.js
+++ b/webapp/app/js/directives/directives.js
@@ -271,7 +271,7 @@ KylinApp.directive('kylinPagination', function ($parse, $q) {
var hour = newDate.getHours()<10?'0'+newDate.getHours():newDate.getHours();
var mins = newDate.getMinutes()<10?'0'+newDate.getMinutes():newDate.getMinutes();
- var seconds = newDate.getSeconds()<10?'0'+newDate.getSeconds():getSeconds();
+ var seconds = newDate.getSeconds()<10?'0'+newDate.getSeconds():newDate.getSeconds();
var viewVal = year+"-"+month+"-"+date+" "+hour+":"+mins+":"+seconds;
return viewVal;
[07/50] [abbrv] kylin git commit: add check on kafka topic and broker
info
Posted by li...@apache.org.
add check on kafka topic and broker info
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d11d019c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d11d019c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d11d019c
Branch: refs/heads/master-cdh5.7
Commit: d11d019c98e11bf058381752d605f842a8eb30b1
Parents: a4ddbbd
Author: shaofengshi <sh...@apache.org>
Authored: Mon Dec 12 16:22:15 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 12 16:22:15 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/kafka/KafkaConfigManager.java | 8 ++++++++
.../java/org/apache/kylin/source/kafka/util/KafkaClient.java | 5 +++++
2 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d11d019c/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
index e76422c..775f052 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java
@@ -191,6 +191,14 @@ public class KafkaConfigManager {
throw new IllegalArgumentException();
}
+ if (StringUtils.isEmpty(kafkaConfig.getTopic())) {
+ throw new IllegalArgumentException("No topic info");
+ }
+
+ if (kafkaConfig.getKafkaClusterConfigs() == null || kafkaConfig.getKafkaClusterConfigs().size() ==0) {
+ throw new IllegalArgumentException("No cluster info");
+ }
+
String path = KafkaConfig.concatResourcePath(kafkaConfig.getName());
getStore().putResource(path, kafkaConfig, KafkaConfig.SERIALIZER);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d11d019c/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
index 2a7b0e8..3b970b3 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/util/KafkaClient.java
@@ -18,6 +18,7 @@
package org.apache.kylin.source.kafka.util;
import com.google.common.collect.Maps;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.common.PartitionInfo;
@@ -96,6 +97,10 @@ public class KafkaClient {
}
}
}
+
+ if (StringUtils.isEmpty(brokers)) {
+ throw new IllegalArgumentException("No cluster info in Kafka config '" + kafkaConfig.getName() + "'");
+ }
return brokers;
}
[38/50] [abbrv] kylin git commit: KYLIN-2297,
Manually edit cube segment start/end time will throw error in UI
Posted by li...@apache.org.
KYLIN-2297, Manually edit cube segment start/end time will throw error in UI
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5fd3c564
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5fd3c564
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5fd3c564
Branch: refs/heads/master-cdh5.7
Commit: 5fd3c56408050b1bb2a939bbb7b2105b4806aef8
Parents: 7446c56
Author: luguosheng <55...@qq.com>
Authored: Mon Dec 19 11:33:51 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Mon Dec 19 13:54:36 2016 +0800
----------------------------------------------------------------------
webapp/app/js/directives/directives.js | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5fd3c564/webapp/app/js/directives/directives.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/directives/directives.js b/webapp/app/js/directives/directives.js
index f5051e8..bf1690c 100644
--- a/webapp/app/js/directives/directives.js
+++ b/webapp/app/js/directives/directives.js
@@ -278,10 +278,11 @@ KylinApp.directive('kylinPagination', function ($parse, $q) {
});
ctrl.$parsers.push(function (value) {
- if (isNaN(value)||value==null) {
+ if(/\d{4}-\d{1,2}-\d{1,2}\s+(\d{1,2}:\d{1,2}:\d{1,2})/.test(value)) {
+ value=new Date(value);
+ }else{
return value;
}
- //value = new Date(value.getFullYear(), value.getMonth(), value.getDate(), 0, 0, 0, 0);
return value.getTime()-(60000 * value.getTimezoneOffset());
});
}
[28/50] [abbrv] kylin git commit: KYLIN-2180: refactor ProjectRequest
Posted by li...@apache.org.
KYLIN-2180: refactor ProjectRequest
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/911bdd00
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/911bdd00
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/911bdd00
Branch: refs/heads/master-cdh5.7
Commit: 911bdd00afb9176af594fe97cbc0a639f376da2d
Parents: 3003d9f
Author: kangkaisen <ka...@live.com>
Authored: Sun Dec 11 17:17:40 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 16 11:18:15 2016 +0800
----------------------------------------------------------------------
.../rest/controller/ProjectController.java | 48 ++++++++++----
.../rest/request/CreateProjectRequest.java | 57 ----------------
.../kylin/rest/request/ProjectRequest.java | 47 +++++++++++++
.../rest/request/UpdateProjectRequest.java | 66 -------------------
.../kylin/rest/service/ProjectService.java | 23 +++----
.../rest/controller/ProjectControllerTest.java | 69 ++++++++++++--------
webapp/app/js/controllers/page.js | 27 +++-----
7 files changed, 143 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/911bdd00/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
index 05af82c..a6edece 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
@@ -24,13 +24,13 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.persistence.AclEntity;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.request.CreateProjectRequest;
-import org.apache.kylin.rest.request.UpdateProjectRequest;
+import org.apache.kylin.rest.request.ProjectRequest;
import org.apache.kylin.rest.service.AccessService;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.ProjectService;
@@ -198,19 +198,22 @@ public class ProjectController extends BasicController {
@RequestMapping(value = "", method = { RequestMethod.POST })
@ResponseBody
- public ProjectInstance saveProject(@RequestBody CreateProjectRequest projectRequest) {
- if (StringUtils.isEmpty(projectRequest.getName())) {
- logger.info("Project name should not be empty.");
- throw new BadRequestException("Project name should not be empty.");
+
+ public ProjectInstance saveProject(@RequestBody ProjectRequest projectRequest) {
+ ProjectInstance projectDesc = deserializeProjectDesc(projectRequest);
+
+ if (StringUtils.isEmpty(projectDesc.getName())) {
+ throw new InternalErrorException("A project name must be given to create a project");
}
- if (!StringUtils.containsOnly(projectRequest.getName(), VALID_PROJECTNAME)) {
- logger.info("Invalid Project name {}, only letters, numbers and underline supported.", projectRequest.getName());
+
+ if (!StringUtils.containsOnly(projectDesc.getName(), VALID_PROJECTNAME)) {
+ logger.info("Invalid Project name {}, only letters, numbers and underline supported.", projectDesc.getName());
throw new BadRequestException("Invalid Project name, only letters, numbers and underline supported.");
}
ProjectInstance createdProj = null;
try {
- createdProj = projectService.createProject(projectRequest);
+ createdProj = projectService.createProject(projectDesc);
} catch (Exception e) {
logger.error("Failed to deal with the request.", e);
throw new InternalErrorException(e.getLocalizedMessage());
@@ -221,15 +224,22 @@ public class ProjectController extends BasicController {
@RequestMapping(value = "", method = { RequestMethod.PUT })
@ResponseBody
- public ProjectInstance updateProject(@RequestBody UpdateProjectRequest projectRequest) {
- if (StringUtils.isEmpty(projectRequest.getFormerProjectName())) {
+ public ProjectInstance updateProject(@RequestBody ProjectRequest projectRequest) {
+ String formerProjectName = projectRequest.getFormerProjectName();
+ if (StringUtils.isEmpty(formerProjectName)) {
throw new InternalErrorException("A project name must be given to update a project");
}
+ ProjectInstance projectDesc = deserializeProjectDesc(projectRequest);
+
ProjectInstance updatedProj = null;
try {
- ProjectInstance currentProject = projectService.getProjectManager().getProject(projectRequest.getFormerProjectName());
- updatedProj = projectService.updateProject(projectRequest, currentProject);
+ ProjectInstance currentProject = projectService.getProjectManager().getProject(formerProjectName);
+ if (currentProject == null) {
+ throw new InternalErrorException("The project named " + formerProjectName + " does not exists");
+ }
+
+ updatedProj = projectService.updateProject(projectDesc, currentProject);
} catch (Exception e) {
logger.error("Failed to deal with the request.", e);
throw new InternalErrorException(e.getLocalizedMessage());
@@ -238,6 +248,18 @@ public class ProjectController extends BasicController {
return updatedProj;
}
+ private ProjectInstance deserializeProjectDesc(ProjectRequest projectRequest) {
+ ProjectInstance projectDesc = null;
+ try {
+ logger.debug("Saving project " + projectRequest.getProjectDescData());
+ projectDesc = JsonUtil.readValue(projectRequest.getProjectDescData(), ProjectInstance.class);
+ } catch (Exception e) {
+ logger.error("Failed to deal with the request.", e);
+ throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+ }
+ return projectDesc;
+ }
+
@RequestMapping(value = "/{projectName}", method = { RequestMethod.DELETE })
@ResponseBody
public void deleteProject(@PathVariable String projectName) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/911bdd00/server-base/src/main/java/org/apache/kylin/rest/request/CreateProjectRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/CreateProjectRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/CreateProjectRequest.java
deleted file mode 100644
index 00fe1eb..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/request/CreateProjectRequest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.request;
-
-import java.util.LinkedHashMap;
-
-/**
- */
-public class CreateProjectRequest {
- private String name;
- private String description;
- private LinkedHashMap<String, String> overrideKylinProps;
-
- public CreateProjectRequest() {
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
- public LinkedHashMap<String, String> getOverrideKylinProps() {
- return overrideKylinProps;
- }
-
- public void setOverrideKylinProps(LinkedHashMap<String, String> overrideKylinProps) {
- this.overrideKylinProps = overrideKylinProps;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/911bdd00/server-base/src/main/java/org/apache/kylin/rest/request/ProjectRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/ProjectRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/ProjectRequest.java
new file mode 100644
index 0000000..da6e190
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/ProjectRequest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.request;
+
+/**
+ */
+public class ProjectRequest {
+ private String formerProjectName;
+
+ private String projectDescData;
+
+ public ProjectRequest() {
+ }
+
+ public String getProjectDescData() {
+ return projectDescData;
+ }
+
+ public void setProjectDescData(String projectDescData) {
+ this.projectDescData = projectDescData;
+ }
+
+ public String getFormerProjectName() {
+ return formerProjectName;
+ }
+
+ public void setFormerProjectName(String formerProjectName) {
+ this.formerProjectName = formerProjectName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/911bdd00/server-base/src/main/java/org/apache/kylin/rest/request/UpdateProjectRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/UpdateProjectRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/UpdateProjectRequest.java
deleted file mode 100644
index f253c9c..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/request/UpdateProjectRequest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.request;
-
-import java.util.LinkedHashMap;
-
-/**
- */
-public class UpdateProjectRequest {
- private String formerProjectName;
- private String newProjectName;
- private String newDescription;
- private LinkedHashMap<String, String> overrideKylinProps;
-
- public UpdateProjectRequest() {
- }
-
- public String getFormerProjectName() {
- return formerProjectName;
- }
-
- public void setFormerProjectName(String formerProjectName) {
-
- this.formerProjectName = formerProjectName;
- }
-
- public String getNewDescription() {
- return newDescription;
- }
-
- public void setNewDescription(String newDescription) {
- this.newDescription = newDescription;
- }
-
- public String getNewProjectName() {
- return newProjectName;
- }
-
- public void setNewProjectName(String newProjectName) {
- this.newProjectName = newProjectName;
- }
-
- public LinkedHashMap<String, String> getOverrideKylinProps() {
- return overrideKylinProps;
- }
-
- public void setOverrideKylinProps(LinkedHashMap<String, String> overrideKylinProps) {
- this.overrideKylinProps = overrideKylinProps;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/911bdd00/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 283cf4a..e6f546c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -27,8 +27,6 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.request.CreateProjectRequest;
-import org.apache.kylin.rest.request.UpdateProjectRequest;
import org.apache.kylin.rest.security.AclPermission;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,10 +49,10 @@ public class ProjectService extends BasicService {
private AccessService accessService;
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
- public ProjectInstance createProject(CreateProjectRequest projectRequest) throws IOException {
- String projectName = projectRequest.getName();
- String description = projectRequest.getDescription();
- LinkedHashMap<String, String> overrideProps = projectRequest.getOverrideKylinProps();
+ public ProjectInstance createProject(ProjectInstance newProject) throws IOException {
+ String projectName = newProject.getName();
+ String description = newProject.getDescription();
+ LinkedHashMap<String, String> overrideProps = newProject.getOverrideKylinProps();
ProjectInstance currentProject = getProjectManager().getProject(projectName);
@@ -70,15 +68,10 @@ public class ProjectService extends BasicService {
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#currentProject, 'ADMINISTRATION') or hasPermission(#currentProject, 'MANAGEMENT')")
- public ProjectInstance updateProject(UpdateProjectRequest projectRequest, ProjectInstance currentProject) throws IOException {
- String formerProjectName = projectRequest.getFormerProjectName();
- String newProjectName = projectRequest.getNewProjectName();
- String newDescription = projectRequest.getNewDescription();
- LinkedHashMap<String, String> overrideProps = projectRequest.getOverrideKylinProps();
-
- if (currentProject == null) {
- throw new InternalErrorException("The project named " + formerProjectName + " does not exists");
- }
+ public ProjectInstance updateProject(ProjectInstance newProject, ProjectInstance currentProject) throws IOException {
+ String newProjectName = newProject.getName();
+ String newDescription = newProject.getDescription();
+ LinkedHashMap<String, String> overrideProps = newProject.getOverrideKylinProps();
ProjectInstance updatedProject = getProjectManager().updateProject(currentProject, newProjectName, newDescription, overrideProps);
http://git-wip-us.apache.org/repos/asf/kylin/blob/911bdd00/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java
index 9a6dfbe..cd9a524 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/ProjectControllerTest.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,13 +19,12 @@
package org.apache.kylin.rest.controller;
import java.io.IOException;
-import java.util.List;
+import java.io.StringWriter;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.request.CreateProjectRequest;
-import org.apache.kylin.rest.request.UpdateProjectRequest;
+import org.apache.kylin.rest.request.ProjectRequest;
import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.service.ServiceTestBase;
import org.junit.Assert;
@@ -33,6 +32,8 @@ import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
/**
*/
public class ProjectControllerTest extends ServiceTestBase {
@@ -48,6 +49,7 @@ public class ProjectControllerTest extends ServiceTestBase {
projectController = new ProjectController();
projectController.setProjectService(projectService);
+
try {
projectController.deleteProject("new_project");
} catch (InternalErrorException e) {
@@ -58,38 +60,34 @@ public class ProjectControllerTest extends ServiceTestBase {
} catch (InternalErrorException e) {
//project doesn't exist
}
-
}
@Test
public void testAddUpdateProject() throws IOException {
+ int originalProjectCount = projectController.getProjects(null, null).size();
- List<ProjectInstance> projects = projectController.getProjects(null, null);
-
- int originalProjectCount = projects.size();
- CreateProjectRequest request = new CreateProjectRequest();
- request.setName("new_project");
- ProjectInstance ret = projectController.saveProject(request);
+ //test add project
+ ProjectInstance project = new ProjectInstance();
+ project.setName("new_project");
+ ProjectInstance ret = projectController.saveProject(getProjectRequest(project, null));
Assert.assertEquals(ret.getOwner(), "ADMIN");
Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
- UpdateProjectRequest updateR = new UpdateProjectRequest();
- updateR.setFormerProjectName("new_project");
- updateR.setNewProjectName("new_project_2");
- projectController.updateProject(updateR);
+ //test update project
+ ProjectInstance newProject = new ProjectInstance();
+ newProject.setName("new_project_2");
+ projectController.updateProject(getProjectRequest(newProject, "new_project"));
Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null);
-
Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project_2"), null);
- // only update desc:
- updateR = new UpdateProjectRequest();
- updateR.setFormerProjectName("new_project_2");
- updateR.setNewProjectName("new_project_2");
- updateR.setNewDescription("hello world");
- projectController.updateProject(updateR);
+ //test update project description only
+ ProjectInstance newProject2 = new ProjectInstance();
+ newProject2.setName("new_project_2");
+ newProject2.setDescription("hello world");
+ projectController.updateProject(getProjectRequest(newProject2, "new_project_2"));
Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null);
@@ -99,8 +97,27 @@ public class ProjectControllerTest extends ServiceTestBase {
@Test(expected = InternalErrorException.class)
public void testAddExistingProject() throws IOException {
- CreateProjectRequest request = new CreateProjectRequest();
- request.setName("default");
- projectController.saveProject(request);
+ ProjectInstance newProject = new ProjectInstance();
+ newProject.setName("default");
+
+ projectController.saveProject(getProjectRequest(newProject, null));
+ }
+
+ private ProjectRequest getProjectRequest(ProjectInstance project, String formerProjectName) throws IOException {
+ ProjectRequest request = new ProjectRequest();
+ request.setProjectDescData(getProjectDescData(project));
+ request.setFormerProjectName(formerProjectName);
+
+ return request;
+ }
+
+ private String getProjectDescData(ProjectInstance project) throws IOException {
+ ObjectMapper projectMapper = new ObjectMapper();
+ StringWriter projectWriter = new StringWriter();
+ projectMapper.writeValue(projectWriter, project);
+
+ System.err.println(projectWriter.toString());
+
+ return projectWriter.toString();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/911bdd00/webapp/app/js/controllers/page.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/page.js b/webapp/app/js/controllers/page.js
index 2b6bc64..056cc48 100644
--- a/webapp/app/js/controllers/page.js
+++ b/webapp/app/js/controllers/page.js
@@ -195,7 +195,7 @@ var projCtrl = function ($scope, $location, $modalInstance, ProjectService, Mess
projectIdx: -1
};
$scope.isEdit = false;
- $scope.proj = {name: '', description: '', overrideKylinProps: {}};
+ $scope.proj = {name: '', description: '', override_kylin_properties: {}};
$scope.convertedProperties = [];
if (project) {
@@ -203,10 +203,10 @@ var projCtrl = function ($scope, $location, $modalInstance, ProjectService, Mess
$scope.state.oldProjName = project.name;
$scope.proj = project;
- for (var key in $scope.proj.overrideKylinProps) {
+ for (var key in $scope.proj.override_kylin_properties) {
$scope.convertedProperties.push({
name: key,
- value: $scope.proj.overrideKylinProps[key]
+ value: $scope.proj.override_kylin_properties[key]
});
}
@@ -220,14 +220,7 @@ var projCtrl = function ($scope, $location, $modalInstance, ProjectService, Mess
$scope.createOrUpdate = function () {
if ($scope.state.isEdit) {
-
- var requestBody = {
- formerProjectName: $scope.state.oldProjName,
- newProjectName: $scope.proj.name,
- newDescription: $scope.proj.description,
- overrideKylinProps: $scope.proj.overrideKylinProps
- };
- ProjectService.update({}, requestBody, function (newProj) {
+ ProjectService.update({}, {formerProjectName: $scope.state.oldProjName, projectDescData: angular.toJson($scope.proj)}, function (newProj) {
SweetAlert.swal('Success!', 'Project update successfully!', 'success');
//update project in project model
@@ -246,7 +239,7 @@ var projCtrl = function ($scope, $location, $modalInstance, ProjectService, Mess
});
}
else {
- ProjectService.save({}, $scope.proj, function (newProj) {
+ ProjectService.save({}, {projectDescData: angular.toJson($scope.proj)}, function (newProj) {
SweetAlert.swal('Success!', 'New project created successfully!', 'success');
$modalInstance.dismiss('cancel');
$cookieStore.put("project", newProj.name);
@@ -271,10 +264,10 @@ var projCtrl = function ($scope, $location, $modalInstance, ProjectService, Mess
};
$scope.addNewProperty = function () {
- if ($scope.proj.overrideKylinProps.hasOwnProperty('')) {
+ if ($scope.proj.override_kylin_properties.hasOwnProperty('')) {
return;
}
- $scope.proj.overrideKylinProps[''] = '';
+ $scope.proj.override_kylin_properties[''] = '';
$scope.convertedProperties.push({
name: '',
value: ''
@@ -282,9 +275,9 @@ var projCtrl = function ($scope, $location, $modalInstance, ProjectService, Mess
};
$scope.refreshPropertiesObj = function () {
- $scope.proj.overrideKylinProps = {};
+ $scope.proj.override_kylin_properties = {};
angular.forEach($scope.convertedProperties, function (item, index) {
- $scope.proj.overrideKylinProps[item.name] = item.value;
+ $scope.proj.override_kylin_properties[item.name] = item.value;
})
};
@@ -299,7 +292,7 @@ var projCtrl = function ($scope, $location, $modalInstance, ProjectService, Mess
if (index > -1) {
arr.splice(index, 1);
}
- delete $scope.proj.overrideKylinProps[item.name];
+ delete $scope.proj.override_kylin_properties[item.name];
}
};
[47/50] [abbrv] kylin git commit: KYLIN-2287 minor, UI enhancement
Posted by li...@apache.org.
KYLIN-2287 minor, UI enhancement
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c72e2320
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c72e2320
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c72e2320
Branch: refs/heads/master-cdh5.7
Commit: c72e2320238edeb2e7279dc4dc152493837e8843
Parents: 64a0a59
Author: zhongjian <ji...@163.com>
Authored: Tue Dec 20 15:43:12 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Tue Dec 20 15:43:12 2016 +0800
----------------------------------------------------------------------
webapp/app/less/app.less | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c72e2320/webapp/app/less/app.less
----------------------------------------------------------------------
diff --git a/webapp/app/less/app.less b/webapp/app/less/app.less
index 5ef5297..85ad937 100644
--- a/webapp/app/less/app.less
+++ b/webapp/app/less/app.less
@@ -72,6 +72,14 @@ body {
min-width: 110px
}
+.cube_model_trees .dropdown-menu{
+ min-width: 60px;
+}
+
+.cube_model_trees a{
+ margin-right:0px !important;
+}
+
.login {
position: fixed;
width: 325px;
[45/50] [abbrv] kylin git commit: KYLIN-2269 Reduce MR memory usage
for global dict
Posted by li...@apache.org.
KYLIN-2269 Reduce MR memory usage for global dict
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f50c0c87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f50c0c87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f50c0c87
Branch: refs/heads/master-cdh5.7
Commit: f50c0c87373e4abacd7106d527df4b885f0a88ea
Parents: 878107e
Author: kangkaisen <ka...@live.com>
Authored: Tue Dec 6 19:26:09 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Dec 19 17:52:41 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 ++++
.../org/apache/kylin/cube/model/CubeDesc.java | 22 ++++++++++++++++++++
.../cube/model/CubeJoinedFlatTableDesc.java | 15 ++++++++-----
.../cube/model/CubeJoinedFlatTableEnrich.java | 7 ++++++-
.../org/apache/kylin/job/JoinedFlatTable.java | 13 ++++++++++--
.../metadata/model/IJoinedFlatTableDesc.java | 11 ++++++----
6 files changed, 60 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5153562..01d1d36 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -492,6 +492,10 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.source.hive.beeline-params", "");
}
+ public String getFlatHiveTableClusterByDictColumn() {
+ return getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
+ }
+
@Deprecated
public String getCreateFlatHiveTableMethod() {
return getOptional("kylin.source.hive.create-flat-table-method", "1");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index f6b68af..3b8d034 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -1090,6 +1090,28 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return null;
}
+
+ /** Get a column which can be used to cluster the source table.
+ * To reduce memory footprint in base cuboid for global dict */
+ // TODO handle more than one ultra high cardinality columns use global dict in one cube
+ TblColRef getClusteredByColumn() {
+ if (getDistributedByColumn() != null) {
+ return null;
+ }
+
+ if (dictionaries == null) {
+ return null;
+ }
+
+ String clusterByColumn = config.getFlatHiveTableClusterByDictColumn();
+ for (DictionaryDesc dictDesc : dictionaries) {
+ if (dictDesc.getColumnRef().getName().equalsIgnoreCase(clusterByColumn)) {
+ return dictDesc.getColumnRef();
+ }
+ }
+
+ return null;
+ }
public String getDictionaryBuilderClass(TblColRef col) {
if (dictionaries == null)
http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index f37f86e..94e1a7c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -49,11 +49,11 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
public CubeJoinedFlatTableDesc(CubeDesc cubeDesc) {
this(cubeDesc, null);
}
-
+
public CubeJoinedFlatTableDesc(CubeSegment cubeSegment) {
this(cubeSegment.getCubeDesc(), cubeSegment);
}
-
+
private CubeJoinedFlatTableDesc(CubeDesc cubeDesc, CubeSegment cubeSegment /* can be null */) {
this.cubeDesc = cubeDesc;
this.cubeSegment = cubeSegment;
@@ -68,7 +68,7 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
return "kylin_intermediate_" + cubeDesc.getName() + "_" + cubeSegment.getUuid().replaceAll("-", "_");
}
}
-
+
protected final void initAddColumn(TblColRef col) {
if (columnIndexMap.containsKey(col))
return;
@@ -77,10 +77,10 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
columnIndexMap.put(col, columnIndex);
columnList.add(col);
columnCount = columnIndexMap.size();
-
+
Preconditions.checkState(columnIndexMap.size() == columnList.size());
}
-
+
// check what columns from hive tables are required, and index them
protected void initParseCubeDesc() {
for (TblColRef col : cubeDesc.listDimensionColumnsExcludingDerived(false)) {
@@ -165,4 +165,9 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc {
return cubeSegment;
}
+ @Override
+ public TblColRef getClusterBy() {
+ return cubeDesc.getClusteredByColumn();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
index 979af76..a1312b5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableEnrich.java
@@ -42,7 +42,7 @@ public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc {
// != works due to object cache
if (cubeDesc.getModel() != flatDesc.getDataModel())
throw new IllegalArgumentException();
-
+
this.cubeDesc = cubeDesc;
this.flatDesc = flatDesc;
parseCubeDesc();
@@ -132,4 +132,9 @@ public class CubeJoinedFlatTableEnrich implements IJoinedFlatTableDesc {
return flatDesc.getSegment();
}
+ @Override
+ public TblColRef getClusterBy() {
+ return flatDesc.getClusterBy();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 9fa0961..9ed563f 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -171,6 +171,10 @@ public class JoinedFlatTable {
}
}
+ private static void appendClusterStatement(StringBuilder sql, TblColRef clusterCol) {
+ sql.append(" CLUSTER BY ").append(colName(clusterCol)).append(";\n");
+ }
+
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql) {
boolean hasCondition = false;
StringBuilder whereBuilder = new StringBuilder();
@@ -219,8 +223,13 @@ public class JoinedFlatTable {
StringBuilder sql = new StringBuilder();
sql.append("INSERT OVERWRITE TABLE " + tableName + " SELECT * FROM " + tableName);
- TblColRef distDcol = flatDesc.getDistributedBy();
- appendDistributeStatement(sql, distDcol);
+ TblColRef clusterCol = flatDesc.getClusterBy();
+ if (clusterCol != null) {
+ appendClusterStatement(sql, clusterCol);
+ } else {
+ appendDistributeStatement(sql, flatDesc.getDistributedBy());
+ }
+
return sql.toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f50c0c87/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
index ffa2680..b545e50 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/IJoinedFlatTableDesc.java
@@ -27,16 +27,19 @@ public interface IJoinedFlatTableDesc {
String getTableName();
DataModelDesc getDataModel();
-
+
List<TblColRef> getAllColumns();
-
+
int getColumnIndex(TblColRef colRef);
long getSourceOffsetStart();
-
+
long getSourceOffsetEnd();
-
+
TblColRef getDistributedBy();
+ TblColRef getClusterBy();
+
ISegment getSegment();
+
}
[15/50] [abbrv] kylin git commit: KYLIN-1832 code review
Posted by li...@apache.org.
KYLIN-1832 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6e330a8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6e330a8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6e330a8
Branch: refs/heads/master-cdh5.7
Commit: e6e330a8bd47f1d2dd5fd6f68b510c3cf0be0287
Parents: f05404d
Author: Li Yang <li...@apache.org>
Authored: Wed Dec 14 15:29:56 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Dec 14 15:29:56 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/util/CubingUtils.java | 12 +-
.../apache/kylin/gridtable/UnitTestSupport.java | 22 +-
.../benchmark/GTScannerBenchmark2.java | 4 +-
.../gridtable/AggregationCacheMemSizeTest.java | 4 +-
.../metadata/measure/MeasureCodecTest.java | 4 +-
.../org/apache/kylin/measure/MeasureType.java | 2 +-
.../kylin/measure/MeasureTypeFactory.java | 2 +-
.../kylin/measure/hllc/DenseRegister.java | 26 +-
.../kylin/measure/hllc/HLLCAggregator.java | 10 +-
.../kylin/measure/hllc/HLLCMeasureType.java | 20 +-
.../kylin/measure/hllc/HLLCSerializer.java | 16 +-
.../apache/kylin/measure/hllc/HLLCounter.java | 377 ++++++++++++++++++
.../kylin/measure/hllc/HLLCounterOld.java | 393 +++++++++++++++++++
.../measure/hllc/HLLDistinctCountAggFunc.java | 22 +-
.../measure/hllc/HyperLogLogPlusCounterNew.java | 388 ------------------
.../measure/hllc/HyperLogLogPlusCounterOld.java | 392 ------------------
.../org/apache/kylin/measure/hllc/Register.java | 4 +-
.../kylin/measure/hllc/SparseRegister.java | 38 +-
.../measure/AggregatorMemEstimateTest.java | 4 +-
.../measure/hll/HyperLogLogCounterOldTest.java | 265 -------------
.../measure/hll2/HyperLogLogCounterNewTest.java | 301 --------------
.../hll2/NewHyperLogLogBenchmarkTest.java | 288 --------------
.../kylin/measure/hllc/HLLCounterOldTest.java | 266 +++++++++++++
.../kylin/measure/hllc/HLLCounterTest.java | 316 +++++++++++++++
.../hllc/NewHyperLogLogBenchmarkTest.java | 291 ++++++++++++++
.../kylin/engine/mr/common/CubeStatsReader.java | 12 +-
.../kylin/engine/mr/common/CubeStatsWriter.java | 6 +-
.../mr/steps/FactDistinctColumnsReducer.java | 8 +-
.../mr/steps/FactDistinctHiveColumnsMapper.java | 10 +-
.../engine/mr/steps/MergeStatisticsStep.java | 6 +-
.../kylin/engine/mr/steps/CubeSamplingTest.java | 8 +-
.../steps/FactDistinctColumnsReducerTest.java | 4 +-
.../apache/kylin/engine/spark/SparkCubing.java | 28 +-
.../cardinality/ColumnCardinalityMapper.java | 10 +-
.../cardinality/ColumnCardinalityReducer.java | 12 +-
.../ColumnCardinalityReducerTest.java | 4 +-
36 files changed, 1802 insertions(+), 1773 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 35139a4..5e63f94 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -38,7 +38,7 @@ import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
@@ -59,7 +59,7 @@ public class CubingUtils {
private static Logger logger = LoggerFactory.getLogger(CubingUtils.class);
- public static Map<Long, HyperLogLogPlusCounterNew> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) {
+ public static Map<Long, HLLCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) {
final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc);
final int rowkeyLength = cubeDesc.getRowkey().getRowKeyColumns().length;
final List<Long> allCuboidIds = new CuboidScheduler(cubeDesc).getAllCuboidIds();
@@ -84,9 +84,9 @@ public class CubingUtils {
return result;
}
});
- final Map<Long, HyperLogLogPlusCounterNew> result = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
+ final Map<Long, HLLCounter> result = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
for (Long cuboidId : allCuboidIds) {
- result.put(cuboidId, new HyperLogLogPlusCounterNew(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
+ result.put(cuboidId, new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
@@ -118,9 +118,9 @@ public class CubingUtils {
}
}
- for (Map.Entry<Long, HyperLogLogPlusCounterNew> longHyperLogLogPlusCounterNewEntry : result.entrySet()) {
+ for (Map.Entry<Long, HLLCounter> longHyperLogLogPlusCounterNewEntry : result.entrySet()) {
Long cuboidId = longHyperLogLogPlusCounterNewEntry.getKey();
- HyperLogLogPlusCounterNew counter = longHyperLogLogPlusCounterNewEntry.getValue();
+ HLLCounter counter = longHyperLogLogPlusCounterNewEntry.getValue();
Hasher hc = hf.newHasher();
final Integer[] cuboidBitSet = allCuboidsBitSet.get(cuboidId);
for (int position = 0; position < cuboidBitSet.length; position++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
index 6cbf237..b8d116c 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
@@ -26,7 +26,7 @@ import java.util.List;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo.Builder;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.LongMutable;
@@ -106,16 +106,16 @@ public class UnitTestSupport {
String d_01_15 = datePlus("2015-01-15", i * 4);
String d_01_16 = datePlus("2015-01-16", i * 4);
String d_01_17 = datePlus("2015-01-17", i * 4);
- result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
- result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
- result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
- result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
- result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
- result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
- result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
- result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
- result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
- result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
+ result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
+ result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"), new HLLCounter(14)));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java
index f80bd24..85d8c37 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java
@@ -34,7 +34,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanRequestBuilder;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.benchmark.SortedGTRecordGenerator.Randomizer;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -80,7 +80,7 @@ public class GTScannerBenchmark2 {
gen.addDimension(100, 4, null);
gen.addMeasure(8);
gen.addMeasure(8, new Randomizer() {
- HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew(12);
+ HLLCounter hllc = new HLLCounter(12);
@Override
public int fillRandom(Random rand, byte[] array, int offset) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 66a6b51..8ffe055 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -26,7 +26,7 @@ import org.apache.kylin.measure.basic.LongSumAggregator;
import org.apache.kylin.measure.bitmap.BitmapAggregator;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.hllc.HLLCAggregator;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.datatype.LongMutable;
import org.github.jamm.MemoryMeter;
@@ -105,7 +105,7 @@ public class AggregationCacheMemSizeTest {
private HLLCAggregator createHLLCAggr() {
HLLCAggregator hllcAggregator = new HLLCAggregator(14);
- hllcAggregator.aggregate(new HyperLogLogPlusCounterNew(14));
+ hllcAggregator.aggregate(new HLLCounter(14));
return hllcAggregator;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index cd1aa96..0f3f3a9 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -26,7 +26,7 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.bitmap.BitmapCounter;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -57,7 +57,7 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase {
DoubleMutable d = new DoubleMutable(1.0);
LongMutable l = new LongMutable(2);
BigDecimal b = new BigDecimal("333.1234");
- HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew(16);
+ HLLCounter hllc = new HLLCounter(16);
hllc.add("1234567");
hllc.add("abcdefg");
BitmapCounter bitmap = new BitmapCounter();
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index 031636e..89ff382 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -36,7 +36,7 @@ import org.apache.kylin.metadata.tuple.TupleInfo;
* MeasureType captures how a kind of aggregation is defined, how it is calculated
* during cube build, and how it is involved in query and storage scan.
*
- * @param <T> the Java type of aggregation data object, e.g. HyperLogLogPlusCounterOld
+ * @param <T> the Java type of aggregation data object, e.g. HLLCounter
*/
abstract public class MeasureType<T> {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
index d94dec9..694459b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -62,7 +62,7 @@ import com.google.common.collect.Maps;
}
</pre>
*
- * @param <T> the Java type of aggregation data object, e.g. HyperLogLogPlusCounterOld
+ * @param <T> the Java type of aggregation data object, e.g. HLLCounter
*/
abstract public class MeasureTypeFactory<T> {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
index 26ee6ab..c5814aa 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
@@ -25,7 +25,6 @@ import java.util.Map;
* Created by xiefan on 16-12-9.
*/
public class DenseRegister implements Register {
- private int p;
private int m;
@@ -41,7 +40,7 @@ public class DenseRegister implements Register {
}
@Override
- public Byte get(int pos) {
+ public byte get(int pos) {
return register[pos];
}
@@ -80,11 +79,28 @@ public class DenseRegister implements Register {
}
@Override
- public int getHashCode() {
- return Arrays.hashCode(register);
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + Arrays.hashCode(register);
+ return result;
}
- public byte[] getRawRegister() {
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ DenseRegister other = (DenseRegister) obj;
+ if (!Arrays.equals(register, other.register))
+ return false;
+ return true;
+ }
+
+ byte[] getRawRegister() {
return this.register;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
index ca73285..5966c04 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
@@ -23,10 +23,10 @@ import org.apache.kylin.measure.MeasureAggregator;
/**
*/
@SuppressWarnings("serial")
-public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounterNew> {
+public class HLLCAggregator extends MeasureAggregator<HLLCounter> {
final int precision;
- HyperLogLogPlusCounterNew sum = null;
+ HLLCounter sum = null;
public HLLCAggregator(int precision) {
this.precision = precision;
@@ -38,15 +38,15 @@ public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounterNew>
}
@Override
- public void aggregate(HyperLogLogPlusCounterNew value) {
+ public void aggregate(HLLCounter value) {
if (sum == null)
- sum = new HyperLogLogPlusCounterNew(value);
+ sum = new HLLCounter(value);
else
sum.merge(value);
}
@Override
- public HyperLogLogPlusCounterNew getState() {
+ public HLLCounter getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index 481fa4e..9601653 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -33,15 +33,15 @@ import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.collect.ImmutableMap;
-public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounterNew> {
+public class HLLCMeasureType extends MeasureType<HLLCounter> {
public static final String FUNC_COUNT_DISTINCT = FunctionDesc.FUNC_COUNT_DISTINCT;
public static final String DATATYPE_HLLC = "hllc";
- public static class Factory extends MeasureTypeFactory<HyperLogLogPlusCounterNew> {
+ public static class Factory extends MeasureTypeFactory<HLLCounter> {
@Override
- public MeasureType<HyperLogLogPlusCounterNew> createMeasureType(String funcName, DataType dataType) {
+ public MeasureType<HLLCounter> createMeasureType(String funcName, DataType dataType) {
return new HLLCMeasureType(funcName, dataType);
}
@@ -56,7 +56,7 @@ public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounterNew> {
}
@Override
- public Class<? extends DataTypeSerializer<HyperLogLogPlusCounterNew>> getAggrDataTypeSerializer() {
+ public Class<? extends DataTypeSerializer<HLLCounter>> getAggrDataTypeSerializer() {
return HLLCSerializer.class;
}
}
@@ -91,13 +91,13 @@ public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounterNew> {
}
@Override
- public MeasureIngester<HyperLogLogPlusCounterNew> newIngester() {
- return new MeasureIngester<HyperLogLogPlusCounterNew>() {
- HyperLogLogPlusCounterNew current = new HyperLogLogPlusCounterNew(dataType.getPrecision());
+ public MeasureIngester<HLLCounter> newIngester() {
+ return new MeasureIngester<HLLCounter>() {
+ HLLCounter current = new HLLCounter(dataType.getPrecision());
@Override
- public HyperLogLogPlusCounterNew valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
- HyperLogLogPlusCounterNew hllc = current;
+ public HLLCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ HLLCounter hllc = current;
hllc.clear();
for (String v : values) {
if (v != null)
@@ -109,7 +109,7 @@ public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounterNew> {
}
@Override
- public MeasureAggregator<HyperLogLogPlusCounterNew> newAggregator() {
+ public MeasureAggregator<HLLCounter> newAggregator() {
return new HLLCAggregator(dataType.getPrecision());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
index 1d01abc..e0992c7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
@@ -28,10 +28,10 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
* @author yangli9
*
*/
-public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounterNew> {
+public class HLLCSerializer extends DataTypeSerializer<HLLCounter> {
// be thread-safe and avoid repeated obj creation
- private ThreadLocal<HyperLogLogPlusCounterNew> current = new ThreadLocal<HyperLogLogPlusCounterNew>();
+ private ThreadLocal<HLLCounter> current = new ThreadLocal<HLLCounter>();
private int precision;
@@ -40,7 +40,7 @@ public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounterNew
}
@Override
- public void serialize(HyperLogLogPlusCounterNew value, ByteBuffer out) {
+ public void serialize(HLLCounter value, ByteBuffer out) {
try {
value.writeRegisters(out);
} catch (IOException e) {
@@ -48,18 +48,18 @@ public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounterNew
}
}
- private HyperLogLogPlusCounterNew current() {
- HyperLogLogPlusCounterNew hllc = current.get();
+ private HLLCounter current() {
+ HLLCounter hllc = current.get();
if (hllc == null) {
- hllc = new HyperLogLogPlusCounterNew(precision);
+ hllc = new HLLCounter(precision);
current.set(hllc);
}
return hllc;
}
@Override
- public HyperLogLogPlusCounterNew deserialize(ByteBuffer in) {
- HyperLogLogPlusCounterNew hllc = current();
+ public HLLCounter deserialize(ByteBuffer in) {
+ HLLCounter hllc = current();
try {
hllc.readRegisters(in);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounter.java
new file mode 100644
index 0000000..22b5e55
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounter.java
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import org.apache.kylin.common.util.BytesUtil;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Collection;
+import java.util.Map;
+
+@SuppressWarnings("serial")
+public class HLLCounter implements Serializable, Comparable<HLLCounter> {
+
+ // not final for test purpose
+ static double OVERFLOW_FACTOR = 0.01;
+
+ private int p;
+
+ private int m;
+
+ private HashFunction hashFunc = Hashing.murmur3_128();
+
+ private Register register;
+
+ public HLLCounter() {
+ this(10, RegisterType.SPARSE, Hashing.murmur3_128());
+ }
+
+ public HLLCounter(int p) {
+ this(p, RegisterType.SPARSE, Hashing.murmur3_128());
+ }
+
+ public HLLCounter(int p, HashFunction hashFunc) {
+ this(p, RegisterType.SPARSE, hashFunc);
+ }
+
+ public HLLCounter(HLLCounter another) {
+ this(another.p, another.hashFunc);
+ merge(another);
+ }
+
+ HLLCounter(int p, RegisterType type) {
+ this(p, type, Hashing.murmur3_128());
+ }
+
+ HLLCounter(int p, RegisterType type, HashFunction hashFunc) {
+ this.p = p;
+ this.m = 1 << p;//(int) Math.pow(2, p);
+ this.hashFunc = hashFunc;
+ if (type == RegisterType.SPARSE) {
+ this.register = new SparseRegister();
+ } else {
+ this.register = new DenseRegister(p);
+ }
+ }
+
+ private boolean isDense(int size) {
+ double over = OVERFLOW_FACTOR * m;
+ return size > (int) over;
+ }
+
+ public void add(int value) {
+ add(hashFunc.hashInt(value).asLong());
+ }
+
+ public void add(String value) {
+ add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
+ }
+
+ public void add(byte[] value) {
+ add(hashFunc.hashBytes(value).asLong());
+ }
+
+ public void add(byte[] value, int offset, int length) {
+ add(hashFunc.hashBytes(value, offset, length).asLong());
+ }
+
+ protected void add(long hash) {
+ int bucketMask = m - 1;
+ int bucket = (int) (hash & bucketMask);
+ int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
+ Byte b = register.get(bucket);
+ if (b == null || (byte) firstOnePos > b) {
+ register.set(bucket, (byte) firstOnePos);
+ }
+ toDenseIfNeeded();
+ }
+
+ private void toDenseIfNeeded() {
+ if (register instanceof SparseRegister) {
+ if (isDense(register.getSize())) {
+ register = ((SparseRegister) register).toDense(p);
+ }
+ }
+ }
+
+ public void merge(HLLCounter another) {
+ assert this.p == another.p;
+ assert this.hashFunc == another.hashFunc;
+ if (register instanceof SparseRegister && another.register instanceof SparseRegister) {
+ register.merge(another.register);
+ toDenseIfNeeded();
+ } else if (register instanceof SparseRegister && another.register instanceof DenseRegister) {
+ register = ((SparseRegister) register).toDense(p);
+ register.merge(another.register);
+ } else {
+ register.merge(another.register);
+ }
+ }
+
+ public long getCountEstimate() {
+ return new HLLCSnapshot(this).getCountEstimate();
+ }
+
+ public int getPrecision() {
+ return this.p;
+ }
+
+ public double getErrorRate() {
+ return 1.04 / Math.sqrt(m);
+ }
+
+ @Override
+ public String toString() {
+ return "" + getCountEstimate();
+ }
+
+ // ============================================================================
+
+ // a memory efficient snapshot of HLL registers which can yield count estimate later
+ public static class HLLCSnapshot {
+ byte p;
+ double registerSum;
+ int zeroBuckets;
+
+ public HLLCSnapshot(HLLCounter hllc) {
+ p = (byte) hllc.p;
+ registerSum = 0;
+ zeroBuckets = 0;
+ Register register = hllc.getRegister();
+ DenseRegister dr;
+ if (register instanceof SparseRegister) {
+ dr = ((SparseRegister) register).toDense(p);
+ } else {
+ dr = (DenseRegister) register;
+ }
+ byte[] registers = dr.getRawRegister();
+ for (int i = 0; i < hllc.m; i++) {
+ if (registers[i] == 0) {
+ registerSum++;
+ zeroBuckets++;
+ } else {
+ registerSum += 1.0 / (1L << registers[i]);
+ }
+ }
+ }
+
+ public long getCountEstimate() {
+ int m = 1 << p;
+ double alpha = 0.7213 / (1 + 1.079 / m);
+ double estimate = alpha * m * m / registerSum;
+
+ // small cardinality adjustment
+ if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
+ estimate = m * Math.log(m * 1.0 / zeroBuckets);
+ } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
+ estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
+ }
+
+ return Math.round(estimate);
+ }
+ }
+
+ public static void main(String[] args) throws IOException {
+ dumpErrorRates();
+ }
+
+ static void dumpErrorRates() {
+ for (int p = 10; p <= 18; p++) {
+ double rate = new HLLCounter(p, RegisterType.SPARSE).getErrorRate();
+ double er = Math.round(rate * 10000) / 100D;
+ double er2 = Math.round(rate * 2 * 10000) / 100D;
+ double er3 = Math.round(rate * 3 * 10000) / 100D;
+ long size = Math.round(Math.pow(2, p));
+ System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
+ }
+ }
+
+ public Register getRegister() {
+ return register;
+ }
+
+ public void clear() {
+ register.clear();
+ }
+
+ // ============================================================================
+
+ public void writeRegisters(final ByteBuffer out) throws IOException {
+
+ final int indexLen = getRegisterIndexSize();
+ int size = register.getSize();
+
+ // decide output scheme -- map (3*size bytes) or array (2^p bytes)
+ byte scheme;
+ if (register instanceof SparseRegister || 5 + (indexLen + 1) * size < m) {
+ scheme = 0; // map
+ } else {
+ scheme = 1; // array
+ }
+ out.put(scheme);
+ if (scheme == 0) { // map scheme
+ BytesUtil.writeVInt(size, out);
+ if (register instanceof SparseRegister) { //sparse register
+ Collection<Map.Entry<Integer, Byte>> allValue = ((SparseRegister) register).getAllValue();
+ for (Map.Entry<Integer, Byte> entry : allValue) {
+ writeUnsigned(entry.getKey(), indexLen, out);
+ out.put(entry.getValue());
+ }
+ } else { //dense register
+ byte[] registers = ((DenseRegister) register).getRawRegister();
+ for (int i = 0; i < m; i++) {
+ if (registers[i] > 0) {
+ writeUnsigned(i, indexLen, out);
+ out.put(registers[i]);
+ }
+ }
+ }
+ } else if (scheme == 1) { // array scheme
+ out.put(((DenseRegister) register).getRawRegister());
+ } else
+ throw new IllegalStateException();
+ }
+
+ public void readRegisters(ByteBuffer in) throws IOException {
+ byte scheme = in.get();
+ if (scheme == 0) { // map scheme
+ clear();
+ int size = BytesUtil.readVInt(in);
+ if (size > m)
+ throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
+ if (isDense(size)) {
+ register = new DenseRegister(p);
+ } else {
+ register = new SparseRegister();//default is sparse
+ }
+ int indexLen = getRegisterIndexSize();
+ int key = 0;
+ for (int i = 0; i < size; i++) {
+ key = readUnsigned(in, indexLen);
+ register.set(key, in.get());
+ }
+ } else if (scheme == 1) { // array scheme
+ if (register instanceof SparseRegister) {
+ register = new DenseRegister(p);
+ }
+ in.get(((DenseRegister) register).getRawRegister());
+ } else
+ throw new IllegalStateException();
+ }
+
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ int len;
+ byte scheme = in.get();
+ if (scheme == 0) { // map scheme
+ int size = BytesUtil.readVInt(in);
+ int indexLen = getRegisterIndexSize();
+ len = in.position() - mark + (indexLen + 1) * size;
+ } else {
+ len = in.position() - mark + m;
+ }
+
+ in.position(mark);
+ return len;
+ }
+
+ public int maxLength() {
+ return 1 + m;
+ }
+
+ private int getRegisterIndexSize() {
+ return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
+ result = prime * result + p;
+ result = prime * result + register.hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ HLLCounter other = (HLLCounter) obj;
+ if (!hashFunc.equals(other.hashFunc))
+ return false;
+ if (p != other.p)
+ return false;
+ if (!register.equals(other.register))
+ return false;
+ return true;
+ }
+
+ @Override
+ public int compareTo(HLLCounter o) {
+ if (o == null)
+ return 1;
+
+ long e1 = this.getCountEstimate();
+ long e2 = o.getCountEstimate();
+
+ if (e1 == e2)
+ return 0;
+ else if (e1 > e2)
+ return 1;
+ else
+ return -1;
+ }
+
+ public static void writeUnsigned(int num, int size, ByteBuffer out) {
+ for (int i = 0; i < size; i++) {
+ out.put((byte) num);
+ num >>>= 8;
+ }
+ }
+
+ public static int readUnsigned(ByteBuffer in, int size) {
+ int integer = 0;
+ int mask = 0xff;
+ int shift = 0;
+ for (int i = 0; i < size; i++) {
+ integer |= (in.get() << shift) & mask;
+ mask = mask << 8;
+ shift += 8;
+ }
+ return integer;
+ }
+
+ public RegisterType getRegisterType() {
+ if (register instanceof SparseRegister)
+ return RegisterType.SPARSE;
+ else
+ return RegisterType.DENSE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounterOld.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounterOld.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounterOld.java
new file mode 100644
index 0000000..5cbdd43
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCounterOld.java
@@ -0,0 +1,393 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+import org.apache.kylin.common.util.BytesUtil;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+
+/**
+ * Deprecated, use HLLCounter instead.
+ *
+ * About compression, test on HLLC data shows
+ *
+ * - LZF compression ratio is around 65%-80%, fast
+ * - GZIP compression ratio is around 41%-46%, very slow
+ */
+@Deprecated
+@SuppressWarnings("serial")
+public class HLLCounterOld implements Serializable, Comparable<HLLCounterOld> {
+
+ private final int p;
+ private final int m;
+ private final HashFunction hashFunc;
+ byte[] registers;
+ int singleBucket;
+
+ public HLLCounterOld() {
+ this(10);
+ }
+
+ public HLLCounterOld(int p) {
+ this(p, Hashing.murmur3_128());
+ }
+
+ public HLLCounterOld(HLLCounterOld another) {
+ this(another.p, another.hashFunc);
+ merge(another);
+ }
+
+ /** The larger p is, the more storage (2^p bytes), the better accuracy */
+ private HLLCounterOld(int p, HashFunction hashFunc) {
+ this.p = p;
+ this.m = 1 << p;//(int) Math.pow(2, p);
+ this.hashFunc = hashFunc;
+ this.registers = new byte[m];
+ this.singleBucket = -1;
+ }
+
+ public void clear() {
+ byte zero = (byte) 0;
+ if (singleBucket == -1) {
+ //nothing
+ } else if (singleBucket >= 0) {
+ registers[singleBucket] = 0;
+ } else {
+ Arrays.fill(registers, zero);
+ }
+ singleBucket = -1;
+ }
+
+ public void add(int value) {
+ add(hashFunc.hashInt(value).asLong());
+ }
+
+ public void add(String value) {
+ add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
+ }
+
+ public void add(byte[] value) {
+ add(hashFunc.hashBytes(value).asLong());
+ }
+
+ public void add(byte[] value, int offset, int length) {
+ add(hashFunc.hashBytes(value, offset, length).asLong());
+ }
+
+ protected void add(long hash) {
+ int bucketMask = m - 1;
+ int bucket = (int) (hash & bucketMask);
+ int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
+
+ if (firstOnePos > registers[bucket])
+ registers[bucket] = (byte) firstOnePos;
+
+ if (singleBucket == -1)
+ singleBucket = bucket;
+ else
+ singleBucket = Integer.MIN_VALUE;
+ }
+
+ public void merge(HLLCounterOld another) {
+ assert this.p == another.p;
+ assert this.hashFunc == another.hashFunc;
+
+ // quick path for single value HLLC
+ if (another.singleBucket == -1) {
+ return;
+ } else if (another.singleBucket >= 0) {
+ int b = another.singleBucket;
+ if (registers[b] < another.registers[b])
+ registers[b] = another.registers[b];
+ } else {
+ // normal path
+ for (int i = 0; i < m; i++) {
+ if (registers[i] < another.registers[i])
+ registers[i] = another.registers[i];
+ }
+ }
+ singleBucket = Integer.MIN_VALUE;
+ }
+
+ public long getCountEstimate() {
+ return new HLLCSnapshot(this).getCountEstimate();
+ }
+
+ public int getPrecision() {
+ return this.p;
+ }
+
+ public double getErrorRate() {
+ return 1.04 / Math.sqrt(m);
+ }
+
+ private int size() {
+ if (singleBucket == -1) {
+ return 0;
+ } else if (singleBucket >= 0) {
+ return 1;
+ } else {
+ int size = 0;
+ for (int i = 0; i < m; i++) {
+ if (registers[i] > 0)
+ size++;
+ }
+ return size;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "" + getCountEstimate();
+ }
+
+ // ============================================================================
+
+ // a memory efficient snapshot of HLL registers which can yield count
+ // estimate later
+ public static class HLLCSnapshot {
+ byte p;
+ double registerSum;
+ int zeroBuckets;
+
+ public HLLCSnapshot(HLLCounterOld hllc) {
+ p = (byte) hllc.p;
+ registerSum = 0;
+ zeroBuckets = 0;
+
+ byte[] registers = hllc.registers;
+ for (int i = 0; i < hllc.m; i++) {
+ if (registers[i] == 0) {
+ registerSum++;
+ zeroBuckets++;
+ } else {
+ registerSum += 1.0 / (1L << registers[i]);
+ }
+ }
+ }
+
+ public long getCountEstimate() {
+ int m = 1 << p;
+ double alpha = 0.7213 / (1 + 1.079 / m);
+ double estimate = alpha * m * m / registerSum;
+
+ // small cardinality adjustment
+ if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
+ estimate = m * Math.log(m * 1.0 / zeroBuckets);
+ } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
+ estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
+ }
+
+ return Math.round(estimate);
+ }
+ }
+
+ // ============================================================================
+
+ public void writeRegisters(final ByteBuffer out) throws IOException {
+
+ final int indexLen = getRegisterIndexSize();
+ int size = size();
+
+ // decide output scheme -- map (3*size bytes) or array (2^p bytes)
+ byte scheme;
+ if (5 + (indexLen + 1) * size < m) // 5 is max len of vint
+ scheme = 0; // map
+ else
+ scheme = 1; // array
+ out.put(scheme);
+
+ if (scheme == 0) { // map scheme
+ BytesUtil.writeVInt(size, out);
+ if (singleBucket == -1) {
+ // no non-zero register
+ } else if (singleBucket >= 0) {
+ writeUnsigned(singleBucket, indexLen, out);
+ out.put(registers[singleBucket]);
+ } else {
+ for (int i = 0; i < m; i++) {
+ if (registers[i] > 0) {
+ writeUnsigned(i, indexLen, out);
+ out.put(registers[i]);
+ }
+ }
+ }
+ } else if (scheme == 1) { // array scheme
+ out.put(registers);
+ } else
+ throw new IllegalStateException();
+ }
+
+ public void readRegisters(ByteBuffer in) throws IOException {
+ byte scheme = in.get();
+
+ if (scheme == 0) { // map scheme
+ clear();
+ int size = BytesUtil.readVInt(in);
+ if (size > m)
+ throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
+ int indexLen = getRegisterIndexSize();
+ int key = 0;
+ for (int i = 0; i < size; i++) {
+ key = readUnsigned(in, indexLen);
+ registers[key] = in.get();
+ }
+
+ if (size == 0)
+ singleBucket = -1;
+ else if (size == 1)
+ singleBucket = key;
+ else
+ singleBucket = Integer.MIN_VALUE;
+
+ } else if (scheme == 1) { // array scheme
+ in.get(registers);
+ singleBucket = Integer.MIN_VALUE;
+ } else
+ throw new IllegalStateException();
+ }
+
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ int len;
+
+ byte scheme = in.get();
+ if (scheme == 0) { // map scheme
+ int size = BytesUtil.readVInt(in);
+ int indexLen = getRegisterIndexSize();
+ len = in.position() - mark + (indexLen + 1) * size;
+ } else {
+ len = in.position() - mark + m;
+ }
+
+ in.position(mark);
+ return len;
+ }
+
+ public int maxLength() {
+ return 1 + m;
+ }
+
+ /*public void writeRegistersArray(final ByteBuffer out) {
+ out.put(this.registers);
+ }
+
+ public void readRegistersArray(ByteBuffer in) {
+ in.get(registers, 0, m);
+ singleBucket = Integer.MIN_VALUE;
+ }*/
+
+ private int getRegisterIndexSize() {
+ return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
+ result = prime * result + p;
+ result = prime * result + Arrays.hashCode(registers);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ HLLCounterOld other = (HLLCounterOld) obj;
+ if (hashFunc == null) {
+ if (other.hashFunc != null)
+ return false;
+ } else if (!hashFunc.equals(other.hashFunc))
+ return false;
+ if (p != other.p)
+ return false;
+ if (!Arrays.equals(registers, other.registers))
+ return false;
+ return true;
+ }
+
+ @Override
+ public int compareTo(HLLCounterOld o) {
+ if (o == null)
+ return 1;
+
+ long e1 = this.getCountEstimate();
+ long e2 = o.getCountEstimate();
+
+ if (e1 == e2)
+ return 0;
+ else if (e1 > e2)
+ return 1;
+ else
+ return -1;
+ }
+
+ public static void main(String[] args) throws IOException {
+ dumpErrorRates();
+ }
+
+ static void dumpErrorRates() {
+ for (int p = 10; p <= 18; p++) {
+ double rate = new HLLCounterOld(p).getErrorRate();
+ double er = Math.round(rate * 10000) / 100D;
+ double er2 = Math.round(rate * 2 * 10000) / 100D;
+ double er3 = Math.round(rate * 3 * 10000) / 100D;
+ long size = Math.round(Math.pow(2, p));
+ System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
+ }
+ }
+
+ /**
+ *
+ * @param num
+ * @param size
+ * @param out
+ */
+ public static void writeUnsigned(int num, int size, ByteBuffer out) {
+ for (int i = 0; i < size; i++) {
+ out.put((byte) num);
+ num >>>= 8;
+ }
+ }
+
+ public static int readUnsigned(ByteBuffer in, int size) {
+ int integer = 0;
+ int mask = 0xff;
+ int shift = 0;
+ for (int i = 0; i < size; i++) {
+ integer |= (in.get() << shift) & mask;
+ mask = mask << 8;
+ shift += 8;
+ }
+ return integer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
index a72ad09..438a33f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
@@ -31,21 +31,21 @@ public class HLLDistinctCountAggFunc {
private static final Logger logger = LoggerFactory.getLogger(HLLDistinctCountAggFunc.class);
- public static HyperLogLogPlusCounterNew init() {
+ public static HLLCounter init() {
return null;
}
- public static HyperLogLogPlusCounterNew initAdd(Object v) {
+ public static HLLCounter initAdd(Object v) {
if (v instanceof Long) { // holistic case
long l = (Long) v;
return new FixedValueHLLCMockup(l);
} else {
- HyperLogLogPlusCounterNew c = (HyperLogLogPlusCounterNew) v;
- return new HyperLogLogPlusCounterNew(c);
+ HLLCounter c = (HLLCounter) v;
+ return new HLLCounter(c);
}
}
- public static HyperLogLogPlusCounterNew add(HyperLogLogPlusCounterNew counter, Object v) {
+ public static HLLCounter add(HLLCounter counter, Object v) {
if (v instanceof Long) { // holistic case
long l = (Long) v;
if (counter == null) {
@@ -58,9 +58,9 @@ public class HLLDistinctCountAggFunc {
return counter;
}
} else {
- HyperLogLogPlusCounterNew c = (HyperLogLogPlusCounterNew) v;
+ HLLCounter c = (HLLCounter) v;
if (counter == null) {
- return new HyperLogLogPlusCounterNew(c);
+ return new HLLCounter(c);
} else {
counter.merge(c);
return counter;
@@ -68,16 +68,16 @@ public class HLLDistinctCountAggFunc {
}
}
- public static HyperLogLogPlusCounterNew merge(HyperLogLogPlusCounterNew counter0, Object counter1) {
+ public static HLLCounter merge(HLLCounter counter0, Object counter1) {
return add(counter0, counter1);
}
- public static long result(HyperLogLogPlusCounterNew counter) {
+ public static long result(HLLCounter counter) {
return counter == null ? 0L : counter.getCountEstimate();
}
@SuppressWarnings("serial")
- private static class FixedValueHLLCMockup extends HyperLogLogPlusCounterNew {
+ private static class FixedValueHLLCMockup extends HLLCounter {
private Long value = null;
@@ -107,7 +107,7 @@ public class HLLDistinctCountAggFunc {
}
@Override
- public void merge(HyperLogLogPlusCounterNew another) {
+ public void merge(HLLCounter another) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterNew.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterNew.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterNew.java
deleted file mode 100644
index d7329f6..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterNew.java
+++ /dev/null
@@ -1,388 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.hllc;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-import org.apache.kylin.common.util.BytesUtil;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.Collection;
-import java.util.Map;
-
-@SuppressWarnings("serial")
-public class HyperLogLogPlusCounterNew implements Serializable, Comparable<HyperLogLogPlusCounterNew> {
-
- private int p;
-
- private int m;
-
- private HashFunction hashFunc = Hashing.murmur3_128();
-
- private Register register;
-
- public static double overflowFactor = 0.01;
-
- public HyperLogLogPlusCounterNew(int p, RegisterType type, HashFunction hashFunc) {
- this.p = p;
- this.m = 1 << p;//(int) Math.pow(2, p);
- this.hashFunc = hashFunc;
- if (type == RegisterType.SPARSE) {
- double over = overflowFactor * m;
- this.register = new SparseRegister((int) over);
- } else {
- this.register = new DenseRegister(p);
- }
- }
-
- public HyperLogLogPlusCounterNew() {
- this(10, RegisterType.SPARSE, Hashing.murmur3_128());
- }
-
- public HyperLogLogPlusCounterNew(int p) {
- this(p, RegisterType.SPARSE, Hashing.murmur3_128());
- }
-
- public HyperLogLogPlusCounterNew(int p, RegisterType type) {
- this(p, type, Hashing.murmur3_128());
- }
-
- public HyperLogLogPlusCounterNew(int p, HashFunction hashFunc) {
- this(p, RegisterType.SPARSE, hashFunc);
- }
-
- public HyperLogLogPlusCounterNew(HyperLogLogPlusCounterNew another) {
- this(another.p, another.hashFunc);
- merge(another);
- }
-
- public void add(int value) {
- add(hashFunc.hashInt(value).asLong());
- }
-
- public void add(String value) {
- add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
- }
-
- public void add(byte[] value) {
- add(hashFunc.hashBytes(value).asLong());
- }
-
- public void add(byte[] value, int offset, int length) {
- add(hashFunc.hashBytes(value, offset, length).asLong());
- }
-
- protected void add(long hash) {
- int bucketMask = m - 1;
- int bucket = (int) (hash & bucketMask);
- int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
- Byte b = register.get(bucket);
- if (b == null || (byte) firstOnePos > b) {
- register.set(bucket, (byte) firstOnePos);
- }
- if (register instanceof SparseRegister) {
- if (((SparseRegister) register).isOverThreshold()) {
- register = ((SparseRegister) register).toDense(p);
- }
- }
- }
-
- public void merge(HyperLogLogPlusCounterNew another) {
- assert this.p == another.p;
- assert this.hashFunc == another.hashFunc;
- if (register instanceof SparseRegister && another.register instanceof SparseRegister) {
- register.merge(another.register);
- if (((SparseRegister) register).isOverThreshold()) {
- register = ((SparseRegister) register).toDense(p);
- }
- } else if (register instanceof SparseRegister && another.register instanceof DenseRegister) {
- register = ((SparseRegister) register).toDense(p);
- register.merge(another.register);
- } else {
- register.merge(another.register);
- }
- }
-
- public long getCountEstimate() {
- return new HLLCSnapshot(this).getCountEstimate();
- }
-
- public int getPrecision() {
- return this.p;
- }
-
- public double getErrorRate() {
- return 1.04 / Math.sqrt(m);
- }
-
- @Override
- public String toString() {
- return "" + getCountEstimate();
- }
-
- // ============================================================================
-
- // a memory efficient snapshot of HLL registers which can yield count
- // estimate later
- public static class HLLCSnapshot {
- byte p;
- double registerSum;
- int zeroBuckets;
-
- public HLLCSnapshot(HyperLogLogPlusCounterNew hllc) {
- p = (byte) hllc.p;
- registerSum = 0;
- zeroBuckets = 0;
- Register register = hllc.getRegister();
- DenseRegister dr;
- if (register instanceof SparseRegister) {
- dr = ((SparseRegister) register).toDense(p);
- } else {
- dr = (DenseRegister) register;
- }
- byte[] registers = dr.getRawRegister();
- for (int i = 0; i < hllc.m; i++) {
- if (registers[i] == 0) {
- registerSum++;
- zeroBuckets++;
- } else {
- registerSum += 1.0 / (1L << registers[i]);
- }
- }
- }
-
- public long getCountEstimate() {
- int m = 1 << p;
- double alpha = 0.7213 / (1 + 1.079 / m);
- double estimate = alpha * m * m / registerSum;
-
- // small cardinality adjustment
- if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
- estimate = m * Math.log(m * 1.0 / zeroBuckets);
- } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
- estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
- }
-
- return Math.round(estimate);
- }
- }
-
- public static void main(String[] args) throws IOException {
- dumpErrorRates();
- }
-
- static void dumpErrorRates() {
- for (int p = 10; p <= 18; p++) {
- double rate = new HyperLogLogPlusCounterNew(p, RegisterType.SPARSE).getErrorRate();
- double er = Math.round(rate * 10000) / 100D;
- double er2 = Math.round(rate * 2 * 10000) / 100D;
- double er3 = Math.round(rate * 3 * 10000) / 100D;
- long size = Math.round(Math.pow(2, p));
- System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
- }
- }
-
- public Register getRegister() {
- return register;
- }
-
- public void clear() {
- register.clear();
- }
-
- public RegisterType getRegisterType() {
- if (register instanceof SparseRegister)
- return RegisterType.SPARSE;
- else
- return RegisterType.DENSE;
- }
-
- // ============================================================================
-
- public void writeRegisters(final ByteBuffer out) throws IOException {
-
- final int indexLen = getRegisterIndexSize();
- int size = size();
-
- // decide output scheme -- map (3*size bytes) or array (2^p bytes)
- byte scheme;
- //byte type;
- if (register instanceof SparseRegister || 5 + (indexLen + 1) * size < m) {
- scheme = 0; //map
- } else {
- scheme = 1; // array
- }
- out.put(scheme);
- if (scheme == 0) { // map scheme
- BytesUtil.writeVInt(size, out);
- if (register instanceof SparseRegister) { //sparse\u3000register
- Collection<Map.Entry<Integer, Byte>> allValue = ((SparseRegister) register).getAllValue();
- for (Map.Entry<Integer, Byte> entry : allValue) {
- writeUnsigned(entry.getKey(), indexLen, out);
- out.put(entry.getValue());
- }
- } else { //dense register
- byte[] registers = ((DenseRegister) register).getRawRegister();
- for (int i = 0; i < m; i++) {
- if (registers[i] > 0) {
- writeUnsigned(i, indexLen, out);
- out.put(registers[i]);
- }
- }
- }
- } else if (scheme == 1) { // array scheme
- out.put(((DenseRegister) register).getRawRegister());
- } else
- throw new IllegalStateException();
- }
-
- public void readRegisters(ByteBuffer in) throws IOException {
- byte scheme = in.get();
- if (scheme == 0) { // map scheme
- clear();
- int size = BytesUtil.readVInt(in);
- if (size > m)
- throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
- double over = overflowFactor * m;
- if (size > (int) over) {
- this.register = new DenseRegister(p);
- } else {
- this.register = new SparseRegister((int) over);//default is sparse
- }
- int indexLen = getRegisterIndexSize();
- int key = 0;
- for (int i = 0; i < size; i++) {
- key = readUnsigned(in, indexLen);
- register.set(key, in.get());
- }
- } else if (scheme == 1) { // array scheme
- this.register = new DenseRegister(p);
- for (int i = 0; i < m; i++) {
- register.set(i, in.get());
- }
- } else
- throw new IllegalStateException();
- }
-
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
- int len;
- byte scheme = in.get();
- if (scheme == 0) { // map scheme
- int size = BytesUtil.readVInt(in);
- int indexLen = getRegisterIndexSize();
- len = in.position() - mark + (indexLen + 1) * size;
- } else {
- len = in.position() - mark + m;
- }
-
- in.position(mark);
- return len;
- }
-
- public int maxLength() {
- return 1 + m;
- }
-
- private int getRegisterIndexSize() {
- return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
- result = prime * result + p;
- result = prime * result + register.getHashCode();
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- HyperLogLogPlusCounterNew other = (HyperLogLogPlusCounterNew) obj;
- if (hashFunc == null) {
- if (other.hashFunc != null)
- return false;
- } else if (!hashFunc.equals(other.hashFunc))
- return false;
- if (p != other.p)
- return false;
- if (this.getRegisterType() != other.getRegisterType())
- return false;
- if (register.getHashCode() != other.register.getHashCode())
- return false;
- return true;
- }
-
- @Override
- public int compareTo(HyperLogLogPlusCounterNew o) {
- if (o == null)
- return 1;
-
- long e1 = this.getCountEstimate();
- long e2 = o.getCountEstimate();
-
- if (e1 == e2)
- return 0;
- else if (e1 > e2)
- return 1;
- else
- return -1;
- }
-
- /**
- *
- * @param num
- * @param size
- * @param out
- */
- public static void writeUnsigned(int num, int size, ByteBuffer out) {
- for (int i = 0; i < size; i++) {
- out.put((byte) num);
- num >>>= 8;
- }
- }
-
- public static int readUnsigned(ByteBuffer in, int size) {
- int integer = 0;
- int mask = 0xff;
- int shift = 0;
- for (int i = 0; i < size; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
- }
- return integer;
- }
-
- private int size() {
- return register.getSize();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterOld.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterOld.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterOld.java
deleted file mode 100644
index cb5533e..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterOld.java
+++ /dev/null
@@ -1,392 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.hllc;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.BytesUtil;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-
-/**
- * About compression, test on HLLC data shows
- *
- * - LZF compression ratio is around 65%-80%, fast
- * - GZIP compression ratio is around 41%-46%, very slow
- *
- * @author yangli9
- */
-@SuppressWarnings("serial")
-public class HyperLogLogPlusCounterOld implements Serializable, Comparable<HyperLogLogPlusCounterOld> {
-
- private final int p;
- private final int m;
- private final HashFunction hashFunc;
- byte[] registers;
- int singleBucket;
-
- public HyperLogLogPlusCounterOld() {
- this(10);
- }
-
- public HyperLogLogPlusCounterOld(int p) {
- this(p, Hashing.murmur3_128());
- }
-
- public HyperLogLogPlusCounterOld(HyperLogLogPlusCounterOld another) {
- this(another.p, another.hashFunc);
- merge(another);
- }
-
- /** The larger p is, the more storage (2^p bytes), the better accuracy */
- private HyperLogLogPlusCounterOld(int p, HashFunction hashFunc) {
- this.p = p;
- this.m = 1 << p;//(int) Math.pow(2, p);
- this.hashFunc = hashFunc;
- this.registers = new byte[m];
- this.singleBucket = -1;
- }
-
- public void clear() {
- byte zero = (byte) 0;
- if (singleBucket == -1) {
- //nothing
- } else if (singleBucket >= 0) {
- registers[singleBucket] = 0;
- } else {
- Arrays.fill(registers, zero);
- }
- singleBucket = -1;
- }
-
- public void add(int value) {
- add(hashFunc.hashInt(value).asLong());
- }
-
- public void add(String value) {
- add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
- }
-
- public void add(byte[] value) {
- add(hashFunc.hashBytes(value).asLong());
- }
-
- public void add(byte[] value, int offset, int length) {
- add(hashFunc.hashBytes(value, offset, length).asLong());
- }
-
- protected void add(long hash) {
- int bucketMask = m - 1;
- int bucket = (int) (hash & bucketMask);
- int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
-
- if (firstOnePos > registers[bucket])
- registers[bucket] = (byte) firstOnePos;
-
- if (singleBucket == -1)
- singleBucket = bucket;
- else
- singleBucket = Integer.MIN_VALUE;
- }
-
- public void merge(HyperLogLogPlusCounterOld another) {
- assert this.p == another.p;
- assert this.hashFunc == another.hashFunc;
-
- // quick path for single value HLLC
- if (another.singleBucket == -1) {
- return;
- } else if (another.singleBucket >= 0) {
- int b = another.singleBucket;
- if (registers[b] < another.registers[b])
- registers[b] = another.registers[b];
- } else {
- // normal path
- for (int i = 0; i < m; i++) {
- if (registers[i] < another.registers[i])
- registers[i] = another.registers[i];
- }
- }
- singleBucket = Integer.MIN_VALUE;
- }
-
- public long getCountEstimate() {
- return new HLLCSnapshot(this).getCountEstimate();
- }
-
- public int getPrecision() {
- return this.p;
- }
-
- public double getErrorRate() {
- return 1.04 / Math.sqrt(m);
- }
-
- private int size() {
- if (singleBucket == -1) {
- return 0;
- } else if (singleBucket >= 0) {
- return 1;
- } else {
- int size = 0;
- for (int i = 0; i < m; i++) {
- if (registers[i] > 0)
- size++;
- }
- return size;
- }
- }
-
- @Override
- public String toString() {
- return "" + getCountEstimate();
- }
-
- // ============================================================================
-
- // a memory efficient snapshot of HLL registers which can yield count
- // estimate later
- public static class HLLCSnapshot {
- byte p;
- double registerSum;
- int zeroBuckets;
-
- public HLLCSnapshot(HyperLogLogPlusCounterOld hllc) {
- p = (byte) hllc.p;
- registerSum = 0;
- zeroBuckets = 0;
-
- byte[] registers = hllc.registers;
- for (int i = 0; i < hllc.m; i++) {
- if (registers[i] == 0) {
- registerSum++;
- zeroBuckets++;
- } else {
- registerSum += 1.0 / (1L << registers[i]);
- }
- }
- }
-
- public long getCountEstimate() {
- int m = 1 << p;
- double alpha = 0.7213 / (1 + 1.079 / m);
- double estimate = alpha * m * m / registerSum;
-
- // small cardinality adjustment
- if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
- estimate = m * Math.log(m * 1.0 / zeroBuckets);
- } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
- estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
- }
-
- return Math.round(estimate);
- }
- }
-
- // ============================================================================
-
- public void writeRegisters(final ByteBuffer out) throws IOException {
-
- final int indexLen = getRegisterIndexSize();
- int size = size();
-
- // decide output scheme -- map (3*size bytes) or array (2^p bytes)
- byte scheme;
- if (5 + (indexLen + 1) * size < m) // 5 is max len of vint
- scheme = 0; // map
- else
- scheme = 1; // array
- out.put(scheme);
-
- if (scheme == 0) { // map scheme
- BytesUtil.writeVInt(size, out);
- if (singleBucket == -1) {
- // no non-zero register
- } else if (singleBucket >= 0) {
- writeUnsigned(singleBucket, indexLen, out);
- out.put(registers[singleBucket]);
- } else {
- for (int i = 0; i < m; i++) {
- if (registers[i] > 0) {
- writeUnsigned(i, indexLen, out);
- out.put(registers[i]);
- }
- }
- }
- } else if (scheme == 1) { // array scheme
- out.put(registers);
- } else
- throw new IllegalStateException();
- }
-
- public void readRegisters(ByteBuffer in) throws IOException {
- byte scheme = in.get();
-
- if (scheme == 0) { // map scheme
- clear();
- int size = BytesUtil.readVInt(in);
- if (size > m)
- throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
- int indexLen = getRegisterIndexSize();
- int key = 0;
- for (int i = 0; i < size; i++) {
- key = readUnsigned(in, indexLen);
- registers[key] = in.get();
- }
-
- if (size == 0)
- singleBucket = -1;
- else if (size == 1)
- singleBucket = key;
- else
- singleBucket = Integer.MIN_VALUE;
-
- } else if (scheme == 1) { // array scheme
- in.get(registers);
- singleBucket = Integer.MIN_VALUE;
- } else
- throw new IllegalStateException();
- }
-
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
- int len;
-
- byte scheme = in.get();
- if (scheme == 0) { // map scheme
- int size = BytesUtil.readVInt(in);
- int indexLen = getRegisterIndexSize();
- len = in.position() - mark + (indexLen + 1) * size;
- } else {
- len = in.position() - mark + m;
- }
-
- in.position(mark);
- return len;
- }
-
- public int maxLength() {
- return 1 + m;
- }
-
- /*public void writeRegistersArray(final ByteBuffer out) {
- out.put(this.registers);
- }
-
- public void readRegistersArray(ByteBuffer in) {
- in.get(registers, 0, m);
- singleBucket = Integer.MIN_VALUE;
- }*/
-
- private int getRegisterIndexSize() {
- return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
- result = prime * result + p;
- result = prime * result + Arrays.hashCode(registers);
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- HyperLogLogPlusCounterOld other = (HyperLogLogPlusCounterOld) obj;
- if (hashFunc == null) {
- if (other.hashFunc != null)
- return false;
- } else if (!hashFunc.equals(other.hashFunc))
- return false;
- if (p != other.p)
- return false;
- if (!Arrays.equals(registers, other.registers))
- return false;
- return true;
- }
-
- @Override
- public int compareTo(HyperLogLogPlusCounterOld o) {
- if (o == null)
- return 1;
-
- long e1 = this.getCountEstimate();
- long e2 = o.getCountEstimate();
-
- if (e1 == e2)
- return 0;
- else if (e1 > e2)
- return 1;
- else
- return -1;
- }
-
- public static void main(String[] args) throws IOException {
- dumpErrorRates();
- }
-
- static void dumpErrorRates() {
- for (int p = 10; p <= 18; p++) {
- double rate = new HyperLogLogPlusCounterOld(p).getErrorRate();
- double er = Math.round(rate * 10000) / 100D;
- double er2 = Math.round(rate * 2 * 10000) / 100D;
- double er3 = Math.round(rate * 3 * 10000) / 100D;
- long size = Math.round(Math.pow(2, p));
- System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
- }
- }
-
- /**
- *
- * @param num
- * @param size
- * @param out
- */
- public static void writeUnsigned(int num, int size, ByteBuffer out) {
- for (int i = 0; i < size; i++) {
- out.put((byte) num);
- num >>>= 8;
- }
- }
-
- public static int readUnsigned(ByteBuffer in, int size) {
- int integer = 0;
- int mask = 0xff;
- int shift = 0;
- for (int i = 0; i < size; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
- }
- return integer;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/Register.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/Register.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/Register.java
index 79c4bba..a6ef94f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/Register.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/Register.java
@@ -24,7 +24,7 @@ public interface Register {
void set(int pos, byte value);
- Byte get(int pos);
+ byte get(int pos);
void merge(Register another);
@@ -32,6 +32,4 @@ public interface Register {
int getSize();
- int getHashCode();
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
index d241e81..d6bb024 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
@@ -27,12 +27,9 @@ import java.util.TreeMap;
*/
public class SparseRegister implements Register {
- private int overThreshold;
-
private Map<Integer, Byte> sparseRegister = new TreeMap<>();
- public SparseRegister(int overThreshold) {
- this.overThreshold = overThreshold;
+ public SparseRegister() {
}
public DenseRegister toDense(int p) {
@@ -49,8 +46,9 @@ public class SparseRegister implements Register {
}
@Override
- public Byte get(int pos) {
- return sparseRegister.get(pos);
+ public byte get(int pos) {
+ Byte b = sparseRegister.get(pos);
+ return b == null ? 0 : b;
}
@Override
@@ -58,8 +56,8 @@ public class SparseRegister implements Register {
assert another instanceof SparseRegister;
SparseRegister sr = (SparseRegister) another;
for (Map.Entry<Integer, Byte> entry : sr.sparseRegister.entrySet()) {
- Byte v = sparseRegister.get(entry.getKey());
- if (v == null || entry.getValue() > v)
+ byte v = get(entry.getKey());
+ if (entry.getValue() > v)
sparseRegister.put(entry.getKey(), entry.getValue());
}
}
@@ -75,20 +73,28 @@ public class SparseRegister implements Register {
}
@Override
- public int getHashCode() {
+ public int hashCode() {
final int prime = 31;
int result = 1;
- for (Map.Entry<Integer, Byte> entry : sparseRegister.entrySet()) {
- result = prime * result + entry.getKey();
- result = prime * result + entry.getValue();
- }
+ result = prime * result + ((sparseRegister == null) ? 0 : sparseRegister.hashCode());
return result;
}
- public boolean isOverThreshold() {
- if (this.sparseRegister.size() > overThreshold)
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
return true;
- return false;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ SparseRegister other = (SparseRegister) obj;
+ if (sparseRegister == null) {
+ if (other.sparseRegister != null)
+ return false;
+ } else if (!sparseRegister.equals(other.sparseRegister))
+ return false;
+ return true;
}
public Collection<Map.Entry<Integer, Byte>> getAllValue() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
index 103e721..0f22610 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -26,7 +26,7 @@ import org.apache.kylin.measure.bitmap.BitmapAggregator;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.measure.hllc.HLLCAggregator;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.datatype.LongMutable;
@@ -94,7 +94,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
@Test
public void testAggregatorEstimate() {
HLLCAggregator hllcAggregator = new HLLCAggregator(14);
- hllcAggregator.aggregate(new HyperLogLogPlusCounterNew(14));
+ hllcAggregator.aggregate(new HLLCounter(14));
BitmapAggregator bitmapAggregator = new BitmapAggregator();
BitmapCounter bitmapCounter = new BitmapCounter();
[40/50] [abbrv] kylin git commit: KYLIN-2290 minor improvements on
limit
Posted by li...@apache.org.
KYLIN-2290 minor improvements on limit
KYLIN-2290 fix compile
KYLIN-2290 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f9bd4a9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f9bd4a9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f9bd4a9
Branch: refs/heads/master-cdh5.7
Commit: 6f9bd4a9998cff24c97bce0e6d86282f48fef388
Parents: 699a88e
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Dec 16 16:28:18 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Dec 19 14:16:00 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 1 +
.../kylin/cube/CubeCapabilityChecker.java | 5 ++
.../kylin/gridtable/GTScanRequestBuilder.java | 2 +-
.../kylin/metadata/realization/SQLDigest.java | 4 +-
.../apache/kylin/storage/StorageContext.java | 74 +++++++-------------
.../storage/gtrecord/CubeScanRangePlanner.java | 9 +--
.../gtrecord/GTCubeStorageQueryBase.java | 1 -
.../gtrecord/SequentialCubeTupleIterator.java | 6 +-
.../apache/kylin/query/ITKylinQueryTest.java | 1 -
.../org/apache/kylin/query/KylinTestBase.java | 7 +-
.../kylin/storage/hbase/ITStorageTest.java | 2 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 4 ++
.../apache/kylin/query/relnode/OLAPContext.java | 6 +-
.../kylin/query/relnode/OLAPFilterRel.java | 2 +-
.../kylin/query/relnode/OLAPLimitRel.java | 6 +-
.../coprocessor/endpoint/CubeVisitService.java | 7 +-
16 files changed, 60 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 610c2af..5153562 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -778,6 +778,7 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.valueOf(getOptional("kylin.query.skip-empty-segments", "true"));
}
+ @Deprecated//Limit is good even it's large. This config is meaning less since we already have scan threshold
public int getStoragePushDownLimitMax() {
return Integer.parseInt(getOptional("kylin.query.max-limit-pushdown", "10000"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 38faed9..c45144b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -120,6 +120,11 @@ public class CubeCapabilityChecker {
return result;
}
+ if (digest.limitPrecedesAggr) {
+ logger.info("Exclude cube " + cube.getName() + " because there's limit preceding aggregation");
+ return result;
+ }
+
if (digest.isRawQuery && rootFactTable.equals(digest.factTable)) {
result.influences.add(new CapabilityInfluence() {
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
index f542de1..bcec1f4 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequestBuilder.java
@@ -36,7 +36,7 @@ public class GTScanRequestBuilder {
private boolean allowStorageAggregation = true;
private double aggCacheMemThreshold = 0;
private int storageScanRowNumThreshold = Integer.MAX_VALUE;// storage should terminate itself when $storageScanRowNumThreshold cuboid rows are scanned, and throw exception.
- private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit working when $toragePushDownLimit aggregated rows are produced.
+ private int storagePushDownLimit = Integer.MAX_VALUE;// storage can quit scanning safely when $toragePushDownLimit aggregated rows are produced.
private long startTime = -1;
private long timeout = -1;
private String storageBehavior = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
index 83fc05c..36f303b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/SQLDigest.java
@@ -59,10 +59,11 @@ public class SQLDigest {
public List<TblColRef> sortColumns;
public List<OrderEnum> sortOrders;
public boolean isRawQuery;
+ public boolean limitPrecedesAggr;
public SQLDigest(String factTable, TupleFilter filter, List<JoinDesc> joinDescs, Set<TblColRef> allColumns, //
List<TblColRef> groupbyColumns, Set<TblColRef> subqueryJoinParticipants, Set<TblColRef> filterColumns, Set<TblColRef> metricColumns, //
- List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef> sortColumns, List<OrderEnum> sortOrders) {
+ List<FunctionDesc> aggregations, List<SQLCall> aggrSqlCalls, List<TblColRef> sortColumns, List<OrderEnum> sortOrders, boolean limitPrecedesAggr) {
this.factTable = factTable;
this.filter = filter;
this.joinDescs = joinDescs;
@@ -76,6 +77,7 @@ public class SQLDigest {
this.sortColumns = sortColumns;
this.sortOrders = sortOrders;
this.isRawQuery = isRawQuery();
+ this.limitPrecedesAggr = limitPrecedesAggr;
}
private boolean isRawQuery() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
index bc43a87..9ef59fd 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java
@@ -34,43 +34,29 @@ import com.google.common.collect.Range;
public class StorageContext {
private static final Logger logger = LoggerFactory.getLogger(StorageContext.class);
- public static final int DEFAULT_THRESHOLD = 1000000;
-
private String connUrl;
private int threshold;
- private int limit;
- private int offset;
- private int finalPushDownLimit;
- private boolean hasSort;
- private boolean acceptPartialResult;
-
- private boolean exactAggregation;
- private boolean needStorageAggregation;
- private boolean enableLimit;
- private boolean enableCoprocessor;
-
- private AtomicLong totalScanCount;
+ private int limit = Integer.MAX_VALUE;
+ private int offset = 0;
+ private int finalPushDownLimit = Integer.MAX_VALUE;
+ private boolean hasSort = false;
+ private boolean acceptPartialResult = false;
+
+ private boolean exactAggregation = false;
+ private boolean needStorageAggregation = false;
+ private boolean limitEnabled = false;
+ private boolean enableCoprocessor = false;
+
+ private AtomicLong totalScanCount = new AtomicLong();
private Cuboid cuboid;
- private boolean partialResultReturned;
-
- private Range<Long> reusedPeriod;
+ private boolean partialResultReturned = false;
public StorageContext() {
- this.threshold = DEFAULT_THRESHOLD;
- this.limit = DEFAULT_THRESHOLD;
- this.totalScanCount = new AtomicLong();
- this.cuboid = null;
- this.hasSort = false;
-
- this.exactAggregation = false;
- this.enableLimit = false;
- this.enableCoprocessor = false;
-
- this.acceptPartialResult = false;
- this.partialResultReturned = false;
- this.finalPushDownLimit = Integer.MAX_VALUE;
+ this.threshold = KylinConfig.getInstanceFromEnv().getScanThreshold();
}
+ private Range<Long> reusedPeriod;
+
public String getConnUrl() {
return connUrl;
}
@@ -92,11 +78,10 @@ public class StorageContext {
}
public void setLimit(int l) {
- if (l > limit) {
- //cases like : select price from (select * from kylin_sales limit 10) limit 5000
- logger.info("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, limit);
+ if (limit != Integer.MAX_VALUE) {
+ logger.warn("Setting limit to {} but in current olap context, the limit is already {}, won't apply", l, limit);
} else {
- this.limit = l;
+ limit = l;
}
}
@@ -109,15 +94,11 @@ public class StorageContext {
}
public void enableLimit() {
- this.enableLimit = true;
+ this.limitEnabled = true;
}
public boolean isLimitEnabled() {
- return this.enableLimit;
- }
-
- private int getStoragePushDownLimit() {
- return this.isLimitEnabled() ? this.getOffset() + this.getLimit() : Integer.MAX_VALUE;
+ return this.limitEnabled;
}
public int getFinalPushDownLimit() {
@@ -126,19 +107,16 @@ public class StorageContext {
public void setFinalPushDownLimit(IRealization realization) {
- //decide the final limit push down
- int tempPushDownLimit = this.getStoragePushDownLimit();
- if (tempPushDownLimit == Integer.MAX_VALUE) {
+ if (this.getLimit() == Integer.MAX_VALUE) {
return;
}
- int pushDownLimitMax = KylinConfig.getInstanceFromEnv().getStoragePushDownLimitMax();
+ int tempPushDownLimit = this.getOffset() + this.getLimit();
+
if (!realization.supportsLimitPushDown()) {
- logger.info("Not enabling limit push down because cube storage type not supported");
- } else if (tempPushDownLimit > pushDownLimitMax) {
- logger.info("Not enabling limit push down because the limit(including offset) {} is larger than kylin.query.max-limit-pushdown {}", //
- tempPushDownLimit, pushDownLimitMax);
+ logger.warn("Not enabling limit push down because cube storage type not supported");
} else {
+ this.limitEnabled = true;
this.finalPushDownLimit = tempPushDownLimit;
logger.info("Enable limit: " + tempPushDownLimit);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
index 8d5a3d4..b05a629 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeScanRangePlanner.java
@@ -148,15 +148,10 @@ public class CubeScanRangePlanner extends ScanRangePlannerBase {
GTScanRequest scanRequest;
List<GTScanRange> scanRanges = this.planScanRanges();
if (scanRanges != null && scanRanges.size() != 0) {
- GTScanRequestBuilder builder = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).//
+ scanRequest = new GTScanRequestBuilder().setInfo(gtInfo).setRanges(scanRanges).setDimensions(gtDimensions).//
setAggrGroupBy(gtAggrGroups).setAggrMetrics(gtAggrMetrics).setAggrMetricsFuncs(gtAggrFuncs).setFilterPushDown(gtFilter).//
setAllowStorageAggregation(context.isNeedStorageAggregation()).setAggCacheMemThreshold(cubeSegment.getCubeInstance().getConfig().getQueryCoprocessorMemGB()).//
- setStorageScanRowNumThreshold(context.getThreshold());
-
- if (context.getFinalPushDownLimit() != Integer.MAX_VALUE)
- builder.setStoragePushDownLimit(context.getFinalPushDownLimit());
-
- scanRequest = builder.createGTScanRequest();
+ setStoragePushDownLimit(context.getFinalPushDownLimit()).setStorageScanRowNumThreshold(context.getThreshold()).createGTScanRequest();
} else {
scanRequest = null;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 31573d0..7b0cbb9 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -383,7 +383,6 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
if (possible) {
- context.enableLimit();
context.setFinalPushDownLimit(cubeInstance);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index bef0e88..3a64de7 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -46,7 +46,6 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
protected List<CubeSegmentScanner> scanners;
protected List<SegmentCubeTupleIterator> segmentCubeTupleIterators;
protected Iterator<ITuple> tupleIterator;
- protected final int storagePushDownLimit;
protected StorageContext context;
private int scanCount;
@@ -62,8 +61,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDimensions, selectedMetrics, returnTupleInfo, context));
}
- this.storagePushDownLimit = context.getFinalPushDownLimit();
- if (storagePushDownLimit == Integer.MAX_VALUE) {
+ if (!context.isLimitEnabled()) {
//normal case
tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
} else {
@@ -75,7 +73,7 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
return input;
}
});
- tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, storagePushDownLimit, segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator();
+ tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), segmentCubeTupleIterators.get(0).getCubeTupleConverter().getTupleDimensionComparator()).getIterator();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 7f04bbb..c2aeabd 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -331,7 +331,6 @@ public class ITKylinQueryTest extends KylinTestBase {
List<File> sqlFiles = getFilesFromFolder(new File(getQueryFolderPrefix() + "src/test/resources/query/sql_limit"), ".sql");
for (File sqlFile : sqlFiles) {
runSQL(sqlFile, false, false);
- assertTrue(checkLimitEnabled());
assertTrue(checkFinalPushDownLimit());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index ddb996c..47b9e37 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -653,14 +653,9 @@ public class KylinTestBase {
}
- protected boolean checkLimitEnabled() {
- OLAPContext context = getFirstOLAPContext();
- return (context.storageContext.isLimitEnabled());
- }
-
protected boolean checkFinalPushDownLimit() {
OLAPContext context = getFirstOLAPContext();
- return (context.storageContext.getFinalPushDownLimit() != Integer.MAX_VALUE);
+ return context.storageContext.isLimitEnabled();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
index db9d133..aea8bef 100644
--- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java
@@ -148,7 +148,7 @@ public class ITStorageTest extends HBaseMetadataTestCase {
int count = 0;
ITupleIterator iterator = null;
try {
- SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>());
+ SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Sets.<TblColRef> newHashSet(), Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations, Collections.<SQLCall> emptyList(), new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false);
iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations));
while (iterator.hasNext()) {
ITuple tuple = iterator.next();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index f5fa74d..c457545 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -156,6 +156,10 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
this.context.groupByColumns.addAll(this.groups);
this.context.aggregations.addAll(this.aggregations);
this.context.afterAggregate = true;
+
+ if (this.context.afterLimit) {
+ this.context.limitPrecedesAggr = true;
+ }
} else {
for (AggregateCall aggCall : aggCalls) {
// check if supported by kylin
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 8278fb0..dde98a6 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -112,7 +112,9 @@ public class OLAPContext {
public Set<OLAPTableScan> allTableScans = new HashSet<>();
public TupleInfo returnTupleInfo = null;
public boolean afterAggregate = false;
- public boolean afterSkippedFilter = false;
+ public boolean afterHavingClauseFilter = false;
+ public boolean afterLimit = false;
+ public boolean limitPrecedesAggr = false;
public boolean afterJoin = false;
public boolean hasJoin = false;
@@ -148,7 +150,7 @@ public class OLAPContext {
public SQLDigest getSQLDigest() {
if (sqlDigest == null)
- sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortColumns, sortOrders);
+ sqlDigest = new SQLDigest(firstTableScan.getTableName(), filter, joins, allColumns, groupByColumns, subqueryJoinParticipants, filterColumns, metricsColumns, aggregations, aggrSqlCalls, sortColumns, sortOrders, limitPrecedesAggr);
return sqlDigest;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
index 8023df4..151131e 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java
@@ -306,7 +306,7 @@ public class OLAPFilterRel extends Filter implements OLAPRel {
if (!context.afterAggregate) {
translateFilter(context);
} else {
- context.afterSkippedFilter = true;//having clause is skipped
+ context.afterHavingClauseFilter = true;//having clause is skipped
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index f0af863..8179807 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -77,7 +77,9 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
this.columnRowType = buildColumnRowType();
this.context = implementor.getContext();
- if (!context.afterSkippedFilter) {
+ // ignore limit after having clause
+ // ignore limit after another limit, e.g. select A, count(*) from (select A,B from fact group by A,B limit 100) limit 10
+ if (!context.afterHavingClauseFilter && !context.afterLimit) {
Number limitValue = (Number) (((RexLiteral) localFetch).getValue());
int limit = limitValue.intValue();
this.context.storageContext.setLimit(limit);
@@ -87,6 +89,8 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
int offset = offsetValue.intValue();
this.context.storageContext.setOffset(offset);
}
+
+ context.afterLimit = true;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f9bd4a9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index da9c932..38efecc 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -254,14 +254,15 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
@Override
public boolean hasNext() {
+ counter++;
+
if (counter > scanReq.getStorageScanRowNumThreshold()) {
- throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter);
+ throw new GTScanExceedThresholdException("Exceed scan threshold at " + counter + ", consider increasing kylin.query.memory-budget-bytes and kylin.query.scan-threshold");
}
if (counter % (10 * GTScanRequest.terminateCheckInterval) == 1) {
- logger.info("Scanned " + counter + " rows from HBase.");
+ logger.info("scanning " + counter + "th row from HBase.");
}
- counter++;
return allCellLists.hasNext();
}
[33/50] [abbrv] kylin git commit: KYLIN-2283 A new data gen tool for
CI
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv b/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv
deleted file mode 100644
index 07756c6..0000000
--- a/examples/test_case_data/localmeta/data/flatten_data_for_ii.csv
+++ /dev/null
@@ -1,402 +0,0 @@
-Auction,0,12,48027,0,10,Coins,2013-07-16 08:23:09,USER_Y,Coins & Paper Money,Paper Money: World,Asia,0,Ebay,USER_S,12,Merchant,2013-03-31,2013-03-31,10000000157,2013-03-31,48027,10000001,184.21,1
-Others,0,5,164261,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Earrings,0,Ebay,USER_S,5,Occasional,2013-11-12,2013-11-10,10000000158,2013-11-12,164261,10000002,172.03,1
-Auction,15,14,82494,15,3,Books,2010-09-13 07:05:54,USER_Y,BookMagazines,NULL,Comic Books,15,Australia,USER_S,14,Regular,2013-04-06,2013-03-31,10000000161,2013-04-06,82494,10000003,66.6,1
-Auction,15,12,66767,15,16,Home & Garden,2011-06-14 07:48:40,USER_Y,Home & Garden,NULL,Dogs,15,Australia,USER_S,12,Merchant,2013-05-17,2013-05-12,10000000137,2013-05-17,66767,10000004,92.98,1
-FP-GTC,0,5,152801,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,NULL,Earrings,0,Ebay,USER_S,5,Occasional,2013-05-20,2013-05-19,10000000147,2013-05-20,152801,10000005,132.33,1
-FP-GTC,0,13,43398,0,36,Home & Living,2013-02-20 23:11:43,USER_Y,Home & Garden,NULL,Cheese & Crackers,0,Ebay,USER_S,13,Entrepreneur,2013-06-16,2013-06-16,10000000155,2013-06-16,43398,10000006,7.12,1
-Auction,0,14,95173,0,37,Health & Beauty,2008-06-03 13:36:00,USER_Y,Health & Beauty,Bath & Body,Bath Sets & Kits,0,Ebay,USER_S,14,Regular,2013-06-14,2013-06-09,10000000170,2013-06-14,95173,10000007,204.28,1
-Auction,15,13,158666,15,28,Toys,2012-09-11 20:27:22,USER_Y,ToyHobbies,Action Figures,Anime & Manga,15,Australia,USER_S,13,Entrepreneur,2013-03-22,2013-03-17,10000000173,2013-03-22,158666,10000008,35.72,1
-Auction,0,12,12688,0,57,Sothebys,2008-06-03 13:36:00,USER_Y,eBay Premier,Books & Manuscripts,Books: Other,0,Ebay,USER_S,12,Merchant,2013-03-10,2013-03-10,10000000178,2013-03-10,12688,10000009,4.13,1
-FP-GTC,15,5,103324,15,9,Clothing & Accessories,2012-05-01 08:54:26,USER_Y,ClothinShoeAccessories,Women's Shoes,Mixed Items,15,Australia,USER_S,5,Occasional,2013-11-01,2013-10-27,10000000163,2013-11-01,103324,10000010,27.48,1
-FP-GTC,15,14,108782,15,8,Auto Parts,2013-09-10 16:52:46,USER_Y,Vehicle Parts & Accessories,CaTruck Parts,Car Care & Cleaning,15,Australia,USER_S,14,Regular,2013-06-16,2013-06-16,10000000166,2013-06-16,108782,10000011,9.26,1
-Auction,0,12,80287,0,12,Computers,2012-05-01 08:54:26,USER_Y,Computers/Tablets & Networking,Software,Office & Business,0,Ebay,USER_S,12,Merchant,2013-09-12,2013-09-08,10000000167,2013-09-12,80287,10000012,3.18,1
-Others,100,13,140746,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,Vintage Car & Truck Parts,100,Ebaymotors,USER_S,13,Entrepreneur,2013-09-28,2013-09-22,10000000086,2013-09-28,140746,10000013,3.18,1
-ABIN,0,14,87118,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Outdoor Sports,Paintball,0,Ebay,USER_S,14,Regular,2013-06-15,2013-06-09,10000000110,2013-06-15,87118,10000014,377.94,1
-Auction,0,12,25147,0,25,Sports Memorabilia,2010-02-08 10:51:44,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Baseball-MLB,0,Ebay,USER_S,12,Merchant,2013-03-14,2013-03-10,10000000113,2013-03-14,25147,10000015,146.33,1
-FP-GTC,15,5,170302,15,15,Crafts,2010-09-13 07:13:34,USER_Y,Crafts,Embroidery,Design CDs,15,Australia,USER_S,5,Occasional,2013-09-01,2013-09-01,10000000069,2013-09-01,170302,10000016,51.23,1
-FP-non GTC,0,13,53064,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,Heavy Equipment,Antique & Vintage Farm Equip,0,Ebay,USER_S,13,Entrepreneur,2013-05-29,2013-05-26,10000000079,2013-05-29,53064,10000017,72.65,1
-Auction,0,13,132939,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Other,0,Ebay,USER_S,13,Entrepreneur,2013-05-31,2013-05-26,10000000080,2013-05-31,132939,10000018,66.6,1
-Auction,15,12,113593,15,40,Cell Phones & Portable Electro,2012-09-11 20:26:04,USER_Y,Phones,Mobile Phones,Mobile Phones,15,Australia,USER_S,12,Merchant,2013-03-18,2013-03-17,10000000130,2013-03-18,113593,10000019,9.26,1
-Auction,100,14,34273,100,8,Auto - Parts,2008-06-03 13:36:00,USER_Y,eBay Motors,Parts & Accessories,Motorcycle,100,Ebaymotors,USER_S,14,Regular,2013-07-19,2013-07-14,10000000268,2013-07-19,34273,10000020,583.44,1
-FP-GTC,15,14,106340,15,16,Home & Garden,2012-09-11 20:26:04,USER_Y,Home & Garden,Gardening,Hand Tools,15,Australia,USER_S,14,Regular,2013-06-23,2013-06-23,10000000132,2013-06-23,106340,10000021,638.72,1
-FP-GTC,15,14,150265,15,2,Baby,2011-04-05 08:25:25,USER_Y,Baby,Baby Clothing,Boys,15,Australia,USER_S,14,Regular,2013-05-20,2013-05-19,10000000115,2013-05-20,150265,10000022,4.54,1
-FP-GTC,0,12,24760,0,25,Sports Memorabilia,2010-02-08 10:51:44,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Hockey-NHL,0,Ebay,USER_S,12,Merchant,2013-05-17,2013-05-12,10000000117,2013-05-17,24760,10000023,319.79,1
-Auction,0,12,37831,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Advertising,Merchandise & Memorabilia,0,Ebay,USER_S,12,Merchant,2013-03-11,2013-03-10,10000000118,2013-03-11,37831,10000024,20.35,1
-FP-non GTC,3,5,1120,3,3,BookComics & Magazines,2008-06-03 13:36:00,USER_Y,Books,First Editions,Other,3,\u82f1\u56fd,USER_S,5,Occasional,2013-01-30,2013-01-27,10000000223,2013-01-30,1120,10000025,223.63,1
-FP-GTC,100,13,43972,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,ATV Parts,100,Ebaymotors,USER_S,13,Entrepreneur,2013-01-26,2013-01-20,10000000224,2013-01-26,43972,10000026,204.28,1
-Auction,15,14,166013,15,12,Computers,2012-05-01 08:54:26,USER_Y,Computers,Computer Components & Parts,Video Capture & TV Tuner Cards,15,Australia,USER_S,14,Regular,2013-03-22,2013-03-17,10000000243,2013-03-22,166013,10000027,5.48,1
-Auction,15,14,15568,15,2,Baby,2011-04-05 08:25:25,USER_Y,Baby,Baby Clothing,Unisex,15,Australia,USER_S,14,Regular,2013-07-23,2013-07-21,10000000217,2013-07-23,15568,10000028,27.48,1
-FP-GTC,15,5,103178,15,9,Clothing & Accessories,2012-05-01 08:54:26,USER_Y,ClothinShoeAccessories,Women's Bags,Women's Bags,15,Australia,USER_S,5,Occasional,2013-07-27,2013-07-21,10000000218,2013-07-27,103178,10000029,21.72,1
-ABIN,0,12,2023,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Team Sports,Basketball,0,Ebay,USER_S,12,Merchant,2013-10-29,2013-10-27,10000000221,2013-10-29,2023,10000030,3.18,1
-FP-GTC,0,11,94847,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,Vehicle Electronics & GPS,Car Video,0,Ebay,USER_S,11,Large Merchant,2013-10-08,2013-10-06,10000000256,2013-10-08,94847,10000031,491.32,1
-Auction,0,14,15868,0,23,Real Estate,2008-06-03 13:36:00,USER_Y,Real Estate,Land,Land,0,Ebay,USER_S,14,Regular,2013-04-26,2013-04-21,10000000257,2013-04-26,15868,10000032,448.8,1
-Auction,0,13,32876,0,36,Home & Living,2012-09-11 20:26:04,USER_Y,Home & Garden,Home Improvement,Plumbing & Fixtures,0,Ebay,USER_S,13,Entrepreneur,2013-01-01,2013-01-01,10000000263,2013-01-01,32876,10000033,415.73,1
-Auction,0,13,62179,0,9,Apparel,2008-06-03 13:36:00,USER_Y,ClothinShoes & Accessories,Women's Clothing,Athletic Apparel,0,Ebay,USER_S,13,Entrepreneur,2013-01-15,2013-01-13,10000000245,2013-01-15,62179,10000034,377.94,1
-FP-GTC,15,14,33038,15,30,Musical Instruments,2010-06-14 07:48:40,USER_Y,Musical Instruments,Instruments,Guitars (Electric),15,Australia,USER_S,14,Regular,2013-05-27,2013-05-26,10000000248,2013-05-27,33038,10000035,146.33,1
-FP-GTC,0,5,156614,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,0,Ebay,USER_S,5,Occasional,2013-11-11,2013-11-10,10000000254,2013-11-11,156614,10000036,7.12,1
-Auction,0,13,106246,0,37,Health & Beauty,2013-04-28 20:37:19,USER_Y,Health & Beauty,Hair Care & Styling,Shampoo & Conditioning,0,Ebay,USER_S,13,Entrepreneur,2013-03-08,2013-03-03,10000000192,2013-03-08,106246,10000037,42.99,1
-Auction,0,13,20865,0,9,Apparel,2008-06-03 13:36:00,USER_Y,ClothinShoes & Accessories,Men's Clothing,Athletic Apparel,0,Ebay,USER_S,13,Entrepreneur,2013-03-25,2013-03-24,10000000196,2013-03-25,20865,10000038,12.85,1
-FP-GTC,0,13,15115,0,32,Video Games,2012-06-19 21:15:09,USER_Y,Video Games & Consoles,Video Games,Video Games,0,Ebay,USER_S,13,Entrepreneur,2013-08-20,2013-08-18,10000000203,2013-08-20,15115,10000039,55.89,1
-FP-GTC,0,14,3838,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Charms & Charm Bracelets,0,Ebay,USER_S,14,Regular,2013-05-17,2013-05-12,10000000179,2013-05-17,3838,10000040,73.26,1
-Auction,0,11,759,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,11,Large Merchant,2013-06-05,2013-06-02,10000000183,2013-06-05,759,10000041,112.56,1
-FP-non GTC,0,11,61323,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,TVideo & Home Audio,TVideo & Audio Accessories,0,Ebay,USER_S,11,Large Merchant,2013-10-08,2013-10-06,10000000185,2013-10-08,61323,10000042,3.49,1
-Auction,0,13,121153,0,2,Baby,2012-09-11 20:26:04,USER_Y,Baby,Nursery Decor,Night Lights,0,Ebay,USER_S,13,Entrepreneur,2013-08-14,2013-08-11,10000000211,2013-08-14,121153,10000043,184.21,1
-Auction,0,13,88750,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,Vehicle Electronics & GPS,Radar & Laser Detectors,0,Ebay,USER_S,13,Entrepreneur,2013-08-14,2013-08-11,10000000213,2013-08-14,88750,10000044,157.14,1
-FP-GTC,15,14,161567,15,12,Computers,2012-05-01 08:54:26,USER_Y,Computers,Laptop & Desktop Accessories,Laptop Batteries,15,Australia,USER_S,14,Regular,2013-05-17,2013-05-12,10000000214,2013-05-17,161567,10000045,72.65,1
-FP-GTC,15,14,113802,15,99,Everything Else,2008-06-03 13:36:00,USER_Y,Lots More...,Metaphysical,Herbs,15,Australia,USER_S,14,Regular,2013-08-09,2013-08-04,10000000204,2013-08-09,113802,10000046,51.23,1
-FP-non GTC,15,14,15808,15,9,Clothing & Accessories,2012-09-11 20:26:04,USER_Y,ClothinShoeAccessories,Women's Clothing,Tops & Blouses,15,Australia,USER_S,14,Regular,2013-06-30,2013-06-30,10000000208,2013-06-30,15808,10000047,15.85,1
-FP-GTC,3,13,174053,3,8,Auto - Parts,2013-04-28 20:37:19,USER_Y,Vehicle Parts & Accessories,Car Parts,External & Body Parts,3,\u82f1\u56fd,USER_S,13,Entrepreneur,2013-06-03,2013-06-02,10000000209,2013-06-03,174053,10000048,7.12,1
-Auction,0,14,2635,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Toy Soldiers,1970-Now,0,Ebay,USER_S,14,Regular,2013-12-31,2013-12-29,10000000042,2013-12-31,2635,10000049,12.04,1
-Auction,3,13,1161,3,18,DVDFilm & TV,2008-06-03 13:36:00,USER_Y,DVFilm & TV,Other Formats,Videos: NTSC (US),3,\u82f1\u56fd,USER_S,13,Entrepreneur,2013-12-25,2013-12-22,10000000044,2013-12-25,1161,10000050,73.26,1
-FP-non GTC,0,5,64076,0,12,Computers,2012-05-01 08:54:26,USER_Y,Computers/Tablets & Networking,Enterprise NetworkinServers,Switches & Hubs,0,Ebay,USER_S,5,Occasional,2013-03-28,2013-03-24,10000000022,2013-03-28,64076,10000051,184.21,1
-FP-GTC,15,13,33977,15,15,Crafts,2010-06-14 07:56:25,USER_Y,Crafts,Scrapbooking,Albums,15,Australia,USER_S,13,Entrepreneur,2013-01-30,2013-01-27,10000000023,2013-01-30,33977,10000052,172.03,1
-FP-GTC,0,14,31673,0,25,Sports Memorabilia,2010-02-08 10:48:39,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Racing-NASCAR,0,Ebay,USER_S,14,Regular,2013-12-05,2013-12-01,10000000047,2013-12-05,31673,10000053,122.78,1
-Auction,3,14,174106,3,8,Auto - Parts,2013-04-28 20:37:19,USER_Y,Vehicle Parts & Accessories,Car Parts,Transmission & Drivetrain,3,\u82f1\u56fd,USER_S,14,Regular,2013-10-08,2013-10-06,10000000056,2013-10-08,174106,10000054,92.98,1
-Auction,0,13,26249,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Printing & Graphic Arts,Commercial Printing Presses,0,Ebay,USER_S,13,Entrepreneur,2013-12-27,2013-12-22,10000000062,2013-12-27,26249,10000055,12.19,1
-FP-GTC,0,5,159184,0,24,Sporting Goods,2013-09-10 16:52:46,USER_Y,Sporting Goods,Winter Sports,Snowboarding,0,Ebay,USER_S,5,Occasional,2013-12-16,2013-12-15,10000000053,2013-12-16,159184,10000056,15.65,1
-FP-GTC,3,11,10058,3,27,Tickets,2013-09-10 16:52:46,USER_Y,Events Tickets,Other Tickets,Other Tickets,3,\u82f1\u56fd,USER_S,11,Large Merchant,2013-10-17,2013-10-13,10000000055,2013-10-17,10058,10000057,101.79,1
-ABIN,0,12,48904,0,9,Apparel,2012-05-01 08:54:26,USER_Y,ClothinShoes & Accessories,Vintage,Women's Vintage Clothing,0,Ebay,USER_S,12,Merchant,2013-11-17,2013-11-17,10000000020,2013-11-17,48904,10000058,7.12,1
-FP-non GTC,0,14,145970,0,28,Toys,2008-10-08 07:18:40,USER_Y,Toys & Hobbies,Models & Kits,Automotive,0,Ebay,USER_S,14,Regular,2013-09-18,2013-09-15,10000000007,2013-09-18,145970,10000059,12.85,1
-FP-GTC,0,13,963,0,9,Apparel,2012-05-01 08:54:26,USER_Y,ClothinShoes & Accessories,Vintage,Women's Vintage Shoes,0,Ebay,USER_S,13,Entrepreneur,2013-06-30,2013-06-30,10000000008,2013-06-30,963,10000060,12.19,1
-FP-GTC,3,13,118687,3,37,Health & Beauty,2013-04-28 20:37:19,USER_Y,Health & Beauty,Fragrances,Women's Fragrances,3,\u82f1\u56fd,USER_S,13,Entrepreneur,2013-10-12,2013-10-06,10000000002,2013-10-12,118687,10000061,92.98,1
-FP-GTC,0,14,20886,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,0,Ebay,USER_S,14,Regular,2013-08-20,2013-08-18,10000000003,2013-08-20,20886,10000062,42.99,1
-Auction,15,13,148324,15,31,Phones,2012-09-18 00:08:03,USER_XIANZHU,Phones,Mobile Accessories,CaseCoverSkins,15,Australia,USER_S,13,Entrepreneur,2013-08-29,2013-08-25,10000000010,2013-08-29,148324,10000063,1.88,1
-Auction,15,14,139255,15,17,JewelrGemWatches,2012-05-01 08:54:26,USER_Y,Jewellery & Watches,Fine Jewellery,Earrings,15,Australia,USER_S,14,Regular,2013-07-17,2013-07-14,10000000016,2013-07-17,139255,10000064,21.14,1
-FP-GTC,0,5,20213,0,11,Collectibles,2008-09-09 22:08:47,USER_Y,Collectibles,Postcards,US StateCities & Towns,0,Ebay,USER_S,5,Occasional,2013-07-23,2013-07-21,10000000017,2013-07-23,20213,10000065,21.14,1
-Auction,15,13,32996,15,42,Entertainment Memorabilia,2012-09-11 20:26:04,USER_Y,Movies,Television Memorabilia,Clippings,15,Australia,USER_S,13,Entrepreneur,2013-01-06,2013-01-06,10000000012,2013-01-06,32996,10000066,132.33,1
-FP-GTC,0,14,99985,0,11,Collectibles,2008-09-09 22:08:47,USER_Y,Collectibles,Trading Cards,Sci-FFantasy,0,Ebay,USER_S,14,Regular,2013-08-14,2013-08-11,10000000013,2013-08-14,99985,10000067,120.87,1
-Auction,3,14,67703,3,17,Jewellery & Watches,2012-06-19 21:15:09,USER_Y,Jewellery & Watches,Jewellery Boxes & Supplies,Jewellery Display,3,\u82f1\u56fd,USER_S,14,Regular,2013-08-10,2013-08-04,10000000067,2013-08-10,67703,10000068,120.87,1
-FP-non GTC,0,11,65,0,11,Collectibles,2013-02-20 23:11:43,USER_Y,Collectibles,Comics,Platinum Age (1897-1937),0,Ebay,USER_S,11,Large Merchant,2013-09-28,2013-09-22,10000000085,2013-09-28,65,10000069,9.26,1
-FP-non GTC,0,14,130,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Transportation,Railroadiana & Trains,0,Ebay,USER_S,14,Regular,2013-08-21,2013-08-18,10000000141,2013-08-21,130,10000070,16.26,1
-FP-GTC,0,14,164,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,CPUProcessors,0,Ebay,USER_S,14,Regular,2013-03-11,2013-03-10,10000000078,2013-03-11,164,10000071,157.14,1
-FP-GTC,0,11,216,0,25,Sports Memorabilia,2013-02-20 23:11:43,USER_Y,Sports MeCards & Fan Shop,Cards,Hockey,0,Ebay,USER_S,11,Large Merchant,2013-05-06,2013-05-05,10000000109,2013-05-06,216,10000072,1.88,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2013-05-17,2013-05-12,10000000096,2013-05-17,223,10000073,12.04,1
-FP-non GTC,0,14,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,14,Regular,2013-01-10,2013-01-06,10000000095,2013-01-10,223,10000074,189.23,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2013-05-05,2013-05-05,10000000098,2013-05-05,223,10000075,73.26,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2013-02-03,2013-02-03,10000000097,2013-02-03,223,10000076,4.13,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2013-11-26,2013-11-24,10000000099,2013-11-26,223,10000077,290.72,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2013-08-30,2013-08-25,10000000100,2013-08-30,223,10000078,265.56,1
-FP-GTC,15,5,279,15,3,Books,2013-09-10 16:52:46,USER_Y,BookMagazines,Children's Books,Children's Books,15,Australia,USER_S,5,Occasional,2013-04-26,2013-04-21,10000000126,2013-04-26,279,10000079,5.91,1
-Auction,0,5,314,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Other,0,Ebay,USER_S,5,Occasional,2013-06-30,2013-06-30,10000000252,2013-06-30,314,10000080,319.79,1
-Auction,211,5,314,211,9,ClothinShoes & Accessories,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Womens' Clothing,Other,211,Philippines,USER_X,5,Occasional,2013-06-30,2013-06-30,10000000052,2013-06-30,314,10000081,246,1
-Auction,211,5,314,211,9,ClothinShoes & Accessories,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Womens' Clothing,Other,211,Philippines,USER_X,5,Occasional,2013-12-16,2013-12-15,10000000253,2013-12-16,314,10000082,20.35,1
-Auction,0,5,314,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Other,0,Ebay,USER_S,5,Occasional,2013-12-15,2013-12-15,10000000051,2013-12-15,314,10000083,36.7,1
-Auction,0,13,533,0,10,Coins,2012-06-19 21:15:09,USER_Y,Coins & Paper Money,Coins: World,Africa,0,Ebay,USER_S,13,Entrepreneur,2013-08-17,2013-08-11,10000000190,2013-08-17,533,10000084,101.79,1
-ABIN,0,5,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,5,Occasional,2013-12-15,2013-12-15,10000000251,2013-12-15,1349,10000085,47.71,1
-ABIN,0,5,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,5,Occasional,2013-02-04,2013-02-03,10000000050,2013-02-04,1349,10000086,3.49,1
-ABIN,0,13,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,13,Entrepreneur,2013-01-11,2013-01-06,10000000049,2013-01-11,1349,10000087,46.44,1
-ABIN,0,13,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,13,Entrepreneur,2013-02-04,2013-02-03,10000000250,2013-02-04,1349,10000088,4.54,1
-ABIN,0,14,1357,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,14,Regular,2013-05-17,2013-05-12,10000000131,2013-05-17,1357,10000089,3.18,1
-FP-GTC,0,14,1504,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Electrical & Test Equipment,Test Equipment,0,Ebay,USER_S,14,Regular,2013-11-12,2013-11-10,10000000172,2013-11-12,1504,10000090,86.58,1
-FP-GTC,0,13,4943,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,13,Entrepreneur,2013-08-21,2013-08-18,10000000142,2013-08-21,4943,10000091,12.85,1
-ABIN,0,13,6762,0,-999,Unknown,2008-06-03 13:36:00,USER_Y,Unknown,Unknown,Unknown,0,Ebay,USER_S,13,Entrepreneur,2013-05-09,2013-05-05,10000000195,2013-05-09,6762,10000092,16.26,1
-Auction,3,13,9426,3,31,Mobile & Home Phones,2012-05-01 08:54:26,USER_Y,Mobile Phones & Communication,Home Phones & Accessories,Phone Accessories,3,\u82f1\u56fd,USER_S,13,Entrepreneur,2013-09-19,2013-09-15,10000000070,2013-09-19,9426,10000093,21.14,1
-FP-non GTC,0,14,10866,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Animals,Farm & Countryside,0,Ebay,USER_S,14,Regular,2013-02-06,2013-02-03,10000000165,2013-02-06,10866,10000094,20.6,1
-Auction,0,13,11554,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Jeans,0,Ebay,USER_S,13,Entrepreneur,2013-02-02,2013-01-27,10000000187,2013-02-02,11554,10000095,246,1
-FP-GTC,0,14,11848,0,37,Health & Beauty,2012-06-19 21:15:09,USER_Y,Health & Beauty,Fragrances,Women,0,Ebay,USER_S,14,Regular,2013-08-23,2013-08-18,10000000189,2013-08-23,11848,10000096,109,1
-Auction,0,13,13836,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Spoons,0,Ebay,USER_S,13,Entrepreneur,2013-08-03,2013-07-28,10000000139,2013-08-03,13836,10000097,39.41,1
-Auction,0,14,13836,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Spoons,0,Ebay,USER_S,14,Regular,2013-05-17,2013-05-12,10000000140,2013-05-17,13836,10000098,16.26,1
-FP-GTC,0,13,13987,0,11,Collectibles,2008-06-03 13:36:00,USER_Y,Collectibles,Paper,Booklets,0,Ebay,USER_S,13,Entrepreneur,2013-06-06,2013-06-02,10000000102,2013-06-06,13987,10000099,112.56,1
-Auction,0,14,15687,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Men's Clothing,T-Shirts,0,Ebay,USER_S,14,Regular,2013-07-02,2013-06-30,10000000076,2013-07-02,15687,10000100,184.21,1
-Auction,0,11,15687,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Men's Clothing,T-Shirts,0,Ebay,USER_S,11,Large Merchant,2013-10-25,2013-10-20,10000000082,2013-10-25,15687,10000001,27.48,1
-FP-non GTC,3,12,16145,3,12,Computers,2013-02-20 23:11:43,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Other Components & Parts,3,\u82f1\u56fd,USER_S,12,Merchant,2013-04-20,2013-04-14,10000000129,2013-04-20,16145,10000002,26.45,1
-FP-non GTC,0,13,16145,0,12,Computers,2013-02-20 23:11:43,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Other,0,Ebay,USER_S,13,Entrepreneur,2013-03-12,2013-03-10,10000000128,2013-03-12,16145,10000003,415.73,1
-ABIN,0,5,16509,0,28,Toys,2012-09-11 20:26:04,USER_Y,Toys & Hobbies,Model Railroads & Trains,S Scale,0,Ebay,USER_S,5,Occasional,2013-03-28,2013-03-24,10000000222,2013-03-28,16509,10000004,56.36,1
-ABIN,0,5,16509,0,28,Toys,2012-09-11 20:26:04,USER_Y,Toys & Hobbies,Model Railroads & Trains,S Scale,0,Ebay,USER_S,5,Occasional,2013-10-29,2013-10-27,10000000021,2013-10-29,16509,10000005,2.44,1
-FP-GTC,0,14,20485,0,36,Home & Living,2012-05-01 08:54:26,USER_Y,Home & Garden,Furniture,Other,0,Ebay,USER_S,14,Regular,2013-05-22,2013-05-19,10000000134,2013-05-22,20485,10000006,269.76,1
-FP-GTC,101,12,20485,101,36,Mobili per la casa,2008-06-03 13:36:00,USER_Y,CasArredamento e Bricolage,Cucina,Altro per cucina,101,Italy,USER_S,12,Merchant,2013-01-25,2013-01-20,10000000135,2013-01-25,20485,10000007,109,1
-FP-GTC,101,12,20485,101,36,Mobili per la casa,2008-06-03 13:36:00,USER_Y,CasArredamento e Bricolage,Cucina,Altro per cucina,101,Italy,USER_S,12,Merchant,2013-06-12,2013-06-09,10000000136,2013-06-12,20485,10000008,101.79,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2013-12-26,2013-12-22,10000000241,2013-12-26,23446,10000009,246,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2013-12-26,2013-12-22,10000000041,2013-12-26,23446,10000010,189.23,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2013-12-31,2013-12-29,10000000242,2013-12-31,23446,10000011,15.65,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2013-10-04,2013-09-29,10000000040,2013-10-04,23446,10000012,28.23,1
-FP-GTC,0,5,24541,0,25,Sports Memorabilia,2013-07-16 08:23:09,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,College-NCAA,0,Ebay,USER_S,5,Occasional,2013-03-16,2013-03-10,10000000194,2013-03-16,24541,10000013,16.26,1
-FP-GTC,0,5,26262,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Advertising,Food & Beverage,0,Ebay,USER_S,5,Occasional,2013-05-21,2013-05-19,10000000101,2013-05-21,26262,10000014,122.78,1
-FP-GTC,3,14,30059,3,21,Photography,2012-09-11 20:26:04,USER_Y,Cameras & Photography,Lenses & Filters,Lens AdapterMounts & Tubes,3,\u82f1\u56fd,USER_S,14,Regular,2013-01-28,2013-01-27,10000000077,2013-01-28,30059,10000015,172.03,1
-Auction,3,14,31387,3,17,Jewellery & Watches,2013-04-28 20:37:19,USER_Y,Jewellery & Watches,Watches,Wristwatches,3,\u82f1\u56fd,USER_S,14,Regular,2013-04-26,2013-04-21,10000000057,2013-04-26,31387,10000016,42.99,1
-Auction,3,14,31387,3,17,Jewellery & Watches,2013-04-28 20:37:19,USER_Y,Jewellery & Watches,Watches,Wristwatches,3,\u82f1\u56fd,USER_S,14,Regular,2013-10-06,2013-10-06,10000000258,2013-10-06,31387,10000017,207.5,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2013-11-06,2013-11-03,10000000261,2013-11-06,31519,10000018,5.91,1
-FP-GTC,3,14,31519,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,3,\u82f1\u56fd,USER_S,14,Regular,2013-10-06,2013-10-06,10000000058,2013-10-06,31519,10000019,39.41,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2013-12-28,2013-12-22,10000000059,2013-12-28,31519,10000020,16.26,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2013-11-06,2013-11-03,10000000060,2013-11-06,31519,10000021,16.26,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2013-11-06,2013-11-03,10000000260,2013-11-06,31519,10000022,78.48,1
-FP-GTC,3,14,31519,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,3,\u82f1\u56fd,USER_S,14,Regular,2013-12-28,2013-12-22,10000000259,2013-12-28,31519,10000023,190.22,1
-FP-GTC,100,12,35570,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,Motorcycle Parts,100,Ebaymotors,USER_S,12,Merchant,2013-06-11,2013-06-09,10000000156,2013-06-11,35570,10000024,2.44,1
-Auction,0,5,36250,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Hunting,Decoys,0,Ebay,USER_S,5,Occasional,2013-01-10,2013-01-06,10000000119,2013-01-10,36250,10000025,7.12,1
-FP-non GTC,0,14,38238,0,36,Home & Living,2012-09-11 20:26:04,USER_Y,Home & Garden,Home Decor,Other,0,Ebay,USER_S,14,Regular,2013-09-17,2013-09-15,10000000186,2013-09-17,38238,10000026,36.7,1
-FP-GTC,3,14,40059,3,33,Consumer Electronics - Audio,2012-05-01 08:57:38,USER_Y,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,3,\u82f1\u56fd,USER_S,14,Regular,2013-08-14,2013-08-11,10000000038,2013-08-14,40059,10000027,35.72,1
-FP-GTC,3,14,40059,3,33,Consumer Electronics - Audio,2012-05-01 08:57:38,USER_Y,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,3,\u82f1\u56fd,USER_S,14,Regular,2013-08-09,2013-08-04,10000000239,2013-08-09,40059,10000028,3.49,1
-FP-GTC,0,13,41940,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,0,Ebay,USER_S,13,Entrepreneur,2013-12-02,2013-12-01,10000000034,2013-12-02,41940,10000029,223.63,1
-FP-GTC,0,13,41940,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,0,Ebay,USER_S,13,Entrepreneur,2013-02-01,2013-01-27,10000000235,2013-02-01,41940,10000030,265.56,1
-FP-non GTC,0,13,43479,0,21,Photo,2012-09-11 20:26:04,USER_Y,Cameras & Photo,Film Photography,Other,0,Ebay,USER_S,13,Entrepreneur,2013-07-28,2013-07-28,10000000127,2013-07-28,43479,10000031,62.02,1
-FP-GTC,0,12,44079,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Exercise & Fitness,GyWorkout & Yoga,0,Ebay,USER_S,12,Merchant,2013-06-16,2013-06-16,10000000103,2013-06-16,44079,10000032,46.44,1
-Auction,101,14,45238,101,9,Vestiti ed Accessori,2012-09-11 20:27:22,USER_Y,Abbigliamento e accessori,Donna: Accessori,SciarpFoulard e Scialli,101,Italy,USER_S,14,Regular,2013-08-23,2013-08-18,10000000201,2013-08-23,45238,10000033,132.33,1
-Auction,0,13,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,13,Entrepreneur,2013-06-15,2013-06-09,10000000122,2013-06-15,45333,10000034,448.8,1
-FP-non GTC,0,14,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,14,Regular,2013-06-15,2013-06-09,10000000123,2013-06-15,45333,10000035,207.5,1
-FP-non GTC,0,14,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,14,Regular,2013-06-01,2013-05-26,10000000124,2013-06-01,45333,10000036,190.22,1
-FP-GTC,0,14,46575,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,Light Equipment & Tools,Air Tools,0,Ebay,USER_S,14,Regular,2013-08-10,2013-08-04,10000000072,2013-08-10,46575,10000037,16.71,1
-FP-non GTC,0,13,50508,0,21,Photo,2013-02-20 23:11:43,USER_Y,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,0,Ebay,USER_S,13,Entrepreneur,2013-03-22,2013-03-17,10000000043,2013-03-22,50508,10000038,4.13,1
-FP-non GTC,0,13,50508,0,21,Photo,2013-02-20 23:11:43,USER_Y,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,0,Ebay,USER_S,13,Entrepreneur,2013-12-25,2013-12-22,10000000244,2013-12-25,50508,10000039,1.88,1
-FP-GTC,0,13,50677,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,0,Ebay,USER_S,13,Entrepreneur,2013-07-22,2013-07-21,10000000121,2013-07-22,50677,10000040,491.32,1
-FP-GTC,0,5,50677,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,0,Ebay,USER_S,5,Occasional,2013-04-13,2013-04-07,10000000120,2013-04-13,50677,10000041,2.44,1
-Auction,0,14,51582,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Kids' ClothinShoes & Accs,Girls' Clothing (Sizes 4 & Up),0,Ebay,USER_S,14,Regular,2013-04-16,2013-04-14,10000000168,2013-04-16,51582,10000042,56.36,1
-FP-GTC,0,13,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,13,Entrepreneur,2013-08-21,2013-08-18,10000000073,2013-08-21,57013,10000043,15.85,1
-FP-non GTC,0,14,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,14,Regular,2013-04-22,2013-04-21,10000000075,2013-04-22,57013,10000044,2.44,1
-FP-GTC,0,14,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,14,Regular,2013-08-29,2013-08-25,10000000074,2013-08-29,57013,10000045,7.12,1
-Auction,0,14,57784,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Baby & Toddler Clothing,Boys' Clothing (Newborn-5T),0,Ebay,USER_S,14,Regular,2013-05-16,2013-05-12,10000000093,2013-05-16,57784,10000046,35.72,1
-Auction,3,11,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,11,Large Merchant,2013-08-23,2013-08-18,10000000265,2013-08-23,57990,10000047,9.26,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,14,Regular,2013-07-10,2013-07-07,10000000266,2013-07-10,57990,10000048,3.18,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,14,Regular,2013-08-10,2013-08-04,10000000267,2013-08-10,57990,10000049,638.72,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,14,Regular,2013-08-23,2013-08-18,10000000065,2013-08-23,57990,10000050,141.7,1
-ABIN,0,13,57990,0,9,Apparel,2013-09-10 16:52:46,USER_Y,ClothinShoes & Accessories,Men's Clothing,Casual Shirts,0,Ebay,USER_S,13,Entrepreneur,2013-04-18,2013-04-14,10000000143,2013-04-18,57990,10000051,12.19,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,14,Regular,2013-07-10,2013-07-07,10000000066,2013-07-10,57990,10000052,132.33,1
-ABIN,3,5,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,5,Occasional,2013-06-16,2013-06-16,10000000144,2013-06-16,57990,10000053,5.48,1
-Auction,3,11,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,11,Large Merchant,2013-07-15,2013-07-14,10000000064,2013-07-15,57990,10000054,1.88,1
-FP-GTC,0,14,60340,0,42,Entertainment Memorabilia,2008-06-03 13:36:00,USER_Y,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,0,Ebay,USER_S,14,Regular,2013-11-06,2013-11-03,10000000061,2013-11-06,60340,10000055,12.85,1
-FP-GTC,0,14,60340,0,42,Entertainment Memorabilia,2008-06-03 13:36:00,USER_Y,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,0,Ebay,USER_S,14,Regular,2013-12-27,2013-12-22,10000000262,2013-12-27,60340,10000056,62.02,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,\u82f1\u56fd,USER_S,12,Merchant,2013-07-29,2013-07-28,10000000019,2013-07-29,60606,10000057,15.85,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,\u82f1\u56fd,USER_S,12,Merchant,2013-11-17,2013-11-17,10000000220,2013-11-17,60606,10000058,9.26,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,\u82f1\u56fd,USER_S,12,Merchant,2013-07-27,2013-07-21,10000000018,2013-07-27,60606,10000059,16.71,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,\u82f1\u56fd,USER_S,12,Merchant,2013-07-29,2013-07-28,10000000219,2013-07-29,60606,10000060,20.6,1
-Auction,3,5,63861,3,9,Clothing & Accessories,2013-07-16 08:23:09,USER_Y,ClotheShoes & Accessories,Women's Clothing,Dresses,3,\u82f1\u56fd,USER_S,5,Occasional,2013-01-09,2013-01-06,10000000145,2013-01-09,63861,10000061,1.88,1
-ABIN,0,5,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,5,Occasional,2013-06-11,2013-06-09,10000000200,2013-06-11,63861,10000062,141.7,1
-ABIN,0,5,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,5,Occasional,2013-01-10,2013-01-06,10000000199,2013-01-10,63861,10000063,1.88,1
-Others,0,11,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,11,Large Merchant,2013-09-16,2013-09-15,10000000237,2013-09-16,63861,10000064,112.56,1
-Others,0,11,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,11,Large Merchant,2013-01-14,2013-01-13,10000000036,2013-01-14,63861,10000065,94.45,1
-Auction,0,14,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,14,Regular,2013-05-17,2013-05-12,10000000125,2013-05-17,63861,10000066,78.48,1
-ABIN,0,13,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,13,Entrepreneur,2013-06-05,2013-06-02,10000000198,2013-06-05,63861,10000067,5.48,1
-Auction,3,14,63864,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Women's Clothing,Skirts,3,\u82f1\u56fd,USER_S,14,Regular,2013-05-24,2013-05-19,10000000094,2013-05-24,63864,10000068,28.23,1
-Others,0,13,63889,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Mixed Items & Lots,0,Ebay,USER_S,13,Entrepreneur,2013-05-15,2013-05-12,10000000104,2013-05-15,63889,10000069,3.49,1
-FP-GTC,2,11,67698,2,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewellery Packaging & Display,2,Canada,USER_S,11,Large Merchant,2013-03-25,2013-03-24,10000000107,2013-03-25,67698,10000070,15.65,1
-FP-GTC,0,11,67698,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewelry Packaging & Display,0,Ebay,USER_S,11,Large Merchant,2013-03-09,2013-03-03,10000000108,2013-03-09,67698,10000071,5.48,1
-FP-GTC,0,11,67698,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewelry Packaging & Display,0,Ebay,USER_S,11,Large Merchant,2013-12-05,2013-12-01,10000000106,2013-12-05,67698,10000072,246,1
-FP-non GTC,0,13,73506,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Tea PotSets,0,Ebay,USER_S,13,Entrepreneur,2013-04-18,2013-04-14,10000000182,2013-04-18,73506,10000073,122.78,1
-FP-GTC,0,14,75665,0,16,Home Improvement,2012-09-11 20:26:04,USER_Y,Home & Garden,YarGarden & Outdoor Living,Gardening Supplies,0,Ebay,USER_S,14,Regular,2013-11-01,2013-10-27,10000000169,2013-11-01,75665,10000074,223.63,1
-ABIN,3,5,75708,3,28,Toys & Games,2012-05-01 08:57:38,USER_Y,Toys & Games,Action Figures,TMovies & Video Games,3,\u82f1\u56fd,USER_S,5,Occasional,2013-05-03,2013-04-28,10000000146,2013-05-03,75708,10000075,141.7,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2013-04-21,2013-04-21,10000000151,2013-04-21,80053,10000076,21.14,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2013-03-12,2013-03-10,10000000149,2013-03-12,80053,10000077,55.89,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2013-05-19,2013-05-19,10000000150,2013-05-19,80053,10000078,51.23,1
-Auction,0,14,80135,0,12,Computers,2013-07-16 08:23:09,USER_Y,Computers/Tablets & Networking,DriveStorage & Blank Media,Blank Media & Accessories,0,Ebay,USER_S,14,Regular,2013-11-23,2013-11-17,10000000083,2013-11-23,80135,10000079,21.72,1
-Auction,3,14,95672,3,9,Clothing & Accessories,2013-07-16 08:23:09,USER_Y,ClotheShoes & Accessories,Women's Shoes,Trainers,3,\u82f1\u56fd,USER_S,14,Regular,2013-10-19,2013-10-13,10000000089,2013-10-19,95672,10000080,204.28,1
-Others,0,11,95672,0,9,Apparel,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Women's Shoes,Athletic,0,Ebay,USER_S,11,Large Merchant,2013-05-18,2013-05-12,10000000152,2013-05-18,95672,10000081,21.14,1
-Others,0,5,100847,0,3,Books,2008-06-03 13:36:00,USER_Y,Half Books,Half Books,Half Books,0,Ebay,USER_S,5,Occasional,2013-02-01,2013-01-27,10000000035,2013-02-01,100847,10000082,204.28,1
-Others,0,5,100847,0,3,Books,2008-06-03 13:36:00,USER_Y,Half Books,Half Books,Half Books,0,Ebay,USER_S,5,Occasional,2013-01-14,2013-01-13,10000000236,2013-01-14,100847,10000083,122.78,1
-ABIN,3,14,139973,3,32,PC & Video Gaming,2012-09-11 20:26:04,USER_Y,Video Games & Consoles,Games,Games,3,\u82f1\u56fd,USER_S,14,Regular,2013-08-05,2013-08-04,10000000090,2013-08-05,139973,10000084,94.45,1
-ABIN,0,11,139973,0,32,Video Games,2012-06-19 21:15:09,USER_Y,Video Games & Consoles,Video Games,Video Games,0,Ebay,USER_S,11,Large Merchant,2013-05-19,2013-05-19,10000000091,2013-05-19,139973,10000085,86.58,1
-Auction,3,14,150047,3,15,Hobbies & Crafts,2012-06-19 21:15:09,USER_Y,Crafts,Jewellery Making,Findings,3,\u82f1\u56fd,USER_S,14,Regular,2013-12-01,2013-12-01,10000000033,2013-12-01,150047,10000086,56.36,1
-Auction,3,14,150047,3,15,Hobbies & Crafts,2012-06-19 21:15:09,USER_Y,Crafts,Jewellery Making,Findings,3,\u82f1\u56fd,USER_S,14,Regular,2013-12-02,2013-12-01,10000000234,2013-12-02,150047,10000087,290.72,1
-FP-GTC,0,13,155226,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,0,Ebay,USER_S,13,Entrepreneur,2013-01-11,2013-01-06,10000000249,2013-01-11,155226,10000088,60.37,1
-FP-GTC,0,13,155226,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,0,Ebay,USER_S,13,Entrepreneur,2013-05-27,2013-05-26,10000000048,2013-05-27,155226,10000089,112.56,1
-FP-GTC,0,13,156356,0,11,Collectibles,2008-06-03 13:36:00,USER_Y,Collectibles,Postcards,BuildingArchitecture,0,Ebay,USER_S,13,Entrepreneur,2013-09-01,2013-09-01,10000000181,2013-09-01,156356,10000090,265.56,1
-FP-GTC,0,11,158798,0,28,Toys,2008-09-09 22:08:47,USER_Y,Toys & Hobbies,Vintage & Antique Toys,Spinning Tops,0,Ebay,USER_S,11,Large Merchant,2013-04-11,2013-04-07,10000000092,2013-04-11,158798,10000091,35.72,1
-FP-non GTC,0,13,165888,0,17,JewelrGemWatches,2009-01-12 07:05:17,USER_Y,Jewelry & Watches,Vintage & Antique Jewelry,Costume,0,Ebay,USER_S,13,Entrepreneur,2013-05-05,2013-05-05,10000000191,2013-05-05,165888,10000092,92.98,1
-Auction,3,11,170083,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),3,\u82f1\u56fd,USER_S,11,Large Merchant,2013-11-21,2013-11-17,10000000229,2013-11-21,170083,10000093,28.23,1
-Auction,3,11,170083,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),3,\u82f1\u56fd,USER_S,11,Large Merchant,2013-10-07,2013-10-06,10000000028,2013-10-07,170083,10000094,27.48,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,14,Regular,2013-07-12,2013-07-07,10000000031,2013-07-12,175750,10000095,9.26,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,14,Regular,2013-06-07,2013-06-02,10000000032,2013-06-07,175750,10000096,3.18,1
-FP-GTC,0,14,175750,0,36,Home & Living,2012-05-01 08:57:38,USER_Y,Home & Garden,Bedding,Blankets & Throws,0,Ebay,USER_S,14,Regular,2013-05-22,2013-05-19,10000000177,2013-05-22,175750,10000097,12.04,1
-Auction,3,13,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,13,Entrepreneur,2013-11-28,2013-11-24,10000000030,2013-11-28,175750,10000098,20.6,1
-Auction,3,13,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,13,Entrepreneur,2013-07-12,2013-07-07,10000000231,2013-07-12,175750,10000099,12.04,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,14,Regular,2013-06-07,2013-06-02,10000000232,2013-06-07,175750,10000100,4.13,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,14,Regular,2013-12-01,2013-12-01,10000000233,2013-12-01,175750,10000201,73.26,1
-Auction,0,12,48027,0,10,Coins,2013-07-16 08:23:09,USER_Y,Coins & Paper Money,Paper Money: World,Asia,0,Ebay,USER_S,12,Merchant,2012-10-12,2012-10-06,10000000002,2012-10-12,48027,10000001,184.21,1
-Others,0,5,164261,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Earrings,0,Ebay,USER_S,5,Occasional,2012-08-20,2012-08-18,10000000003,2012-08-20,164261,10000002,172.03,1
-Auction,15,14,82494,15,3,Books,2010-09-13 07:05:54,USER_Y,BookMagazines,NULL,Comic Books,15,Australia,USER_S,14,Regular,2012-09-18,2012-09-15,10000000007,2012-09-18,82494,10000003,66.6,1
-Auction,15,12,66767,15,16,Home & Garden,2011-06-14 07:48:40,USER_Y,Home & Garden,NULL,Dogs,15,Australia,USER_S,12,Merchant,2012-06-30,2012-06-30,10000000008,2012-06-30,66767,10000004,92.98,1
-FP-GTC,0,5,152801,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,NULL,Earrings,0,Ebay,USER_S,5,Occasional,2012-08-29,2012-08-25,10000000010,2012-08-29,152801,10000005,132.33,1
-FP-GTC,0,13,43398,0,36,Home & Living,2013-02-20 23:11:43,USER_Y,Home & Garden,NULL,Cheese & Crackers,0,Ebay,USER_S,13,Entrepreneur,2012-01-06,2012-01-06,10000000012,2012-01-06,43398,10000006,7.12,1
-Auction,0,14,95173,0,37,Health & Beauty,2008-06-03 13:36:00,USER_Y,Health & Beauty,Bath & Body,Bath Sets & Kits,0,Ebay,USER_S,14,Regular,2012-08-14,2012-08-11,10000000013,2012-08-14,95173,10000007,204.28,1
-Auction,15,13,158666,15,28,Toys,2012-09-11 20:27:22,USER_Y,ToyHobbies,Action Figures,Anime & Manga,15,Australia,USER_S,13,Entrepreneur,2012-07-17,2012-07-14,10000000016,2012-07-17,158666,10000008,35.72,1
-Auction,0,12,12688,0,57,Sothebys,2008-06-03 13:36:00,USER_Y,eBay Premier,Books & Manuscripts,Books: Other,0,Ebay,USER_S,12,Merchant,2012-07-23,2012-07-21,10000000017,2012-07-23,12688,10000009,4.13,1
-FP-GTC,15,5,103324,15,9,Clothing & Accessories,2012-05-01 08:54:26,USER_Y,ClothinShoeAccessories,Women's Shoes,Mixed Items,15,Australia,USER_S,5,Occasional,2012-07-27,2012-07-21,10000000018,2012-07-27,103324,10000010,27.48,1
-FP-GTC,15,14,108782,15,8,Auto Parts,2013-09-10 16:52:46,USER_Y,Vehicle Parts & Accessories,CaTruck Parts,Car Care & Cleaning,15,Australia,USER_S,14,Regular,2012-07-29,2012-07-28,10000000019,2012-07-29,108782,10000011,9.26,1
-Auction,0,12,80287,0,12,Computers,2012-05-01 08:54:26,USER_Y,Computers/Tablets & Networking,Software,Office & Business,0,Ebay,USER_S,12,Merchant,2012-11-17,2012-11-17,10000000020,2012-11-17,80287,10000012,3.18,1
-Others,100,13,140746,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,Vintage Car & Truck Parts,100,Ebaymotors,USER_S,13,Entrepreneur,2012-10-29,2012-10-27,10000000021,2012-10-29,140746,10000013,3.18,1
-ABIN,0,14,87118,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Outdoor Sports,Paintball,0,Ebay,USER_S,14,Regular,2012-03-28,2012-03-24,10000000022,2012-03-28,87118,10000014,377.94,1
-Auction,0,12,25147,0,25,Sports Memorabilia,2010-02-08 10:51:44,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Baseball-MLB,0,Ebay,USER_S,12,Merchant,2012-01-30,2012-01-27,10000000023,2012-01-30,25147,10000015,146.33,1
-FP-GTC,15,5,170302,15,15,Crafts,2010-09-13 07:13:34,USER_Y,Crafts,Embroidery,Design CDs,15,Australia,USER_S,5,Occasional,2012-10-07,2012-10-06,10000000028,2012-10-07,170302,10000016,51.23,1
-FP-non GTC,0,13,53064,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,Heavy Equipment,Antique & Vintage Farm Equip,0,Ebay,USER_S,13,Entrepreneur,2012-11-28,2012-11-24,10000000030,2012-11-28,53064,10000017,72.65,1
-Auction,0,13,132939,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Other,0,Ebay,USER_S,13,Entrepreneur,2012-07-12,2012-07-07,10000000031,2012-07-12,132939,10000018,66.6,1
-Auction,15,12,113593,15,40,Cell Phones & Portable Electro,2012-09-11 20:26:04,USER_Y,Phones,Mobile Phones,Mobile Phones,15,Australia,USER_S,12,Merchant,2012-06-07,2012-06-02,10000000032,2012-06-07,113593,10000019,9.26,1
-Auction,100,14,34273,100,8,Auto - Parts,2008-06-03 13:36:00,USER_Y,eBay Motors,Parts & Accessories,Motorcycle,100,Ebaymotors,USER_S,14,Regular,2012-12-01,2012-12-01,10000000033,2012-12-01,34273,10000020,583.44,1
-FP-GTC,15,14,106340,15,16,Home & Garden,2012-09-11 20:26:04,USER_Y,Home & Garden,Gardening,Hand Tools,15,Australia,USER_S,14,Regular,2012-12-02,2012-12-01,10000000034,2012-12-02,106340,10000021,638.72,1
-FP-GTC,15,14,150265,15,2,Baby,2011-04-05 08:25:25,USER_Y,Baby,Baby Clothing,Boys,15,Australia,USER_S,14,Regular,2012-02-01,2012-01-27,10000000035,2012-02-01,150265,10000022,4.54,1
-FP-GTC,0,12,24760,0,25,Sports Memorabilia,2010-02-08 10:51:44,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Hockey-NHL,0,Ebay,USER_S,12,Merchant,2012-01-14,2012-01-13,10000000036,2012-01-14,24760,10000023,319.79,1
-Auction,0,12,37831,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Advertising,Merchandise & Memorabilia,0,Ebay,USER_S,12,Merchant,2012-08-14,2012-08-11,10000000038,2012-08-14,37831,10000024,20.35,1
-FP-non GTC,3,5,1120,3,3,BookComics & Magazines,2008-06-03 13:36:00,USER_Y,Books,First Editions,Other,3,\u82f1\u56fd,USER_S,5,Occasional,2012-10-04,2012-09-29,10000000040,2012-10-04,1120,10000025,223.63,1
-FP-GTC,100,13,43972,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,ATV Parts,100,Ebaymotors,USER_S,13,Entrepreneur,2012-12-26,2012-12-22,10000000041,2012-12-26,43972,10000026,204.28,1
-Auction,15,14,166013,15,12,Computers,2012-05-01 08:54:26,USER_Y,Computers,Computer Components & Parts,Video Capture & TV Tuner Cards,15,Australia,USER_S,14,Regular,2012-12-31,2012-12-29,10000000042,2012-12-31,166013,10000027,5.48,1
-Auction,15,14,15568,15,2,Baby,2011-04-05 08:25:25,USER_Y,Baby,Baby Clothing,Unisex,15,Australia,USER_S,14,Regular,2012-03-22,2012-03-17,10000000043,2012-03-22,15568,10000028,27.48,1
-FP-GTC,15,5,103178,15,9,Clothing & Accessories,2012-05-01 08:54:26,USER_Y,ClothinShoeAccessories,Women's Bags,Women's Bags,15,Australia,USER_S,5,Occasional,2012-12-25,2012-12-22,10000000044,2012-12-25,103178,10000029,21.72,1
-ABIN,0,12,2023,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Team Sports,Basketball,0,Ebay,USER_S,12,Merchant,2012-12-05,2012-12-01,10000000047,2012-12-05,2023,10000030,3.18,1
-FP-GTC,0,11,94847,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,Vehicle Electronics & GPS,Car Video,0,Ebay,USER_S,11,Large Merchant,2012-05-27,2012-05-26,10000000048,2012-05-27,94847,10000031,491.32,1
-Auction,0,14,15868,0,23,Real Estate,2008-06-03 13:36:00,USER_Y,Real Estate,Land,Land,0,Ebay,USER_S,14,Regular,2012-01-11,2012-01-06,10000000049,2012-01-11,15868,10000032,448.8,1
-Auction,0,13,32876,0,36,Home & Living,2012-09-11 20:26:04,USER_Y,Home & Garden,Home Improvement,Plumbing & Fixtures,0,Ebay,USER_S,13,Entrepreneur,2012-02-04,2012-02-03,10000000050,2012-02-04,32876,10000033,415.73,1
-Auction,0,13,62179,0,9,Apparel,2008-06-03 13:36:00,USER_Y,ClothinShoes & Accessories,Women's Clothing,Athletic Apparel,0,Ebay,USER_S,13,Entrepreneur,2012-12-15,2012-12-15,10000000051,2012-12-15,62179,10000034,377.94,1
-FP-GTC,15,14,33038,15,30,Musical Instruments,2010-06-14 07:48:40,USER_Y,Musical Instruments,Instruments,Guitars (Electric),15,Australia,USER_S,14,Regular,2012-06-30,2012-06-30,10000000052,2012-06-30,33038,10000035,146.33,1
-FP-GTC,0,5,156614,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,0,Ebay,USER_S,5,Occasional,2012-12-16,2012-12-15,10000000053,2012-12-16,156614,10000036,7.12,1
-Auction,0,13,106246,0,37,Health & Beauty,2013-04-28 20:37:19,USER_Y,Health & Beauty,Hair Care & Styling,Shampoo & Conditioning,0,Ebay,USER_S,13,Entrepreneur,2012-10-17,2012-10-13,10000000055,2012-10-17,106246,10000037,42.99,1
-Auction,0,13,20865,0,9,Apparel,2008-06-03 13:36:00,USER_Y,ClothinShoes & Accessories,Men's Clothing,Athletic Apparel,0,Ebay,USER_S,13,Entrepreneur,2012-10-08,2012-10-06,10000000056,2012-10-08,20865,10000038,12.85,1
-FP-GTC,0,13,15115,0,32,Video Games,2012-06-19 21:15:09,USER_Y,Video Games & Consoles,Video Games,Video Games,0,Ebay,USER_S,13,Entrepreneur,2012-04-26,2012-04-21,10000000057,2012-04-26,15115,10000039,55.89,1
-FP-GTC,0,14,3838,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Charms & Charm Bracelets,0,Ebay,USER_S,14,Regular,2012-10-06,2012-10-06,10000000058,2012-10-06,3838,10000040,73.26,1
-Auction,0,11,759,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,11,Large Merchant,2012-12-28,2012-12-22,10000000059,2012-12-28,759,10000041,112.56,1
-FP-non GTC,0,11,61323,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,TVideo & Home Audio,TVideo & Audio Accessories,0,Ebay,USER_S,11,Large Merchant,2012-11-06,2012-11-03,10000000060,2012-11-06,61323,10000042,3.49,1
-Auction,0,13,121153,0,2,Baby,2012-09-11 20:26:04,USER_Y,Baby,Nursery Decor,Night Lights,0,Ebay,USER_S,13,Entrepreneur,2012-11-06,2012-11-03,10000000061,2012-11-06,121153,10000043,184.21,1
-Auction,0,13,88750,0,13,Consumer Electronics - Other,2012-05-01 08:54:26,USER_Y,Consumer Electronics,Vehicle Electronics & GPS,Radar & Laser Detectors,0,Ebay,USER_S,13,Entrepreneur,2012-12-27,2012-12-22,10000000062,2012-12-27,88750,10000044,157.14,1
-FP-GTC,15,14,161567,15,12,Computers,2012-05-01 08:54:26,USER_Y,Computers,Laptop & Desktop Accessories,Laptop Batteries,15,Australia,USER_S,14,Regular,2012-07-15,2012-07-14,10000000064,2012-07-15,161567,10000045,72.65,1
-FP-GTC,15,14,113802,15,99,Everything Else,2008-06-03 13:36:00,USER_Y,Lots More...,Metaphysical,Herbs,15,Australia,USER_S,14,Regular,2012-08-23,2012-08-18,10000000065,2012-08-23,113802,10000046,51.23,1
-FP-non GTC,15,14,15808,15,9,Clothing & Accessories,2012-09-11 20:26:04,USER_Y,ClothinShoeAccessories,Women's Clothing,Tops & Blouses,15,Australia,USER_S,14,Regular,2012-07-10,2012-07-07,10000000066,2012-07-10,15808,10000047,15.85,1
-FP-GTC,3,13,174053,3,8,Auto - Parts,2013-04-28 20:37:19,USER_Y,Vehicle Parts & Accessories,Car Parts,External & Body Parts,3,\u82f1\u56fd,USER_S,13,Entrepreneur,2012-08-10,2012-08-04,10000000067,2012-08-10,174053,10000048,7.12,1
-Auction,0,14,2635,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Toy Soldiers,1970-Now,0,Ebay,USER_S,14,Regular,2012-09-01,2012-09-01,10000000069,2012-09-01,2635,10000049,12.04,1
-Auction,3,13,1161,3,18,DVDFilm & TV,2008-06-03 13:36:00,USER_Y,DVFilm & TV,Other Formats,Videos: NTSC (US),3,\u82f1\u56fd,USER_S,13,Entrepreneur,2012-09-19,2012-09-15,10000000070,2012-09-19,1161,10000050,73.26,1
-FP-non GTC,0,5,64076,0,12,Computers,2012-05-01 08:54:26,USER_Y,Computers/Tablets & Networking,Enterprise NetworkinServers,Switches & Hubs,0,Ebay,USER_S,5,Occasional,2012-08-10,2012-08-04,10000000072,2012-08-10,64076,10000051,184.21,1
-FP-GTC,15,13,33977,15,15,Crafts,2010-06-14 07:56:25,USER_Y,Crafts,Scrapbooking,Albums,15,Australia,USER_S,13,Entrepreneur,2012-08-21,2012-08-18,10000000073,2012-08-21,33977,10000052,172.03,1
-FP-GTC,0,14,31673,0,25,Sports Memorabilia,2010-02-08 10:48:39,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,Racing-NASCAR,0,Ebay,USER_S,14,Regular,2012-08-29,2012-08-25,10000000074,2012-08-29,31673,10000053,122.78,1
-Auction,3,14,174106,3,8,Auto - Parts,2013-04-28 20:37:19,USER_Y,Vehicle Parts & Accessories,Car Parts,Transmission & Drivetrain,3,\u82f1\u56fd,USER_S,14,Regular,2012-04-22,2012-04-21,10000000075,2012-04-22,174106,10000054,92.98,1
-Auction,0,13,26249,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Printing & Graphic Arts,Commercial Printing Presses,0,Ebay,USER_S,13,Entrepreneur,2012-07-02,2012-06-30,10000000076,2012-07-02,26249,10000055,12.19,1
-FP-GTC,0,5,159184,0,24,Sporting Goods,2013-09-10 16:52:46,USER_Y,Sporting Goods,Winter Sports,Snowboarding,0,Ebay,USER_S,5,Occasional,2012-01-28,2012-01-27,10000000077,2012-01-28,159184,10000056,15.65,1
-FP-GTC,3,11,10058,3,27,Tickets,2013-09-10 16:52:46,USER_Y,Events Tickets,Other Tickets,Other Tickets,3,\u82f1\u56fd,USER_S,11,Large Merchant,2012-03-11,2012-03-10,10000000078,2012-03-11,10058,10000057,101.79,1
-ABIN,0,12,48904,0,9,Apparel,2012-05-01 08:54:26,USER_Y,ClothinShoes & Accessories,Vintage,Women's Vintage Clothing,0,Ebay,USER_S,12,Merchant,2012-05-29,2012-05-26,10000000079,2012-05-29,48904,10000058,7.12,1
-FP-non GTC,0,14,145970,0,28,Toys,2008-10-08 07:18:40,USER_Y,Toys & Hobbies,Models & Kits,Automotive,0,Ebay,USER_S,14,Regular,2012-05-31,2012-05-26,10000000080,2012-05-31,145970,10000059,12.85,1
-FP-GTC,0,13,963,0,9,Apparel,2012-05-01 08:54:26,USER_Y,ClothinShoes & Accessories,Vintage,Women's Vintage Shoes,0,Ebay,USER_S,13,Entrepreneur,2012-10-25,2012-10-20,10000000082,2012-10-25,963,10000060,12.19,1
-FP-GTC,3,13,118687,3,37,Health & Beauty,2013-04-28 20:37:19,USER_Y,Health & Beauty,Fragrances,Women's Fragrances,3,\u82f1\u56fd,USER_S,13,Entrepreneur,2012-11-23,2012-11-17,10000000083,2012-11-23,118687,10000061,92.98,1
-FP-GTC,0,14,20886,0,28,Toys,2010-03-22 10:34:30,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,Cars: RacinNASCAR,0,Ebay,USER_S,14,Regular,2012-09-28,2012-09-22,10000000085,2012-09-28,20886,10000062,42.99,1
-Auction,15,13,148324,15,31,Phones,2012-09-18 00:08:03,USER_XIANZHU,Phones,Mobile Accessories,CaseCoverSkins,15,Australia,USER_S,13,Entrepreneur,2012-09-28,2012-09-22,10000000086,2012-09-28,148324,10000063,1.88,1
-Auction,15,14,139255,15,17,JewelrGemWatches,2012-05-01 08:54:26,USER_Y,Jewellery & Watches,Fine Jewellery,Earrings,15,Australia,USER_S,14,Regular,2012-10-19,2012-10-13,10000000089,2012-10-19,139255,10000064,21.14,1
-FP-GTC,0,5,20213,0,11,Collectibles,2008-09-09 22:08:47,USER_Y,Collectibles,Postcards,US StateCities & Towns,0,Ebay,USER_S,5,Occasional,2012-08-05,2012-08-04,10000000090,2012-08-05,20213,10000065,21.14,1
-Auction,15,13,32996,15,42,Entertainment Memorabilia,2012-09-11 20:26:04,USER_Y,Movies,Television Memorabilia,Clippings,15,Australia,USER_S,13,Entrepreneur,2012-05-19,2012-05-19,10000000091,2012-05-19,32996,10000066,132.33,1
-FP-GTC,0,14,99985,0,11,Collectibles,2008-09-09 22:08:47,USER_Y,Collectibles,Trading Cards,Sci-FFantasy,0,Ebay,USER_S,14,Regular,2012-04-11,2012-04-07,10000000092,2012-04-11,99985,10000067,120.87,1
-Auction,3,14,67703,3,17,Jewellery & Watches,2012-06-19 21:15:09,USER_Y,Jewellery & Watches,Jewellery Boxes & Supplies,Jewellery Display,3,\u82f1\u56fd,USER_S,14,Regular,2012-05-16,2012-05-12,10000000093,2012-05-16,67703,10000068,120.87,1
-FP-non GTC,0,11,65,0,11,Collectibles,2013-02-20 23:11:43,USER_Y,Collectibles,Comics,Platinum Age (1897-1937),0,Ebay,USER_S,11,Large Merchant,2012-05-24,2012-05-19,10000000094,2012-05-24,65,10000069,9.26,1
-FP-non GTC,0,14,130,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Transportation,Railroadiana & Trains,0,Ebay,USER_S,14,Regular,2012-01-10,2012-01-06,10000000095,2012-01-10,130,10000070,16.26,1
-FP-GTC,0,14,164,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,CPUProcessors,0,Ebay,USER_S,14,Regular,2012-05-17,2012-05-12,10000000096,2012-05-17,164,10000071,157.14,1
-FP-GTC,0,11,216,0,25,Sports Memorabilia,2013-02-20 23:11:43,USER_Y,Sports MeCards & Fan Shop,Cards,Hockey,0,Ebay,USER_S,11,Large Merchant,2012-02-03,2012-02-03,10000000097,2012-02-03,216,10000072,1.88,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2012-05-05,2012-05-05,10000000098,2012-05-05,223,10000073,12.04,1
-FP-non GTC,0,14,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,14,Regular,2012-11-26,2012-11-24,10000000099,2012-11-26,223,10000074,189.23,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2012-08-30,2012-08-25,10000000100,2012-08-30,223,10000075,73.26,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2012-05-21,2012-05-19,10000000101,2012-05-21,223,10000076,4.13,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2012-06-06,2012-06-02,10000000102,2012-06-06,223,10000077,290.72,1
-FP-non GTC,0,5,223,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,5,Occasional,2012-06-16,2012-06-16,10000000103,2012-06-16,223,10000078,265.56,1
-FP-GTC,15,5,279,15,3,Books,2013-09-10 16:52:46,USER_Y,BookMagazines,Children's Books,Children's Books,15,Australia,USER_S,5,Occasional,2012-05-15,2012-05-12,10000000104,2012-05-15,279,10000079,5.91,1
-Auction,0,5,314,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Other,0,Ebay,USER_S,5,Occasional,2012-12-05,2012-12-01,10000000106,2012-12-05,314,10000080,319.79,1
-Auction,211,5,314,211,9,ClothinShoes & Accessories,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Womens' Clothing,Other,211,Philippines,USER_X,5,Occasional,2012-03-25,2012-03-24,10000000107,2012-03-25,314,10000081,246,1
-Auction,211,5,314,211,9,ClothinShoes & Accessories,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Womens' Clothing,Other,211,Philippines,USER_X,5,Occasional,2012-03-09,2012-03-03,10000000108,2012-03-09,314,10000082,20.35,1
-Auction,0,5,314,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Other,0,Ebay,USER_S,5,Occasional,2012-05-06,2012-05-05,10000000109,2012-05-06,314,10000083,36.7,1
-Auction,0,13,533,0,10,Coins,2012-06-19 21:15:09,USER_Y,Coins & Paper Money,Coins: World,Africa,0,Ebay,USER_S,13,Entrepreneur,2012-06-15,2012-06-09,10000000110,2012-06-15,533,10000084,101.79,1
-ABIN,0,5,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,5,Occasional,2012-03-14,2012-03-10,10000000113,2012-03-14,1349,10000085,47.71,1
-ABIN,0,5,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,5,Occasional,2012-05-20,2012-05-19,10000000115,2012-05-20,1349,10000086,3.49,1
-ABIN,0,13,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,13,Entrepreneur,2012-05-17,2012-05-12,10000000117,2012-05-17,1349,10000087,46.44,1
-ABIN,0,13,1349,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,13,Entrepreneur,2012-03-11,2012-03-10,10000000118,2012-03-11,1349,10000088,4.54,1
-ABIN,0,14,1357,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Decorative Collectible Brands,0,Ebay,USER_S,14,Regular,2012-01-10,2012-01-06,10000000119,2012-01-10,1357,10000089,3.18,1
-FP-GTC,0,14,1504,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Electrical & Test Equipment,Test Equipment,0,Ebay,USER_S,14,Regular,2012-04-13,2012-04-07,10000000120,2012-04-13,1504,10000090,86.58,1
-FP-GTC,0,13,4943,0,28,Toys,2013-04-28 20:37:19,USER_Y,Toys & Hobbies,Diecast & Toy Vehicles,CarTrucks & Vans,0,Ebay,USER_S,13,Entrepreneur,2012-07-22,2012-07-21,10000000121,2012-07-22,4943,10000091,12.85,1
-ABIN,0,13,6762,0,-999,Unknown,2008-06-03 13:36:00,USER_Y,Unknown,Unknown,Unknown,0,Ebay,USER_S,13,Entrepreneur,2012-06-15,2012-06-09,10000000122,2012-06-15,6762,10000092,16.26,1
-Auction,3,13,9426,3,31,Mobile & Home Phones,2012-05-01 08:54:26,USER_Y,Mobile Phones & Communication,Home Phones & Accessories,Phone Accessories,3,\u82f1\u56fd,USER_S,13,Entrepreneur,2012-06-15,2012-06-09,10000000123,2012-06-15,9426,10000093,21.14,1
-FP-non GTC,0,14,10866,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Animals,Farm & Countryside,0,Ebay,USER_S,14,Regular,2012-06-01,2012-05-26,10000000124,2012-06-01,10866,10000094,20.6,1
-Auction,0,13,11554,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Jeans,0,Ebay,USER_S,13,Entrepreneur,2012-05-17,2012-05-12,10000000125,2012-05-17,11554,10000095,246,1
-FP-GTC,0,14,11848,0,37,Health & Beauty,2012-06-19 21:15:09,USER_Y,Health & Beauty,Fragrances,Women,0,Ebay,USER_S,14,Regular,2012-04-26,2012-04-21,10000000126,2012-04-26,11848,10000096,109,1
-Auction,0,13,13836,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Spoons,0,Ebay,USER_S,13,Entrepreneur,2012-07-28,2012-07-28,10000000127,2012-07-28,13836,10000097,39.41,1
-Auction,0,14,13836,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Spoons,0,Ebay,USER_S,14,Regular,2012-03-12,2012-03-10,10000000128,2012-03-12,13836,10000098,16.26,1
-FP-GTC,0,13,13987,0,11,Collectibles,2008-06-03 13:36:00,USER_Y,Collectibles,Paper,Booklets,0,Ebay,USER_S,13,Entrepreneur,2012-04-20,2012-04-14,10000000129,2012-04-20,13987,10000099,112.56,1
-Auction,0,14,15687,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Men's Clothing,T-Shirts,0,Ebay,USER_S,14,Regular,2012-03-18,2012-03-17,10000000130,2012-03-18,15687,10000100,184.21,1
-Auction,0,11,15687,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Men's Clothing,T-Shirts,0,Ebay,USER_S,11,Large Merchant,2012-05-17,2012-05-12,10000000131,2012-05-17,15687,10000001,27.48,1
-FP-non GTC,3,12,16145,3,12,Computers,2013-02-20 23:11:43,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Other Components & Parts,3,\u82f1\u56fd,USER_S,12,Merchant,2012-06-23,2012-06-23,10000000132,2012-06-23,16145,10000002,26.45,1
-FP-non GTC,0,13,16145,0,12,Computers,2013-02-20 23:11:43,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Other,0,Ebay,USER_S,13,Entrepreneur,2012-05-22,2012-05-19,10000000134,2012-05-22,16145,10000003,415.73,1
-ABIN,0,5,16509,0,28,Toys,2012-09-11 20:26:04,USER_Y,Toys & Hobbies,Model Railroads & Trains,S Scale,0,Ebay,USER_S,5,Occasional,2012-01-25,2012-01-20,10000000135,2012-01-25,16509,10000004,56.36,1
-ABIN,0,5,16509,0,28,Toys,2012-09-11 20:26:04,USER_Y,Toys & Hobbies,Model Railroads & Trains,S Scale,0,Ebay,USER_S,5,Occasional,2012-06-12,2012-06-09,10000000136,2012-06-12,16509,10000005,2.44,1
-FP-GTC,0,14,20485,0,36,Home & Living,2012-05-01 08:54:26,USER_Y,Home & Garden,Furniture,Other,0,Ebay,USER_S,14,Regular,2012-05-17,2012-05-12,10000000137,2012-05-17,20485,10000006,269.76,1
-FP-GTC,101,12,20485,101,36,Mobili per la casa,2008-06-03 13:36:00,USER_Y,CasArredamento e Bricolage,Cucina,Altro per cucina,101,Italy,USER_S,12,Merchant,2012-08-03,2012-07-28,10000000139,2012-08-03,20485,10000007,109,1
-FP-GTC,101,12,20485,101,36,Mobili per la casa,2008-06-03 13:36:00,USER_Y,CasArredamento e Bricolage,Cucina,Altro per cucina,101,Italy,USER_S,12,Merchant,2012-05-17,2012-05-12,10000000140,2012-05-17,20485,10000008,101.79,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2012-08-21,2012-08-18,10000000141,2012-08-21,23446,10000009,246,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2012-08-21,2012-08-18,10000000142,2012-08-21,23446,10000010,189.23,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2012-04-18,2012-04-14,10000000143,2012-04-18,23446,10000011,15.65,1
-Auction,23,14,23446,23,9,Vtements et Accessoires,2012-06-19 21:15:09,USER_Y,Mode & Accessoires,Chaussures de femme,Sandales & Sandalettes,23,Belgium (French),USER_S,14,Regular,2012-06-16,2012-06-16,10000000144,2012-06-16,23446,10000012,28.23,1
-FP-GTC,0,5,24541,0,25,Sports Memorabilia,2013-07-16 08:23:09,USER_Y,Sports MeCards & Fan Shop,Fan Apparel & Souvenirs,College-NCAA,0,Ebay,USER_S,5,Occasional,2012-01-09,2012-01-06,10000000145,2012-01-09,24541,10000013,16.26,1
-FP-GTC,0,5,26262,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Advertising,Food & Beverage,0,Ebay,USER_S,5,Occasional,2012-05-03,2012-04-28,10000000146,2012-05-03,26262,10000014,122.78,1
-FP-GTC,3,14,30059,3,21,Photography,2012-09-11 20:26:04,USER_Y,Cameras & Photography,Lenses & Filters,Lens AdapterMounts & Tubes,3,\u82f1\u56fd,USER_S,14,Regular,2012-05-20,2012-05-19,10000000147,2012-05-20,30059,10000015,172.03,1
-Auction,3,14,31387,3,17,Jewellery & Watches,2013-04-28 20:37:19,USER_Y,Jewellery & Watches,Watches,Wristwatches,3,\u82f1\u56fd,USER_S,14,Regular,2012-03-12,2012-03-10,10000000149,2012-03-12,31387,10000016,42.99,1
-Auction,3,14,31387,3,17,Jewellery & Watches,2013-04-28 20:37:19,USER_Y,Jewellery & Watches,Watches,Wristwatches,3,\u82f1\u56fd,USER_S,14,Regular,2012-05-19,2012-05-19,10000000150,2012-05-19,31387,10000017,207.5,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2012-04-21,2012-04-21,10000000151,2012-04-21,31519,10000018,5.91,1
-FP-GTC,3,14,31519,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,3,\u82f1\u56fd,USER_S,14,Regular,2012-05-18,2012-05-12,10000000152,2012-05-18,31519,10000019,39.41,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2012-06-16,2012-06-16,10000000155,2012-06-16,31519,10000020,16.26,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2012-06-11,2012-06-09,10000000156,2012-06-11,31519,10000021,16.26,1
-FP-GTC,0,14,31519,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,0,Ebay,USER_S,14,Regular,2012-03-31,2012-03-31,10000000157,2012-03-31,31519,10000022,78.48,1
-FP-GTC,3,14,31519,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Laptop & Desktop Accessories,Laptop Cases & Bags,3,\u82f1\u56fd,USER_S,14,Regular,2012-11-12,2012-11-10,10000000158,2012-11-12,31519,10000023,190.22,1
-FP-GTC,100,12,35570,100,8,Auto - Parts,2012-06-19 21:15:09,USER_Y,eBay Motors,Parts & Accessories,Motorcycle Parts,100,Ebaymotors,USER_S,12,Merchant,2012-04-06,2012-03-31,10000000161,2012-04-06,35570,10000024,2.44,1
-Auction,0,5,36250,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Hunting,Decoys,0,Ebay,USER_S,5,Occasional,2012-11-01,2012-10-27,10000000163,2012-11-01,36250,10000025,7.12,1
-FP-non GTC,0,14,38238,0,36,Home & Living,2012-09-11 20:26:04,USER_Y,Home & Garden,Home Decor,Other,0,Ebay,USER_S,14,Regular,2012-02-06,2012-02-03,10000000165,2012-02-06,38238,10000026,36.7,1
-FP-GTC,3,14,40059,3,33,Consumer Electronics - Audio,2012-05-01 08:57:38,USER_Y,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,3,\u82f1\u56fd,USER_S,14,Regular,2012-06-16,2012-06-16,10000000166,2012-06-16,40059,10000027,35.72,1
-FP-GTC,3,14,40059,3,33,Consumer Electronics - Audio,2012-05-01 08:57:38,USER_Y,Mobile Phones & Communication,Radio Communication Equipment,Parts & Accessories,3,\u82f1\u56fd,USER_S,14,Regular,2012-09-12,2012-09-08,10000000167,2012-09-12,40059,10000028,3.49,1
-FP-GTC,0,13,41940,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,0,Ebay,USER_S,13,Entrepreneur,2012-04-16,2012-04-14,10000000168,2012-04-16,41940,10000029,223.63,1
-FP-GTC,0,13,41940,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Manufacturing & Metalworking,Metalworking Tooling,0,Ebay,USER_S,13,Entrepreneur,2012-11-01,2012-10-27,10000000169,2012-11-01,41940,10000030,265.56,1
-FP-non GTC,0,13,43479,0,21,Photo,2012-09-11 20:26:04,USER_Y,Cameras & Photo,Film Photography,Other,0,Ebay,USER_S,13,Entrepreneur,2012-06-14,2012-06-09,10000000170,2012-06-14,43479,10000031,62.02,1
-FP-GTC,0,12,44079,0,24,Sporting Goods,2013-04-28 20:37:19,USER_Y,Sporting Goods,Exercise & Fitness,GyWorkout & Yoga,0,Ebay,USER_S,12,Merchant,2012-11-12,2012-11-10,10000000172,2012-11-12,44079,10000032,46.44,1
-Auction,101,14,45238,101,9,Vestiti ed Accessori,2012-09-11 20:27:22,USER_Y,Abbigliamento e accessori,Donna: Accessori,SciarpFoulard e Scialli,101,Italy,USER_S,14,Regular,2012-03-22,2012-03-17,10000000173,2012-03-22,45238,10000033,132.33,1
-Auction,0,13,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,13,Entrepreneur,2012-05-22,2012-05-19,10000000177,2012-05-22,45333,10000034,448.8,1
-FP-non GTC,0,14,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,14,Regular,2012-03-10,2012-03-10,10000000178,2012-03-10,45333,10000035,207.5,1
-FP-non GTC,0,14,45333,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Flats & Oxfords,0,Ebay,USER_S,14,Regular,2012-05-17,2012-05-12,10000000179,2012-05-17,45333,10000036,190.22,1
-FP-GTC,0,14,46575,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,Light Equipment & Tools,Air Tools,0,Ebay,USER_S,14,Regular,2012-09-01,2012-09-01,10000000181,2012-09-01,46575,10000037,16.71,1
-FP-non GTC,0,13,50508,0,21,Photo,2013-02-20 23:11:43,USER_Y,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,0,Ebay,USER_S,13,Entrepreneur,2012-04-18,2012-04-14,10000000182,2012-04-18,50508,10000038,4.13,1
-FP-non GTC,0,13,50508,0,21,Photo,2013-02-20 23:11:43,USER_Y,Cameras & Photo,Camera & Photo Accessories,LCD Hoods,0,Ebay,USER_S,13,Entrepreneur,2012-06-05,2012-06-02,10000000183,2012-06-05,50508,10000039,1.88,1
-FP-GTC,0,13,50677,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,0,Ebay,USER_S,13,Entrepreneur,2012-10-08,2012-10-06,10000000185,2012-10-08,50677,10000040,491.32,1
-FP-GTC,0,5,50677,0,17,JewelrGemWatches,2012-09-11 20:26:04,USER_Y,Jewelry & Watches,Fashion Jewelry,Pins & Brooches,0,Ebay,USER_S,5,Occasional,2012-09-17,2012-09-15,10000000186,2012-09-17,50677,10000041,2.44,1
-Auction,0,14,51582,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Kids' ClothinShoes & Accs,Girls' Clothing (Sizes 4 & Up),0,Ebay,USER_S,14,Regular,2012-02-02,2012-01-27,10000000187,2012-02-02,51582,10000042,56.36,1
-FP-GTC,0,13,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,13,Entrepreneur,2012-08-23,2012-08-18,10000000189,2012-08-23,57013,10000043,15.85,1
-FP-non GTC,0,14,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,14,Regular,2012-08-17,2012-08-11,10000000190,2012-08-17,57013,10000044,2.44,1
-FP-GTC,0,14,57013,0,4,Business (Office & Industrial),2013-04-28 20:37:19,USER_Y,Business & Industrial,MRO & Industrial Supply,Pumps & Plumbing,0,Ebay,USER_S,14,Regular,2012-05-05,2012-05-05,10000000191,2012-05-05,57013,10000045,7.12,1
-Auction,0,14,57784,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Baby & Toddler Clothing,Boys' Clothing (Newborn-5T),0,Ebay,USER_S,14,Regular,2012-03-08,2012-03-03,10000000192,2012-03-08,57784,10000046,35.72,1
-Auction,3,11,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,11,Large Merchant,2012-03-16,2012-03-10,10000000194,2012-03-16,57990,10000047,9.26,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,14,Regular,2012-05-09,2012-05-05,10000000195,2012-05-09,57990,10000048,3.18,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,14,Regular,2012-03-25,2012-03-24,10000000196,2012-03-25,57990,10000049,638.72,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,14,Regular,2012-06-05,2012-06-02,10000000198,2012-06-05,57990,10000050,141.7,1
-ABIN,0,13,57990,0,9,Apparel,2013-09-10 16:52:46,USER_Y,ClothinShoes & Accessories,Men's Clothing,Casual Shirts,0,Ebay,USER_S,13,Entrepreneur,2012-01-10,2012-01-06,10000000199,2012-01-10,57990,10000051,12.19,1
-Auction,3,14,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,14,Regular,2012-06-11,2012-06-09,10000000200,2012-06-11,57990,10000052,132.33,1
-ABIN,3,5,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,5,Occasional,2012-08-23,2012-08-18,10000000201,2012-08-23,57990,10000053,5.48,1
-Auction,3,11,57990,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Men's Clothing,Casual Shirts & Tops,3,\u82f1\u56fd,USER_S,11,Large Merchant,2012-08-20,2012-08-18,10000000203,2012-08-20,57990,10000054,1.88,1
-FP-GTC,0,14,60340,0,42,Entertainment Memorabilia,2008-06-03 13:36:00,USER_Y,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,0,Ebay,USER_S,14,Regular,2012-08-09,2012-08-04,10000000204,2012-08-09,60340,10000055,12.85,1
-FP-GTC,0,14,60340,0,42,Entertainment Memorabilia,2008-06-03 13:36:00,USER_Y,Entertainment Memorabilia,Movie Memorabilia,Pressbooks,0,Ebay,USER_S,14,Regular,2012-06-30,2012-06-30,10000000208,2012-06-30,60340,10000056,62.02,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,\u82f1\u56fd,USER_S,12,Merchant,2012-06-03,2012-06-02,10000000209,2012-06-03,60606,10000057,15.85,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,\u82f1\u56fd,USER_S,12,Merchant,2012-08-14,2012-08-11,10000000211,2012-08-14,60606,10000058,9.26,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,\u82f1\u56fd,USER_S,12,Merchant,2012-08-14,2012-08-11,10000000213,2012-08-14,60606,10000059,16.71,1
-FP-GTC,3,12,60606,3,11,Collectables,2012-06-19 21:15:09,USER_Y,Collectables,Badges/ Patches,Golly Badges,3,\u82f1\u56fd,USER_S,12,Merchant,2012-05-17,2012-05-12,10000000214,2012-05-17,60606,10000060,20.6,1
-Auction,3,5,63861,3,9,Clothing & Accessories,2013-07-16 08:23:09,USER_Y,ClotheShoes & Accessories,Women's Clothing,Dresses,3,\u82f1\u56fd,USER_S,5,Occasional,2012-07-23,2012-07-21,10000000217,2012-07-23,63861,10000061,1.88,1
-ABIN,0,5,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,5,Occasional,2012-07-27,2012-07-21,10000000218,2012-07-27,63861,10000062,141.7,1
-ABIN,0,5,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,5,Occasional,2012-07-29,2012-07-28,10000000219,2012-07-29,63861,10000063,1.88,1
-Others,0,11,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,11,Large Merchant,2012-11-17,2012-11-17,10000000220,2012-11-17,63861,10000064,112.56,1
-Others,0,11,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,11,Large Merchant,2012-10-29,2012-10-27,10000000221,2012-10-29,63861,10000065,94.45,1
-Auction,0,14,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,14,Regular,2012-03-28,2012-03-24,10000000222,2012-03-28,63861,10000066,78.48,1
-ABIN,0,13,63861,0,9,Apparel,2013-07-16 08:23:09,USER_Y,ClothinShoes & Accessories,Women's Clothing,Dresses,0,Ebay,USER_S,13,Entrepreneur,2012-01-30,2012-01-27,10000000223,2012-01-30,63861,10000067,5.48,1
-Auction,3,14,63864,3,9,Clothing & Accessories,2012-06-19 21:15:09,USER_Y,ClotheShoes & Accessories,Women's Clothing,Skirts,3,\u82f1\u56fd,USER_S,14,Regular,2012-01-26,2012-01-20,10000000224,2012-01-26,63864,10000068,28.23,1
-Others,0,13,63889,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Shoes,Mixed Items & Lots,0,Ebay,USER_S,13,Entrepreneur,2012-11-21,2012-11-17,10000000229,2012-11-21,63889,10000069,3.49,1
-FP-GTC,2,11,67698,2,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewellery Packaging & Display,2,Canada,USER_S,11,Large Merchant,2012-07-12,2012-07-07,10000000231,2012-07-12,67698,10000070,15.65,1
-FP-GTC,0,11,67698,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewelry Packaging & Display,0,Ebay,USER_S,11,Large Merchant,2012-06-07,2012-06-02,10000000232,2012-06-07,67698,10000071,5.48,1
-FP-GTC,0,11,67698,0,4,Business (Office & Industrial),2012-09-11 20:26:04,USER_Y,Business & Industrial,Retail & Services,Jewelry Packaging & Display,0,Ebay,USER_S,11,Large Merchant,2012-12-01,2012-12-01,10000000233,2012-12-01,67698,10000072,246,1
-FP-non GTC,0,13,73506,0,11,Collectibles,2012-09-11 20:26:04,USER_Y,Collectibles,Decorative Collectibles,Tea PotSets,0,Ebay,USER_S,13,Entrepreneur,2012-12-02,2012-12-01,10000000234,2012-12-02,73506,10000073,122.78,1
-FP-GTC,0,14,75665,0,16,Home Improvement,2012-09-11 20:26:04,USER_Y,Home & Garden,YarGarden & Outdoor Living,Gardening Supplies,0,Ebay,USER_S,14,Regular,2012-02-01,2012-01-27,10000000235,2012-02-01,75665,10000074,223.63,1
-ABIN,3,5,75708,3,28,Toys & Games,2012-05-01 08:57:38,USER_Y,Toys & Games,Action Figures,TMovies & Video Games,3,\u82f1\u56fd,USER_S,5,Occasional,2012-01-14,2012-01-13,10000000236,2012-01-14,75708,10000075,141.7,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2012-09-16,2012-09-15,10000000237,2012-09-16,80053,10000076,21.14,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2012-08-09,2012-08-04,10000000239,2012-08-09,80053,10000077,55.89,1
-FP-non GTC,0,11,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,11,Large Merchant,2012-12-26,2012-12-22,10000000241,2012-12-26,80053,10000078,51.23,1
-Auction,0,14,80135,0,12,Computers,2013-07-16 08:23:09,USER_Y,Computers/Tablets & Networking,DriveStorage & Blank Media,Blank Media & Accessories,0,Ebay,USER_S,14,Regular,2012-12-31,2012-12-29,10000000242,2012-12-31,80135,10000079,21.72,1
-Auction,3,14,95672,3,9,Clothing & Accessories,2013-07-16 08:23:09,USER_Y,ClotheShoes & Accessories,Women's Shoes,Trainers,3,\u82f1\u56fd,USER_S,14,Regular,2012-03-22,2012-03-17,10000000243,2012-03-22,95672,10000080,204.28,1
-Others,0,11,95672,0,9,Apparel,2013-02-20 23:11:43,USER_Y,ClothinShoes & Accessories,Women's Shoes,Athletic,0,Ebay,USER_S,11,Large Merchant,2012-12-25,2012-12-22,10000000244,2012-12-25,95672,10000081,21.14,1
-Others,0,5,100847,0,3,Books,2008-06-03 13:36:00,USER_Y,Half Books,Half Books,Half Books,0,Ebay,USER_S,5,Occasional,2012-01-15,2012-01-13,10000000245,2012-01-15,100847,10000082,204.28,1
-Others,0,5,100847,0,3,Books,2008-06-03 13:36:00,USER_Y,Half Books,Half Books,Half Books,0,Ebay,USER_S,5,Occasional,2012-05-27,2012-05-26,10000000248,2012-05-27,100847,10000083,122.78,1
-ABIN,3,14,139973,3,32,PC & Video Gaming,2012-09-11 20:26:04,USER_Y,Video Games & Consoles,Games,Games,3,\u82f1\u56fd,USER_S,14,Regular,2012-01-11,2012-01-06,10000000249,2012-01-11,139973,10000084,94.45,1
-ABIN,0,11,139973,0,32,Video Games,2012-06-19 21:15:09,USER_Y,Video Games & Consoles,Video Games,Video Games,0,Ebay,USER_S,11,Large Merchant,2012-02-04,2012-02-03,10000000250,2012-02-04,139973,10000085,86.58,1
-Auction,3,14,150047,3,15,Hobbies & Crafts,2012-06-19 21:15:09,USER_Y,Crafts,Jewellery Making,Findings,3,\u82f1\u56fd,USER_S,14,Regular,2012-12-15,2012-12-15,10000000251,2012-12-15,150047,10000086,56.36,1
-Auction,3,14,150047,3,15,Hobbies & Crafts,2012-06-19 21:15:09,USER_Y,Crafts,Jewellery Making,Findings,3,\u82f1\u56fd,USER_S,14,Regular,2012-06-30,2012-06-30,10000000252,2012-06-30,150047,10000087,290.72,1
-FP-GTC,0,13,155226,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,0,Ebay,USER_S,13,Entrepreneur,2012-12-16,2012-12-15,10000000253,2012-12-16,155226,10000088,60.37,1
-FP-GTC,0,13,155226,0,9,Apparel,2012-09-11 20:26:04,USER_Y,ClothinShoes & Accessories,Women's Clothing,Sweats & Hoodies,0,Ebay,USER_S,13,Entrepreneur,2012-11-11,2012-11-10,10000000254,2012-11-11,155226,10000089,112.56,1
-FP-GTC,0,13,156356,0,11,Collectibles,2008-06-03 13:36:00,USER_Y,Collectibles,Postcards,BuildingArchitecture,0,Ebay,USER_S,13,Entrepreneur,2012-10-08,2012-10-06,10000000256,2012-10-08,156356,10000090,265.56,1
-FP-GTC,0,11,158798,0,28,Toys,2008-09-09 22:08:47,USER_Y,Toys & Hobbies,Vintage & Antique Toys,Spinning Tops,0,Ebay,USER_S,11,Large Merchant,2012-04-26,2012-04-21,10000000257,2012-04-26,158798,10000091,35.72,1
-FP-non GTC,0,13,165888,0,17,JewelrGemWatches,2009-01-12 07:05:17,USER_Y,Jewelry & Watches,Vintage & Antique Jewelry,Costume,0,Ebay,USER_S,13,Entrepreneur,2012-10-06,2012-10-06,10000000258,2012-10-06,165888,10000092,92.98,1
-Auction,3,11,170083,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),3,\u82f1\u56fd,USER_S,11,Large Merchant,2012-12-28,2012-12-22,10000000259,2012-12-28,170083,10000093,28.23,1
-Auction,3,11,170083,3,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,Computer Components & Parts,Memory (RAM),3,\u82f1\u56fd,USER_S,11,Large Merchant,2012-11-06,2012-11-03,10000000260,2012-11-06,170083,10000094,27.48,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,14,Regular,2012-11-06,2012-11-03,10000000261,2012-11-06,175750,10000095,9.26,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,14,Regular,2012-12-27,2012-12-22,10000000262,2012-12-27,175750,10000096,3.18,1
-FP-GTC,0,14,175750,0,36,Home & Living,2012-05-01 08:57:38,USER_Y,Home & Garden,Bedding,Blankets & Throws,0,Ebay,USER_S,14,Regular,2012-01-01,2012-01-01,10000000263,2012-01-01,175750,10000097,12.04,1
-Auction,3,13,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,13,Entrepreneur,2012-08-23,2012-08-18,10000000265,2012-08-23,175750,10000098,20.6,1
-Auction,3,13,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,13,Entrepreneur,2012-07-10,2012-07-07,10000000266,2012-07-10,175750,10000099,12.04,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,14,Regular,2012-08-10,2012-08-04,10000000267,2012-08-10,175750,10000100,4.13,1
-Auction,3,14,175750,3,16,Home,2012-09-11 20:26:04,USER_Y,HomFurniture & DIY,Bedding,Blankets,3,\u82f1\u56fd,USER_S,14,Regular,2012-07-19,2012-07-14,10000000268,2012-07-19,175750,10000201,73.26,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index fdfb322..ea9c701 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -1,23 +1,27 @@
{
-
"uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
"name" : "TEST_KYLIN_FACT",
+ "data_gen" : "1",
"columns" : [ {
"id" : "1",
"name" : "TRANS_ID",
- "datatype" : "bigint"
+ "datatype" : "bigint",
+ "data_gen" : "ID"
}, {
"id" : "2",
"name" : "CAL_DT",
- "datatype" : "date"
+ "datatype" : "date",
+ "data_gen" : "FK,order"
}, {
"id" : "3",
"name" : "LSTG_FORMAT_NAME",
- "datatype" : "string"
+ "datatype" : "string",
+ "data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others"
}, {
"id" : "4",
"name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
+ "datatype" : "bigint",
+ "data_gen" : "FK,null,nullstr=0"
}, {
"id" : "5",
"name" : "LSTG_SITE_ID",
@@ -29,27 +33,33 @@
}, {
"id" : "7",
"name" : "PRICE",
- "datatype" : "decimal"
+ "datatype" : "decimal",
+ "data_gen" : "RAND|.##|-100|1000"
}, {
"id" : "8",
"name" : "ITEM_COUNT",
- "datatype" : "int"
+ "datatype" : "int",
+ "data_gen" : "RAND"
}, {
"id" : "9",
"name" : "SELLER_ID",
- "datatype" : "bigint"
+ "datatype" : "bigint",
+ "data_gen" : "RAND||10000000|10001000"
}, {
"id" : "10",
"name" : "USER_ID",
- "datatype" : "varchar(32)"
+ "datatype" : "varchar(32)",
+ "data_gen" : "RAND,order"
}, {
"id" : "11",
"name" : "BUYER_COUNTRY",
- "datatype" : "string"
+ "datatype" : "string",
+ "data_gen" : "CN|DE|FR|JP|UK|US"
}, {
"id" : "12",
"name" : "SELLER_COUNTRY",
- "datatype" : "string"
+ "datatype" : "string",
+ "data_gen" : "CN|DE|FR|JP|UK|US"
} ],
"database" : "DEFAULT",
"last_modified" : 0
[42/50] [abbrv] kylin git commit: KYLIN-2295 Refactor CI,
blend view cubes into the rest
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
index 655d4e7..d186f76 100644
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
@@ -1,408 +1,408 @@
-{
- "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
-
- "name" : "KYLIN_CAL_DT",
- "columns" : [ {
- "id" : "1",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "2",
- "name" : "YEAR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "QTR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "4",
- "name" : "MONTH_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "5",
- "name" : "WEEK_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "6",
- "name" : "AGE_FOR_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "AGE_FOR_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "8",
- "name" : "AGE_FOR_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "9",
- "name" : "AGE_FOR_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "10",
- "name" : "AGE_FOR_DT_ID",
- "datatype" : "smallint"
- }, {
- "id" : "11",
- "name" : "AGE_FOR_RTL_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "12",
- "name" : "AGE_FOR_RTL_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "13",
- "name" : "AGE_FOR_RTL_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "14",
- "name" : "AGE_FOR_RTL_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "15",
- "name" : "AGE_FOR_CS_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "16",
- "name" : "DAY_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "17",
- "name" : "DAY_OF_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "18",
- "name" : "DAY_OF_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "19",
- "name" : "DAY_OF_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "20",
- "name" : "DAY_OF_WEEK_ID",
- "datatype" : "int"
- }, {
- "id" : "21",
- "name" : "WEEK_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "22",
- "name" : "WEEK_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "23",
- "name" : "MONTH_OF_QTR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "24",
- "name" : "MONTH_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "25",
- "name" : "MONTH_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "26",
- "name" : "QTR_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "27",
- "name" : "QTR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "28",
- "name" : "YEAR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "29",
- "name" : "YEAR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "30",
- "name" : "QTR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "MONTH_END_DT",
- "datatype" : "string"
- }, {
- "id" : "32",
- "name" : "WEEK_END_DT",
- "datatype" : "string"
- }, {
- "id" : "33",
- "name" : "CAL_DT_NAME",
- "datatype" : "string"
- }, {
- "id" : "34",
- "name" : "CAL_DT_DESC",
- "datatype" : "string"
- }, {
- "id" : "35",
- "name" : "CAL_DT_SHORT_NAME",
- "datatype" : "string"
- }, {
- "id" : "36",
- "name" : "YTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "37",
- "name" : "QTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "38",
- "name" : "MTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "39",
- "name" : "WTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "40",
- "name" : "SEASON_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "41",
- "name" : "DAY_IN_YEAR_COUNT",
- "datatype" : "smallint"
- }, {
- "id" : "42",
- "name" : "DAY_IN_QTR_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "43",
- "name" : "DAY_IN_MONTH_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "44",
- "name" : "DAY_IN_WEEK_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "45",
- "name" : "RTL_YEAR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "46",
- "name" : "RTL_QTR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "47",
- "name" : "RTL_MONTH_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "48",
- "name" : "RTL_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "49",
- "name" : "CS_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "50",
- "name" : "CAL_DATE",
- "datatype" : "string"
- }, {
- "id" : "51",
- "name" : "DAY_OF_WEEK",
- "datatype" : "string"
- }, {
- "id" : "52",
- "name" : "MONTH_ID",
- "datatype" : "string"
- }, {
- "id" : "53",
- "name" : "PRD_DESC",
- "datatype" : "string"
- }, {
- "id" : "54",
- "name" : "PRD_FLAG",
- "datatype" : "string"
- }, {
- "id" : "55",
- "name" : "PRD_ID",
- "datatype" : "string"
- }, {
- "id" : "56",
- "name" : "PRD_IND",
- "datatype" : "string"
- }, {
- "id" : "57",
- "name" : "QTR_DESC",
- "datatype" : "string"
- }, {
- "id" : "58",
- "name" : "QTR_ID",
- "datatype" : "string"
- }, {
- "id" : "59",
- "name" : "QTR_IND",
- "datatype" : "string"
- }, {
- "id" : "60",
- "name" : "RETAIL_WEEK",
- "datatype" : "string"
- }, {
- "id" : "61",
- "name" : "RETAIL_YEAR",
- "datatype" : "string"
- }, {
- "id" : "62",
- "name" : "RETAIL_START_DATE",
- "datatype" : "string"
- }, {
- "id" : "63",
- "name" : "RETAIL_WK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "64",
- "name" : "WEEK_IND",
- "datatype" : "string"
- }, {
- "id" : "65",
- "name" : "WEEK_NUM_DESC",
- "datatype" : "string"
- }, {
- "id" : "66",
- "name" : "WEEK_BEG_DATE",
- "datatype" : "string"
- }, {
- "id" : "67",
- "name" : "WEEK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "68",
- "name" : "WEEK_IN_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "69",
- "name" : "WEEK_ID",
- "datatype" : "string"
- }, {
- "id" : "70",
- "name" : "WEEK_BEG_END_DESC_MDY",
- "datatype" : "string"
- }, {
- "id" : "71",
- "name" : "WEEK_BEG_END_DESC_MD",
- "datatype" : "string"
- }, {
- "id" : "72",
- "name" : "YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "73",
- "name" : "YEAR_IND",
- "datatype" : "string"
- }, {
- "id" : "74",
- "name" : "CAL_DT_MNS_1YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "75",
- "name" : "CAL_DT_MNS_2YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "76",
- "name" : "CAL_DT_MNS_1QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "77",
- "name" : "CAL_DT_MNS_2QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "78",
- "name" : "CAL_DT_MNS_1MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "79",
- "name" : "CAL_DT_MNS_2MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "80",
- "name" : "CAL_DT_MNS_1WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "81",
- "name" : "CAL_DT_MNS_2WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "82",
- "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "83",
- "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "84",
- "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "85",
- "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "86",
- "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "87",
- "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "88",
- "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "89",
- "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "90",
- "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "91",
- "name" : "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "92",
- "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "93",
- "name" : "SEASON_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "94",
- "name" : "YTM_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "95",
- "name" : "YTQ_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "96",
- "name" : "YTW_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "97",
- "name" : "KYLIN_CAL_DT_CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "98",
- "name" : "KYLIN_CAL_DT_CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "99",
- "name" : "KYLIN_CAL_DT_UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "100",
- "name" : "KYLIN_CAL_DT_UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "SNOWTEST",
- "last_modified" : 0
+{
+ "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+
+ "name" : "KYLIN_CAL_DT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "CAL_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "2",
+ "name" : "YEAR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "QTR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "4",
+ "name" : "MONTH_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "5",
+ "name" : "WEEK_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "6",
+ "name" : "AGE_FOR_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "AGE_FOR_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "8",
+ "name" : "AGE_FOR_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "9",
+ "name" : "AGE_FOR_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "10",
+ "name" : "AGE_FOR_DT_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "11",
+ "name" : "AGE_FOR_RTL_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "12",
+ "name" : "AGE_FOR_RTL_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "13",
+ "name" : "AGE_FOR_RTL_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "14",
+ "name" : "AGE_FOR_RTL_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "15",
+ "name" : "AGE_FOR_CS_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "16",
+ "name" : "DAY_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "17",
+ "name" : "DAY_OF_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "18",
+ "name" : "DAY_OF_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "19",
+ "name" : "DAY_OF_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "20",
+ "name" : "DAY_OF_WEEK_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "21",
+ "name" : "WEEK_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "22",
+ "name" : "WEEK_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "23",
+ "name" : "MONTH_OF_QTR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "24",
+ "name" : "MONTH_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "25",
+ "name" : "MONTH_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "26",
+ "name" : "QTR_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "27",
+ "name" : "QTR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "28",
+ "name" : "YEAR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "29",
+ "name" : "YEAR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "30",
+ "name" : "QTR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "MONTH_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "32",
+ "name" : "WEEK_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "33",
+ "name" : "CAL_DT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "34",
+ "name" : "CAL_DT_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "35",
+ "name" : "CAL_DT_SHORT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "36",
+ "name" : "YTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "37",
+ "name" : "QTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "38",
+ "name" : "MTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "39",
+ "name" : "WTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "40",
+ "name" : "SEASON_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "41",
+ "name" : "DAY_IN_YEAR_COUNT",
+ "datatype" : "smallint"
+ }, {
+ "id" : "42",
+ "name" : "DAY_IN_QTR_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "43",
+ "name" : "DAY_IN_MONTH_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "44",
+ "name" : "DAY_IN_WEEK_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "45",
+ "name" : "RTL_YEAR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "46",
+ "name" : "RTL_QTR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "47",
+ "name" : "RTL_MONTH_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "48",
+ "name" : "RTL_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "49",
+ "name" : "CS_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "50",
+ "name" : "CAL_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "51",
+ "name" : "DAY_OF_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "52",
+ "name" : "MONTH_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "53",
+ "name" : "PRD_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "54",
+ "name" : "PRD_FLAG",
+ "datatype" : "string"
+ }, {
+ "id" : "55",
+ "name" : "PRD_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "56",
+ "name" : "PRD_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "57",
+ "name" : "QTR_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "58",
+ "name" : "QTR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "59",
+ "name" : "QTR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "60",
+ "name" : "RETAIL_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "61",
+ "name" : "RETAIL_YEAR",
+ "datatype" : "string"
+ }, {
+ "id" : "62",
+ "name" : "RETAIL_START_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "63",
+ "name" : "RETAIL_WK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "64",
+ "name" : "WEEK_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "65",
+ "name" : "WEEK_NUM_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "66",
+ "name" : "WEEK_BEG_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "67",
+ "name" : "WEEK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "68",
+ "name" : "WEEK_IN_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "69",
+ "name" : "WEEK_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "70",
+ "name" : "WEEK_BEG_END_DESC_MDY",
+ "datatype" : "string"
+ }, {
+ "id" : "71",
+ "name" : "WEEK_BEG_END_DESC_MD",
+ "datatype" : "string"
+ }, {
+ "id" : "72",
+ "name" : "YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "73",
+ "name" : "YEAR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "74",
+ "name" : "CAL_DT_MNS_1YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "75",
+ "name" : "CAL_DT_MNS_2YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "76",
+ "name" : "CAL_DT_MNS_1QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "77",
+ "name" : "CAL_DT_MNS_2QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "78",
+ "name" : "CAL_DT_MNS_1MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "79",
+ "name" : "CAL_DT_MNS_2MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "80",
+ "name" : "CAL_DT_MNS_1WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "81",
+ "name" : "CAL_DT_MNS_2WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "82",
+ "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "83",
+ "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "84",
+ "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "85",
+ "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "86",
+ "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "87",
+ "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "88",
+ "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "89",
+ "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "90",
+ "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "91",
+ "name" : "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "92",
+ "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "93",
+ "name" : "SEASON_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "94",
+ "name" : "YTM_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "95",
+ "name" : "YTQ_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "96",
+ "name" : "YTW_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "97",
+ "name" : "KYLIN_CAL_DT_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "98",
+ "name" : "KYLIN_CAL_DT_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "99",
+ "name" : "KYLIN_CAL_DT_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "100",
+ "name" : "KYLIN_CAL_DT_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
index abee6d1..c101bd1 100644
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
@@ -1,152 +1,152 @@
-{
- "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
-
- "name" : "KYLIN_CATEGORY_GROUPINGS",
- "columns" : [ {
- "id" : "1",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "LEAF_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "3",
- "name" : "SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "4",
- "name" : "CATEG_BUSN_MGR",
- "datatype" : "string"
- }, {
- "id" : "5",
- "name" : "CATEG_BUSN_UNIT",
- "datatype" : "string"
- }, {
- "id" : "6",
- "name" : "REGN_CATEG",
- "datatype" : "string"
- }, {
- "id" : "7",
- "name" : "USER_DEFINED_FIELD1",
- "datatype" : "string"
- }, {
- "id" : "8",
- "name" : "USER_DEFINED_FIELD3",
- "datatype" : "string"
- }, {
- "id" : "9",
- "name" : "KYLIN_GROUPINGS_CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "10",
+{
+ "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
+
+ "name" : "KYLIN_CATEGORY_GROUPINGS",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "LEAF_CATEG_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "LEAF_CATEG_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "3",
+ "name" : "SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "4",
+ "name" : "CATEG_BUSN_MGR",
+ "datatype" : "string"
+ }, {
+ "id" : "5",
+ "name" : "CATEG_BUSN_UNIT",
+ "datatype" : "string"
+ }, {
+ "id" : "6",
+ "name" : "REGN_CATEG",
+ "datatype" : "string"
+ }, {
+ "id" : "7",
+ "name" : "USER_DEFINED_FIELD1",
+ "datatype" : "string"
+ }, {
+ "id" : "8",
+ "name" : "USER_DEFINED_FIELD3",
+ "datatype" : "string"
+ }, {
+ "id" : "9",
+ "name" : "KYLIN_GROUPINGS_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "10",
"name" : "KYLIN_GROUPINGS_UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "11",
- "name" : "KYLIN_GROUPINGS_CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "12",
+ "datatype" : "string"
+ }, {
+ "id" : "11",
+ "name" : "KYLIN_GROUPINGS_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "12",
"name" : "KYLIN_GROUPINGS_UPD_USER",
- "datatype" : "string"
- }, {
- "id" : "13",
- "name" : "META_CATEG_ID",
- "datatype" : "decimal"
- }, {
- "id" : "14",
- "name" : "META_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "15",
- "name" : "CATEG_LVL2_ID",
- "datatype" : "decimal"
- }, {
- "id" : "16",
- "name" : "CATEG_LVL3_ID",
- "datatype" : "decimal"
- }, {
- "id" : "17",
- "name" : "CATEG_LVL4_ID",
- "datatype" : "decimal"
- }, {
- "id" : "18",
- "name" : "CATEG_LVL5_ID",
- "datatype" : "decimal"
- }, {
- "id" : "19",
- "name" : "CATEG_LVL6_ID",
- "datatype" : "decimal"
- }, {
- "id" : "20",
- "name" : "CATEG_LVL7_ID",
- "datatype" : "decimal"
- }, {
- "id" : "21",
- "name" : "CATEG_LVL2_NAME",
- "datatype" : "string"
- }, {
- "id" : "22",
- "name" : "CATEG_LVL3_NAME",
- "datatype" : "string"
- }, {
- "id" : "23",
- "name" : "CATEG_LVL4_NAME",
- "datatype" : "string"
- }, {
- "id" : "24",
- "name" : "CATEG_LVL5_NAME",
- "datatype" : "string"
- }, {
- "id" : "25",
- "name" : "CATEG_LVL6_NAME",
- "datatype" : "string"
- }, {
- "id" : "26",
- "name" : "CATEG_LVL7_NAME",
- "datatype" : "string"
- }, {
- "id" : "27",
- "name" : "CATEG_FLAGS",
- "datatype" : "decimal"
- }, {
- "id" : "28",
- "name" : "ADULT_CATEG_YN",
- "datatype" : "string"
- }, {
- "id" : "29",
- "name" : "DOMAIN_ID",
- "datatype" : "decimal"
- }, {
- "id" : "30",
- "name" : "USER_DEFINED_FIELD5",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "VCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "32",
- "name" : "GCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "33",
- "name" : "MOVE_TO",
- "datatype" : "decimal"
- }, {
- "id" : "34",
- "name" : "SAP_CATEGORY_ID",
- "datatype" : "decimal"
- }, {
- "id" : "35",
- "name" : "SRC_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "36",
- "name" : "BSNS_VRTCL_NAME",
- "datatype" : "string"
- } ],
- "database" : "SNOWTEST",
- "last_modified" : 0
+ "datatype" : "string"
+ }, {
+ "id" : "13",
+ "name" : "META_CATEG_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "14",
+ "name" : "META_CATEG_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "15",
+ "name" : "CATEG_LVL2_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "16",
+ "name" : "CATEG_LVL3_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "17",
+ "name" : "CATEG_LVL4_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "18",
+ "name" : "CATEG_LVL5_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "19",
+ "name" : "CATEG_LVL6_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "20",
+ "name" : "CATEG_LVL7_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "21",
+ "name" : "CATEG_LVL2_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "22",
+ "name" : "CATEG_LVL3_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "23",
+ "name" : "CATEG_LVL4_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "24",
+ "name" : "CATEG_LVL5_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "25",
+ "name" : "CATEG_LVL6_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "26",
+ "name" : "CATEG_LVL7_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "27",
+ "name" : "CATEG_FLAGS",
+ "datatype" : "decimal"
+ }, {
+ "id" : "28",
+ "name" : "ADULT_CATEG_YN",
+ "datatype" : "string"
+ }, {
+ "id" : "29",
+ "name" : "DOMAIN_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "30",
+ "name" : "USER_DEFINED_FIELD5",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "VCS_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "32",
+ "name" : "GCS_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "33",
+ "name" : "MOVE_TO",
+ "datatype" : "decimal"
+ }, {
+ "id" : "34",
+ "name" : "SAP_CATEGORY_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "35",
+ "name" : "SRC_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "36",
+ "name" : "BSNS_VRTCL_NAME",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
index 64708c7..2f4d723 100644
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
@@ -1,24 +1,24 @@
-{
- "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882",
-
- "name" : "KYLIN_COUNTRY",
- "columns" : [ {
- "id" : "1",
- "name" : "COUNTRY",
- "datatype" : "string"
- }, {
- "id" : "2",
- "name" : "LATITUDE",
- "datatype" : "double"
- }, {
- "id" : "3",
- "name" : "LONGITUDE",
- "datatype" : "double"
- }, {
- "id" : "4",
- "name" : "NAME",
- "datatype" : "string"
- } ],
- "database" : "SNOWTEST",
- "last_modified" : 0
+{
+ "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882",
+
+ "name" : "KYLIN_COUNTRY",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "COUNTRY",
+ "datatype" : "string"
+ }, {
+ "id" : "2",
+ "name" : "LATITUDE",
+ "datatype" : "double"
+ }, {
+ "id" : "3",
+ "name" : "LONGITUDE",
+ "datatype" : "double"
+ }, {
+ "id" : "4",
+ "name" : "NAME",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
index 4aa82b5..a9dcc08 100644
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
+++ b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
@@ -1,56 +1,56 @@
-{
- "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
-
- "name" : "KYLIN_SALES",
- "columns" : [ {
- "id" : "1",
- "name" : "TRANS_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "PART_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "LSTG_FORMAT_NAME",
- "datatype" : "string"
- }, {
- "id" : "4",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "5",
- "name" : "LSTG_SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "6",
- "name" : "SLR_SEGMENT_CD",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "PRICE",
- "datatype" : "decimal(19,4)"
- }, {
- "id" : "8",
- "name" : "ITEM_COUNT",
- "datatype" : "bigint"
- }, {
- "id" : "9",
- "name" : "SELLER_ID",
- "datatype" : "bigint"
- }, {
- "id" : "10",
- "name" : "BUYER_ID",
- "datatype" : "bigint"
- }, {
- "id" : "11",
- "name" : "OPS_USER_ID",
- "datatype" : "string"
- }, {
- "id" : "12",
- "name" : "OPS_REGION",
- "datatype" : "string"
- } ],
- "database" : "SNOWTEST",
- "last_modified" : 0
-}
+{
+ "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
+
+ "name" : "KYLIN_SALES",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "TRANS_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "PART_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "LSTG_FORMAT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "4",
+ "name" : "LEAF_CATEG_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "5",
+ "name" : "LSTG_SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "6",
+ "name" : "SLR_SEGMENT_CD",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "PRICE",
+ "datatype" : "decimal(19,4)"
+ }, {
+ "id" : "8",
+ "name" : "ITEM_COUNT",
+ "datatype" : "bigint"
+ }, {
+ "id" : "9",
+ "name" : "SELLER_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "10",
+ "name" : "BUYER_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "11",
+ "name" : "OPS_USER_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "12",
+ "name" : "OPS_REGION",
+ "datatype" : "string"
+ } ],
+ "database" : "SNOWTEST",
+ "last_modified" : 0
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index 67b62d5..bfbeb70 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -165,7 +165,6 @@ public class BuildCubeWithEngine {
KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(1.0f);
testInner();
testLeft();
- testViewAsLookup();
KylinConfig.getInstanceFromEnv().setHBaseHFileSizeGB(0.0f);
}
@@ -194,11 +193,6 @@ public class BuildCubeWithEngine {
runTestAndAssertSucceed(testCase);
}
- private void testViewAsLookup() throws Exception {
- String[] testCase = new String[] { "testInnerJoinCubeWithView", "testLeftJoinCubeWithView" };
- runTestAndAssertSucceed(testCase);
- }
-
private void runTestAndAssertSucceed(String[] testCase) throws Exception {
ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
@@ -271,18 +265,18 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
// called by reflection
- private Boolean testInnerJoinCubeWithoutSlr() throws Exception {
-
- final String cubeName = "test_kylin_cube_without_slr_empty";
+ private boolean testLeftJoinCubeWithSlr() throws Exception {
+ String cubeName = "test_kylin_cube_with_slr_left_join_empty";
clearSegment(cubeName);
+
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = 0;
+ long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
long date2 = f.parse("2013-01-01").getTime();
long date3 = f.parse("2013-07-01").getTime();
long date4 = f.parse("2022-01-01").getTime();
- List<String> result = Lists.newArrayList();
+ List<String> result = Lists.newArrayList();
if (fastBuildMode) {
return buildSegment(cubeName, date1, date4);
} else {
@@ -295,96 +289,63 @@ public class BuildCubeWithEngine {
}
}
return false;
-
}
@SuppressWarnings("unused")
// called by reflection
- private boolean testLeftJoinCubeWithoutSlr() throws Exception {
+ private boolean testInnerJoinCubeWithoutSlr() throws Exception {
+
+ final String cubeName = "test_kylin_cube_without_slr_empty";
+ clearSegment(cubeName);
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long date1 = 0;
+ long date2 = f.parse("2013-01-01").getTime();
+ long date3 = f.parse("2013-07-01").getTime();
+ long date4 = f.parse("2022-01-01").getTime();
List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
- clearSegment(cubeName);
-
- long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
- long date2 = f.parse("2012-06-01").getTime();
- long date3 = f.parse("2022-01-01").getTime();
- long date4 = f.parse("2023-01-01").getTime();
if (fastBuildMode) {
return buildSegment(cubeName, date1, date4);
} else {
if (buildSegment(cubeName, date1, date2) == true) {
if (buildSegment(cubeName, date2, date3) == true) {
- if (buildSegment(cubeName, date3, date4) == true) { //empty segment
+ if (buildSegment(cubeName, date3, date4) == true) {
return mergeSegment(cubeName, date1, date3);//don't merge all segments
}
}
}
}
-
return false;
}
@SuppressWarnings("unused")
// called by reflection
- private boolean testLeftJoinCubeWithView() throws Exception {
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_with_view_left_join_empty";
- clearSegment(cubeName);
-
- long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
- long date4 = f.parse("2023-01-01").getTime();
-
- return buildSegment(cubeName, date1, date4);
-
- }
-
- @SuppressWarnings("unused")
- // called by reflection
- private boolean testInnerJoinCubeWithView() throws Exception {
+ private boolean testLeftJoinCubeWithoutSlr() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
- final String cubeName = "test_kylin_cube_with_view_inner_join_empty";
+ final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
clearSegment(cubeName);
long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
+ long date2 = f.parse("2012-06-01").getTime();
+ long date3 = f.parse("2022-01-01").getTime();
long date4 = f.parse("2023-01-01").getTime();
- return buildSegment(cubeName, date1, date4);
-
- }
-
- @SuppressWarnings("unused")
- // called by reflection
- private boolean testLeftJoinCubeWithSlr() throws Exception {
- String cubeName = "test_kylin_cube_with_slr_left_join_empty";
- clearSegment(cubeName);
-
- SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
- f.setTimeZone(TimeZone.getTimeZone("GMT"));
- long date1 = cubeManager.getCube(cubeName).getDescriptor().getPartitionDateStart();
- long date2 = f.parse("2013-01-01").getTime();
- long date3 = f.parse("2013-07-01").getTime();
- long date4 = f.parse("2022-01-01").getTime();
-
- List<String> result = Lists.newArrayList();
if (fastBuildMode) {
return buildSegment(cubeName, date1, date4);
} else {
if (buildSegment(cubeName, date1, date2) == true) {
if (buildSegment(cubeName, date2, date3) == true) {
- if (buildSegment(cubeName, date3, date4) == true) {
+ if (buildSegment(cubeName, date3, date4) == true) { //empty segment
return mergeSegment(cubeName, date1, date3);//don't merge all segments
}
}
}
}
+
return false;
}
@@ -413,10 +374,6 @@ public class BuildCubeWithEngine {
DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST");
jobService.addJob(job);
ExecutableState state = waitForJob(job.getId());
- // if (segment.getCubeDesc().getEngineType() == IEngineAware.ID_MR_V1
- // || segment.getCubeDesc().getStorageType() == IStorageAware.ID_SHARDED_HBASE) {
- // checkHFilesInHBase(segment);
- // }
return Boolean.valueOf(ExecutableState.SUCCEED == state);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
index 7cf072f..48b4452 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
@@ -28,6 +28,7 @@ import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
+import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
@@ -72,10 +73,10 @@ public class H2Database {
try {
tempFile = File.createTempFile("tmp_h2", ".csv");
FileOutputStream tempFileStream = new FileOutputStream(tempFile);
- String normalPath = "/data/" + tableDesc.getIdentity() + ".csv";
- InputStream csvStream = metaMgr.getStore().getResource(normalPath).inputStream;
+ String path = path(tableDesc);
+ InputStream csvStream = metaMgr.getStore().getResource(path).inputStream;
- org.apache.commons.io.IOUtils.copy(csvStream, tempFileStream);
+ IOUtils.copy(csvStream, tempFileStream);
csvStream.close();
tempFileStream.close();
@@ -97,6 +98,13 @@ public class H2Database {
tempFile.delete();
}
+ private String path(TableDesc tableDesc) {
+ if ("EDW.TEST_SELLER_TYPE_DIM".equals(tableDesc.getIdentity())) // it is a view of table below
+ return "/data/" + "EDW.TEST_SELLER_TYPE_DIM_TABLE" + ".csv";
+ else
+ return "/data/" + tableDesc.getIdentity() + ".csv";
+ }
+
private String generateCreateH2TableSql(TableDesc tableDesc, String csvFilePath) {
StringBuilder ddl = new StringBuilder();
StringBuilder csvColumns = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index c2aeabd..3a7b685 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -62,7 +62,7 @@ public class ITKylinQueryTest extends KylinTestBase {
priorities.put(RealizationType.INVERTED_INDEX, 0);
Candidate.setPriorities(priorities);
- joinType = "left";
+ joinType = "inner";
setupAll();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 47b9e37..0a795ef 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -639,7 +639,6 @@ public class KylinTestBase {
// Load H2 Tables (inner join)
H2Database h2DB = new H2Database(h2Connection, config);
h2DB.loadAllTables();
-
}
protected static void clean() {
[24/50] [abbrv] kylin git commit: KYLIN 1875 update cube designer
Posted by li...@apache.org.
KYLIN 1875 update cube designer
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a9230473
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a9230473
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a9230473
Branch: refs/heads/master-cdh5.7
Commit: a9230473e3c3bad774f930ad90f4479dcbe34fa0
Parents: 076fbcf
Author: chenzhx <34...@qq.com>
Authored: Tue Dec 13 15:32:54 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 15 18:57:36 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 5 --
webapp/app/js/controllers/cubeDimensions.js | 75 ++++++++++----------
webapp/app/js/controllers/cubeEdit.js | 60 +++++++++-------
webapp/app/js/controllers/cubeMeasures.js | 21 +-----
webapp/app/js/controllers/modelDataModel.js | 11 ++-
.../cubeDesigner/advanced_settings.html | 3 +-
.../app/partials/cubeDesigner/dimensions.html | 46 ++++--------
webapp/app/partials/cubeDesigner/measures.html | 7 +-
8 files changed, 104 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9230473/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 8192419..e557afa 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -38,12 +38,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
//rowkey
$scope.convertedRowkeys = [];
angular.forEach($scope.cubeMetaFrame.rowkey.rowkey_columns,function(item){
- //var _isDictionaries = item.encoding === "dict"?"true":"false";
- //var version=$scope.getTypeVersion(encoding);
item.encoding=$scope.removeVersion(item.encoding);
- //var _isFixedLength = item.encoding.substring(0,12) === "fixed_length"?"true":"false";//fixed_length:12
- //var _isIntegerLength = item.encoding.substring(0,7) === "integer"?"true":"false";
- //var _isIntLength = item.encoding.substring(0,3) === "int"?"true":"false";
var _encoding = item.encoding;
var _valueLength ;
var baseKey=item.encoding.replace(/:\d+/,'');
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9230473/webapp/app/js/controllers/cubeDimensions.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeDimensions.js b/webapp/app/js/controllers/cubeDimensions.js
index 7cb850b..e2787a5 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -18,7 +18,7 @@
'use strict';
-KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cubesManager,SweetAlert) {
+KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cubesManager,SweetAlert, VdmUtil) {
$scope.cubeManager = cubesManager;
// Available columns list derived from cube data model.
@@ -27,10 +27,6 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
// Columns selected and disabled status bound to UI, group by table.
$scope.selectedColumns = {};
- // Available tables cache: 1st is the fact table, next are lookup tables.
- $scope.availableTables = [];
-
-
/**
* Helper func to get columns that dimensions based on, three cases:
* 1. normal dimension: column array.
@@ -40,17 +36,10 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
var dimCols = function (dim) {
var referredCols = [];
- // Case 3.
if (dim.derived && dim.derived.length) {
referredCols = referredCols.concat(dim.derived);
}
- // Case 2.
- //if (dim.hierarchy && dim.column.length) {
- // referredCols = referredCols.concat(dim.column);
- //}
-
- // Case 1.
if (!dim.derived && dim.column) {
referredCols.push(dim.column);
}
@@ -60,52 +49,66 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
// Dump available columns plus column table name, whether is from lookup table.
$scope.initColumns = function () {
- var factTable = $scope.metaModel.model.fact_table;
+ var rootFactTable = VdmUtil.removeNameSpace($scope.metaModel.model.fact_table);
+
+ if($scope.aliasName.length==0){
+ $scope.aliasName.push(rootFactTable);
+ $scope.aliasTableMap[rootFactTable]=$scope.metaModel.model.fact_table;
+ $scope.tableAliasMap[$scope.metaModel.model.fact_table]=rootFactTable;
+ angular.forEach($scope.metaModel.model.lookups,function(joinTable){
+ if(!joinTable.alias){
+ joinTable.alias=VdmUtil.removeNameSpace(joinTable.table);
+ }
+ $scope.aliasTableMap[joinTable.alias]=joinTable.table;
+ $scope.tableAliasMap[joinTable.table]=joinTable.alias;
+ $scope.aliasName.push(joinTable.alias);
+ });
+ }
// At first dump the columns of fact table.
-// var cols = $scope.getColumnsByTable(factTable);
- var cols = $scope.getDimColumnsByTable(factTable);
+
+ var cols = $scope.getDimColumnsByAlias(rootFactTable);
// Initialize selected available.
var factSelectAvailable = {};
for (var i = 0; i < cols.length; i++) {
- cols[i].table = factTable;
- cols[i].isLookup = false;
+ cols[i].table = rootFactTable;
// Default not selected and not disabled.
factSelectAvailable[cols[i].name] = {name:cols[i].name ,selected: false};
}
- $scope.availableColumns[factTable] = cols;
+ $scope.availableColumns[rootFactTable] = cols;
factSelectAvailable.all=false;
- $scope.selectedColumns[factTable] = factSelectAvailable;
- $scope.availableTables.push(factTable);
-
+ $scope.selectedColumns[rootFactTable] = factSelectAvailable;
+// $scope.availableTables.push(rootFactTable);
+ $scope.availableFactTables.push(rootFactTable);
// Then dump each lookup tables.
var lookups = $scope.metaModel.model.lookups;
for (var j = 0; j < lookups.length; j++) {
- var cols2 = $scope.getDimColumnsByTable(lookups[j].table);
+ if(lookups[j].kind=="FACT"){
+ $scope.availableFactTables.push(lookups[j].alias);
+ }else{
+ $scope.availableLookupTables.push(lookups[j].alias);
+ }
+ var cols2 = $scope.getDimColumnsByAlias(lookups[j].alias);
// Initialize selected available.
var lookupSelectAvailable = {};
for (var k = 0; k < cols2.length; k++) {
- cols2[k].table = lookups[j].table;
- cols2[k].isLookup = true;
+ cols2[k].table = lookups[j].alias;
// Default not selected and not disabled.
lookupSelectAvailable[cols2[k].name] = {name:cols2[k].name,selected: false};
}
- $scope.availableColumns[lookups[j].table] = cols2;
+ $scope.availableColumns[lookups[j].alias] = cols2;
lookupSelectAvailable.all=false;
- $scope.selectedColumns[lookups[j].table] = lookupSelectAvailable;
- if($scope.availableTables.indexOf(lookups[j].table)==-1){
- $scope.availableTables.push(lookups[j].table);
- }
+ $scope.selectedColumns[lookups[j].alias] = lookupSelectAvailable;
}
};
@@ -178,10 +181,6 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
types.push('derived');
}
- //if (dim.hierarchy && dim.column.length) {
- // types.push('hierarchy');
- //}
-
if (!types.length) {
types.push('normal');
}
@@ -259,7 +258,6 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
errorInfo+="\n"+item;
});
if(errors.length){
-// SweetAlert.swal('Warning!', errorInfo, '');
SweetAlert.swal('', errorInfo, 'warning');
return false;
}else{
@@ -384,7 +382,7 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
var selectedCols = $scope.getSelectedCols();
dimList=[];
angular.forEach(selectedCols, function (cols, table) {
- if ($scope.metaModel.model.fact_table == table) {
+ if ($scope.availableFactTables.indexOf(table)!=-1) {
// Fact table: for each selected column, create one normal dimension.
for (var i = 0; i < cols.length; i++) {
dimList.push(Dimension(cols[i].name, table, [cols[i].col], 'normal'));
@@ -404,8 +402,8 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
};
- $scope.autoChange= function(table,name){
- if($scope.metaModel.model.fact_table==table){
+ $scope.autoChange = function(table,name){
+ if($scope.availableFactTables.indexOf(table)!=-1){
if($scope.selectedColumns[table][name].selected==false){
$scope.selectedColumns[table].all=false;
}else{
@@ -437,8 +435,9 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
}
}
}
+
$scope.autoChangeAll= function(table){
- if($scope.metaModel.model.fact_table==table){
+ if($scope.availableFactTables.indexOf(table)!=-1){
if($scope.selectedColumns[table].all==true){
angular.forEach($scope.selectedColumns[table],function(col){
if(typeof col==="object"){
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9230473/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index b901e48..d58f08f 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -22,6 +22,11 @@
KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $location, $templateCache, $interpolate, MessageService, TableService, CubeDescService, CubeService, loadingRequest, SweetAlert, $log, cubeConfig, CubeDescModel, MetaModel, TableModel, ModelDescService, modelsManager, cubesManager, ProjectModel, StreamingModel, StreamingService,VdmUtil) {
$scope.cubeConfig = cubeConfig;
$scope.metaModel = {};
+ $scope.tableAliasMap={};
+ $scope.aliasTableMap={};
+ $scope.aliasName=[];
+ $scope.availableFactTables = [];
+ $scope.availableLookupTables = [];
$scope.modelsManager = modelsManager;
//add or edit ?
@@ -112,7 +117,15 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
}
return filterEncoding;
}
-
+ $scope.getColumnsByAlias = function (alias) {
+ var temp = [];
+ angular.forEach(TableModel.selectProjectTables, function (table) {
+ if (table.name == $scope.aliasTableMap[alias]) {
+ temp = table.columns;
+ }
+ });
+ return temp;
+ };
$scope.getColumnsByTable = function (tableName) {
var temp = [];
angular.forEach(TableModel.selectProjectTables, function (table) {
@@ -124,13 +137,13 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
};
//get columns from model
- $scope.getDimColumnsByTable = function (tableName) {
- if (!tableName) {
+ $scope.getDimColumnsByAlias = function (alias) {
+ if (!alias) {
return [];
}
- var tableColumns = $scope.getColumnsByTable(tableName);
+ var tableColumns = $scope.getColumnsByAlias(alias);
var tableDim = _.find($scope.metaModel.model.dimensions, function (dimension) {
- return dimension.table == tableName
+ return dimension.table == alias
});
if(!tableDim){
return [];
@@ -169,17 +182,18 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
$scope.getAllModelDimMeasureColumns = function () {
var me_columns = [];
if($scope.metaModel.model.metrics){
- angular.forEach($scope.metaModel.model.metrics,function(metric,index){
+ angular.forEach($scope.metaModel.model.metrics,function(metric){
me_columns.push(metric);
})
}
- angular.forEach($scope.metaModel.model.dimensions,function(dimension,index){
+ angular.forEach($scope.metaModel.model.dimensions,function(dimension){
if(dimension.columns){
- me_columns = me_columns.concat(dimension.columns);
+ angular.forEach(dimension.columns,function(column){
+ me_columns = me_columns.concat(dimension.table+"."+column);
+ });
}
- })
-
+ });
return distinct_array(me_columns);
};
@@ -187,7 +201,9 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
var me_columns = [];
angular.forEach($scope.metaModel.model.dimensions,function(dimension,index){
if(dimension.columns){
- me_columns = me_columns.concat(dimension.columns);
+ angular.forEach(dimension.columns,function(column){
+ me_columns = me_columns.concat(dimension.table+"."+column);
+ });
}
})
@@ -209,11 +225,11 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
var me_columns = [];
//add cube dimension column for specific measure
angular.forEach($scope.cubeMetaFrame.dimensions,function(dimension,index){
- if($scope.metaModel.model.fact_table !== dimension.table){
+ if($scope.availableFactTables.indexOf(dimension.table)==-1){
return;
}
if(dimension.column && dimension.derived == null){
- me_columns.push(dimension.column);
+ me_columns.push(dimension.table+"."+dimension.column);
}
});
return me_columns;
@@ -224,12 +240,12 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
var me_columns = [];
angular.forEach($scope.cubeMetaFrame.dimensions,function(dimension,index){
if(dimension.column && dimension.derived == null){
- me_columns.push(dimension.column);
+ me_columns.push(dimension.table+"."+dimension.column);
}
else{
angular.forEach(dimension.derived,function(derived){
- me_columns.push(derived);
+ me_columns.push(dimension.table+"."+derived);
});
}
@@ -256,8 +272,8 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
- $scope.getColumnType = function (_column, table) {
- var columns = $scope.getColumnsByTable(table);
+ $scope.getColumnType = function (_column, alias) {
+ var columns = $scope.getColumnsByAlias(alias);
var type;
angular.forEach(columns, function (column) {
if (_column === column.name) {
@@ -314,10 +330,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
$scope.metaModel.model = _model;
});
}
-
$scope.state.cubeSchema = angular.toJson($scope.cubeMetaFrame, true);
-
-
}
});
@@ -520,7 +533,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
}
function reGenerateRowKey() {
- $log.log("reGen rowkey & agg group");
//var fk_pk = {};
var tmpRowKeyColumns = [];
var tmpAggregationItems = [];//put all aggregation item
@@ -530,7 +542,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
for( var i=0;i<$scope.metaModel.model.lookups.length;i++){
var lookup = $scope.metaModel.model.lookups[i];
- var table = lookup.table;
+ var table = lookup.alias;
pfkMap[table] = {};
for(var j=0;j<lookup.join.primary_key.length;j++){
var pk = lookup.join.primary_key[j];
@@ -543,7 +555,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
//derived column
if (dimension.derived && dimension.derived.length) {
var lookup = _.find($scope.metaModel.model.lookups, function (lookup) {
- return lookup.table == dimension.table
+ return lookup.alias == dimension.table
});
angular.forEach(lookup.join.foreign_key, function (fk, index) {
for (var i = 0; i < tmpRowKeyColumns.length; i++) {
@@ -568,7 +580,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
var tableName = dimension.table;
var columnName = dimension.column;
- var rowkeyColumn = dimension.column;
+ var rowkeyColumn = dimension.table+"."+dimension.column;
if(pfkMap[tableName]&&pfkMap[tableName][columnName]){
//lookup table primary key column as dimension
rowkeyColumn = pfkMap[tableName][columnName];
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9230473/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 18f53c2..67a107b 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -98,24 +98,8 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
if(/topn\.encoding\./.test(configuration)){
var _name=configuration.slice(14);
var item=$scope.newMeasure.function.configuration[configuration];
- //var _isFixedLength = item.substring(0,12) === "fixed_length"?"true":"false";//fixed_length:12
- //var _isIntegerLength = item.substring(0,7) === "integer"?"true":"false";
- //var _isIntLength = item.substring(0,3) === "int"?"true":"false";
var _encoding = item;
var _valueLength = 0 ;
- //if(_isFixedLength !=="false"){
- // _valueLength = item.substring(13,item.length);
- // _encoding = "fixed_length";
- //}
- //if(_isIntLength!="false" && _isIntegerLength=="false" ){
- // _valueLength = item.substring(4,item.length);
- // _encoding = "int";
- //}
- //
- //if(_isIntegerLength!="false" ){
- // _valueLength = item.substring(8,item.length);
- // _encoding = "integer";
- //}
var version=$scope.newMeasure.function.configuration['topn.encoding_version.'+_name]||1;
item=$scope.removeVersion(item);
var baseKey=item.replace(/:\d+/,'');
@@ -369,8 +353,9 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
if($scope.newMeasure.function.parameter.type=="column"&&$scope.newMeasure.function.expression!=="COUNT_DISTINCT"){
var column = $scope.newMeasure.function.parameter.value;
- var colType = $scope.getColumnType(column, $scope.metaModel.model.fact_table); // $scope.getColumnType defined in cubeEdit.js
-
+ if(column){
+ var colType = $scope.getColumnType(VdmUtil.removeNameSpace(column), VdmUtil.getNameSpaceAliasName(column)); // $scope.getColumnType defined in cubeEdit.js
+ }
if(colType==""||!colType){
$scope.newMeasure.function.returntype = "";
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9230473/webapp/app/js/controllers/modelDataModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelDataModel.js b/webapp/app/js/controllers/modelDataModel.js
index 95a4c70..a09101b 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -20,15 +20,20 @@
KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cubeConfig,MetaModel,SweetAlert,ModelGraphService,$log,TableModel,ModelService,loadingRequest,modelsManager,VdmUtil) {
$scope.modelsManager = modelsManager;
+ angular.forEach($scope.modelsManager.selectedModel.lookups,function(joinTable){
+ if(!joinTable.alias){
+ joinTable.alias=VdmUtil.removeNameSpace(joinTable.table);
+ }
+ });
$scope.init = function (){
$scope.FactTable={root:$scope.modelsManager.selectedModel.fact_table};
$scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)]=$scope.modelsManager.selectedModel.fact_table;
$scope.tableAliasMap[$scope.modelsManager.selectedModel.fact_table]=VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table);
$scope.aliasName.push(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table));
angular.forEach($scope.modelsManager.selectedModel.lookups,function(joinTable){
- $scope.aliasTableMap[joinTable.alias]=joinTable.table;
- $scope.tableAliasMap[joinTable.table]=joinTable.alias;
- $scope.aliasName.push(joinTable.alias);
+ $scope.aliasTableMap[joinTable.alias]=joinTable.table;
+ $scope.tableAliasMap[joinTable.table]=joinTable.alias;
+ $scope.aliasName.push(joinTable.alias);
});
}
if($scope.state.mode=='edit'){
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9230473/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index 2728240..438eb1f 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -63,7 +63,6 @@
{{rowkey_column.column}}
</ui-select-choices>
</ui-select>
-
<p ng-if="state.mode=='view'">
{{aggregation_group.includes}}</p>
</td>
@@ -203,7 +202,7 @@
<thead>
<tr>
<th>ID</th>
- <th style="width:200px;">Column</th>
+ <th class="col-xs-4">Column</th>
<th>Encoding</th>
<th>Length</th>
<th>Shard By</th>
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9230473/webapp/app/partials/cubeDesigner/dimensions.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html b/webapp/app/partials/cubeDesigner/dimensions.html
index ed265c4..8c1cdab 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -22,16 +22,6 @@
<div class="dataTables_wrapper form-inline no-footer">
<div class="row">
<div class="col-xs-6" ng-if="state.mode=='edit'">
- <div class="btn-group">
- <button type="button" class="btn btn-primary dropdown-toggle" data-toggle="dropdown" ng-disabled="instance.status=='READY'">
- <i class="fa fa-plus"></i> Add Dimension <span class="ace-icon fa fa-caret-down icon-on-right"></span>
- </button>
- <ul class="dropdown-menu" role="menu">
- <li><a ng-click="addDim('normal')">Normal</a></li>
- <!--<li><a ng-click="addDim('hierarchy')">Hierarchy</a></li>-->
- <li><a ng-click="addDim('derived')">Derived</a></li>
- </ul>
- </div>
<button type="button" class="btn btn-default" ng-disabled="!metaModel.model.fact_table.length||instance.status=='READY'"
ng-click="openAutoGenModal()" >
<i class="fa fa-building-o"></i> Auto Generator
@@ -51,7 +41,7 @@
<th>Name</th>
<th>Table Name</th>
<th>Type</th>
- <th></th>
+ <th>Column</th>
<th ng-if="state.mode=='edit'">Actions</th>
</tr>
</thead>
@@ -67,7 +57,7 @@
</td>
<!--Table Name -->
<td>
- <span tooltip="{{dimension.table == metaModel.model.fact_table ? 'Fact Table' : 'Lookup Table'}}">{{dimension.table}}</span>
+ <span>{{dimension.table}}</span>
</td>
<!--Type-->
<td>
@@ -77,18 +67,12 @@
<td>
<div ng-repeat="t in getDimType(dimension)">
<div ng-switch="t">
- <!--<dl class="dl-horizontal" ng-switch-when="hierarchy">-->
- <!--<dt>Hierarchy</dt>-->
- <!--<dd>{{dimension.column}}</dd>-->
- <!--</dl>-->
- <dl class="dl-horizontal" ng-switch-when="derived">
- <dt>Derived</dt>
- <dd>{{dimension.derived}}</dd>
- </dl>
- <dl class="dl-horizontal" ng-switch-when="normal">
- <dt>Column</dt>
- <dd>{{dimension.column}}</dd>
- </dl>
+ <span ng-switch-when="derived">
+ {{dimension.derived}}
+ </span>
+ <span ng-switch-when="normal">
+ {{dimension.column}}
+ </span>
</div>
</div>
</td>
@@ -156,13 +140,13 @@
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Table Name</b></label>
<div class="col-xs-12 col-sm-6" ng-if="dimType.indexOf('derived') >= 0">
<select class="form-control" required="true" chosen data-placeholder="Select a lookup table"
- ng-model="newDimension.table" ng-options="table for table in availableTables.slice(1)">
+ ng-model="newDimension.table" ng-options="table for table in availableLookupTables">
<option value=""></option>
</select>
</div>
<div class="col-xs-12 col-sm-6" ng-if="dimType.indexOf('derived') == -1">
<select class="form-control" required="true" chosen data-placeholder="Select a table"
- ng-model="newDimension.table" ng-options="table for table in availableTables">
+ ng-model="newDimension.table" ng-options="table for table in availableFactTables">
<option value=""></option>
</select>
</div>
@@ -180,7 +164,7 @@
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Column Name</b></label>
<div class="col-xs-12 col-sm-6">
<select class="form-control" required="true" chosen data-placeholder="Select a column"
- ng-model="newDimension.column" ng-options="column.name as column.name for column in getDimColumnsByTable(newDimension.table)">
+ ng-model="newDimension.column" ng-options="column.name as column.name for column in getDimColumnsByAlias(newDimension.table)">
<option value=""></option>
</select>
</div>
@@ -195,7 +179,7 @@
<div>
<select chosen style="width: 80%;" data-placeholder="Derived Columns.."
ng-model="newDimension.derived[$index]"
- ng-options="columns.name as columns.name for columns in getDimColumnsByTable(newDimension.table)" >
+ ng-options="columns.name as columns.name for columns in getDimColumnsByAlias(newDimension.table)" >
<option value=""></option>
</select>
<button class="pull-right btn btn-xs btn-danger" style="cursor: pointer " tooltip="Delete"
@@ -236,9 +220,9 @@
<div class="box-body">
<ul class="list-unstyled columns-region">
<!--FactTable-->
- <div ng-repeat="table in availableTables track by $index" ng-if="$index == 0" class="panel-default " >
+ <div ng-repeat="table in availableFactTables track by $index" class="panel-default " >
<h4>{{table}}[FactTable]</h4>
- <table class="table table-striped table-hover ng-scope" >
+ <table class="table table-striped table-hover ng-scope">
<tr >
<td class="col-xs-2"><label><input type="checkbox" ng-model="selectedColumns[table].all" ng-change="autoChangeAll(table)">Select All</label></td>
<td class="col-xs-4"><label>Name</label></td>
@@ -262,7 +246,7 @@
</table>
</div>
<!--LookUp Table-->
- <div ng-repeat="table in availableTables track by $index" ng-if="$index > 0" class="panel-default" >
+ <div ng-repeat="table in availableLookupTables track by $index" class="panel-default" >
<h4>{{table}}[LookupTable]</h4>
<table class="table table-striped table-hover ng-scope">
<tr class="row" >
http://git-wip-us.apache.org/repos/asf/kylin/blob/a9230473/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 8961cc6..492220c 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -148,6 +148,7 @@
ng-if="newMeasure.function.parameter.type == 'constant'"
ng-init="newMeasure.function.parameter.value = 1"><b> 1</b></span>
<!--!COUNT_DISTINCT-->
+
<select class="form-control" chosen
ng-if="newMeasure.function.parameter.type == 'column' && (newMeasure.function.expression == 'COUNT_DISTINCT'||newMeasure.function.expression == 'RAW')"
ng-model="newMeasure.function.parameter.value" required
@@ -239,7 +240,7 @@
<thead>
<tr>
<th>ID</th>
- <th>Column</th>
+ <th class="col-xs-5" >Column</th>
<th>Encoding</th>
<th>Length</th>
<th ng-if="state.mode=='edit'"></th>
@@ -257,9 +258,9 @@
</td>
<!--Column Name -->
<td>
- <select class="form-control" chosen ng-if="nextPara.type !== 'constant'" required
+ <select class="form-control col-xs-12" chosen ng-if="nextPara.type !== 'constant'" required
ng-model="groupby_column.name"
- ng-options="column as column for column in getAllModelDimColumns()" style="width:200px;">
+ ng-options="column as column for column in getAllModelDimColumns()" >
<option value="">--Select A Column--</option>
</select>
</td>
[20/50] [abbrv] kylin git commit: minor,
test query of group by pushdown
Posted by li...@apache.org.
minor, test query of group by pushdown
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/520b05f7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/520b05f7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/520b05f7
Branch: refs/heads/master-cdh5.7
Commit: 520b05f744b13ad4cbd37372ae70272099ea48a1
Parents: 8d0b504
Author: Yang Li <li...@apache.org>
Authored: Sat Dec 3 12:33:14 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 15 18:57:36 2016 +0800
----------------------------------------------------------------------
.../src/test/resources/query/sql_subquery/query02.sql | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/520b05f7/kylin-it/src/test/resources/query/sql_subquery/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
index 3ecfb1b..81b4887 100644
--- a/kylin-it/src/test/resources/query/sql_subquery/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
@@ -18,19 +18,14 @@
SELECT
week_beg_dt
- ,sum(price) as sum_price
+ ,sum(price)
FROM
(
select
test_cal_dt.week_beg_dt
,test_kylin_fact.price
from test_kylin_fact
- inner JOIN edw.test_cal_dt as test_cal_dt
- ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
- inner JOIN test_category_groupings
- ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
- inner JOIN edw.test_sites as test_sites
- ON test_kylin_fact.lstg_site_id = test_sites.site_id
+ inner join edw.test_cal_dt as test_cal_dt
+ on test_kylin_fact.cal_dt = test_cal_dt.cal_dt
) t
group by week_beg_dt
-order by week_beg_dt
\ No newline at end of file
[12/50] [abbrv] kylin git commit: KYLIN-1832 HyperLogLog performance
optimization
Posted by li...@apache.org.
KYLIN-1832 HyperLogLog performance optimization
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f05404d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f05404d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f05404d5
Branch: refs/heads/master-cdh5.7
Commit: f05404d5576b52c70cf26eb1bccde1c27cd3852f
Parents: 5303651
Author: xiefan46 <95...@qq.com>
Authored: Fri Dec 9 16:53:04 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Wed Dec 14 11:07:42 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/util/CubingUtils.java | 14 +-
.../apache/kylin/gridtable/UnitTestSupport.java | 22 +-
.../benchmark/GTScannerBenchmark2.java | 4 +-
.../gridtable/AggregationCacheMemSizeTest.java | 4 +-
.../metadata/measure/MeasureCodecTest.java | 4 +-
.../org/apache/kylin/measure/MeasureType.java | 2 +-
.../kylin/measure/MeasureTypeFactory.java | 2 +-
.../kylin/measure/hllc/DenseRegister.java | 91 +++++
.../kylin/measure/hllc/HLLCAggregator.java | 10 +-
.../kylin/measure/hllc/HLLCMeasureType.java | 20 +-
.../kylin/measure/hllc/HLLCSerializer.java | 16 +-
.../measure/hllc/HLLDistinctCountAggFunc.java | 22 +-
.../measure/hllc/HyperLogLogPlusCounter.java | 392 -------------------
.../measure/hllc/HyperLogLogPlusCounterNew.java | 388 ++++++++++++++++++
.../measure/hllc/HyperLogLogPlusCounterOld.java | 392 +++++++++++++++++++
.../org/apache/kylin/measure/hllc/Register.java | 37 ++
.../apache/kylin/measure/hllc/RegisterType.java | 25 ++
.../kylin/measure/hllc/SparseRegister.java | 98 +++++
.../measure/AggregatorMemEstimateTest.java | 4 +-
.../measure/hll/HyperLogLogCounterOldTest.java | 265 +++++++++++++
.../measure/hll/HyperLogLogCounterTest.java | 265 -------------
.../measure/hll2/HyperLogLogCounterNewTest.java | 301 ++++++++++++++
.../hll2/NewHyperLogLogBenchmarkTest.java | 288 ++++++++++++++
.../kylin/engine/mr/common/CubeStatsReader.java | 12 +-
.../kylin/engine/mr/common/CubeStatsWriter.java | 6 +-
.../mr/steps/FactDistinctColumnsReducer.java | 8 +-
.../mr/steps/FactDistinctHiveColumnsMapper.java | 10 +-
.../engine/mr/steps/MergeStatisticsStep.java | 6 +-
.../kylin/engine/mr/steps/CubeSamplingTest.java | 8 +-
.../steps/FactDistinctColumnsReducerTest.java | 4 +-
.../apache/kylin/engine/spark/SparkCubing.java | 28 +-
.../cardinality/ColumnCardinalityMapper.java | 12 +-
.../cardinality/ColumnCardinalityReducer.java | 12 +-
.../ColumnCardinalityReducerTest.java | 4 +-
34 files changed, 2002 insertions(+), 774 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
index 413b907..35139a4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/util/CubingUtils.java
@@ -38,7 +38,7 @@ import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ReadableTable;
@@ -59,7 +59,7 @@ public class CubingUtils {
private static Logger logger = LoggerFactory.getLogger(CubingUtils.class);
- public static Map<Long, HyperLogLogPlusCounter> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) {
+ public static Map<Long, HyperLogLogPlusCounterNew> sampling(CubeDesc cubeDesc, IJoinedFlatTableDesc flatDescIn, Iterable<List<String>> streams) {
final CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(flatDescIn, cubeDesc);
final int rowkeyLength = cubeDesc.getRowkey().getRowKeyColumns().length;
final List<Long> allCuboidIds = new CuboidScheduler(cubeDesc).getAllCuboidIds();
@@ -84,9 +84,9 @@ public class CubingUtils {
return result;
}
});
- final Map<Long, HyperLogLogPlusCounter> result = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
+ final Map<Long, HyperLogLogPlusCounterNew> result = Maps.newHashMapWithExpectedSize(allCuboidIds.size());
for (Long cuboidId : allCuboidIds) {
- result.put(cuboidId, new HyperLogLogPlusCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
+ result.put(cuboidId, new HyperLogLogPlusCounterNew(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
Integer[] cuboidBitSet = new Integer[Long.bitCount(cuboidId)];
long mask = Long.highestOneBit(baseCuboidId);
@@ -118,9 +118,9 @@ public class CubingUtils {
}
}
- for (Map.Entry<Long, HyperLogLogPlusCounter> longHyperLogLogPlusCounterEntry : result.entrySet()) {
- Long cuboidId = longHyperLogLogPlusCounterEntry.getKey();
- HyperLogLogPlusCounter counter = longHyperLogLogPlusCounterEntry.getValue();
+ for (Map.Entry<Long, HyperLogLogPlusCounterNew> longHyperLogLogPlusCounterNewEntry : result.entrySet()) {
+ Long cuboidId = longHyperLogLogPlusCounterNewEntry.getKey();
+ HyperLogLogPlusCounterNew counter = longHyperLogLogPlusCounterNewEntry.getValue();
Hasher hc = hf.newHasher();
final Integer[] cuboidBitSet = allCuboidsBitSet.get(cuboidId);
for (int position = 0; position < cuboidBitSet.length; position++) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
index 3396fd2..6cbf237 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/UnitTestSupport.java
@@ -26,7 +26,7 @@ import java.util.List;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.gridtable.GTInfo.Builder;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.LongMutable;
@@ -106,16 +106,16 @@ public class UnitTestSupport {
String d_01_15 = datePlus("2015-01-15", i * 4);
String d_01_16 = datePlus("2015-01-16", i * 4);
String d_01_17 = datePlus("2015-01-17", i * 4);
- result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14)));
- result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14)));
- result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14)));
- result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14)));
- result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14)));
- result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14)));
- result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14)));
- result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14)));
- result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14)));
- result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounter(14)));
+ result.add(newRec(info, d_01_14, "Yang", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
+ result.add(newRec(info, d_01_14, "Luke", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
+ result.add(newRec(info, d_01_15, "Xu", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
+ result.add(newRec(info, d_01_15, "Dong", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
+ result.add(newRec(info, d_01_15, "Jason", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
+ result.add(newRec(info, d_01_16, "Mahone", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
+ result.add(newRec(info, d_01_16, "Shaofeng", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
+ result.add(newRec(info, d_01_16, "Qianhao", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
+ result.add(newRec(info, d_01_16, "George", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
+ result.add(newRec(info, d_01_17, "Kejia", "Food", new LongMutable(10), new BigDecimal("10.5"), new HyperLogLogPlusCounterNew(14)));
}
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java
index 40a5e01..f80bd24 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/benchmark/GTScannerBenchmark2.java
@@ -34,7 +34,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTScanRequestBuilder;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.gridtable.benchmark.SortedGTRecordGenerator.Randomizer;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
@@ -80,7 +80,7 @@ public class GTScannerBenchmark2 {
gen.addDimension(100, 4, null);
gen.addMeasure(8);
gen.addMeasure(8, new Randomizer() {
- HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(12);
+ HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew(12);
@Override
public int fillRandom(Random rand, byte[] array, int offset) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
index 00c0bd0..66a6b51 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/AggregationCacheMemSizeTest.java
@@ -26,7 +26,7 @@ import org.apache.kylin.measure.basic.LongSumAggregator;
import org.apache.kylin.measure.bitmap.BitmapAggregator;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.hllc.HLLCAggregator;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.datatype.LongMutable;
import org.github.jamm.MemoryMeter;
@@ -105,7 +105,7 @@ public class AggregationCacheMemSizeTest {
private HLLCAggregator createHLLCAggr() {
HLLCAggregator hllcAggregator = new HLLCAggregator(14);
- hllcAggregator.aggregate(new HyperLogLogPlusCounter(14));
+ hllcAggregator.aggregate(new HyperLogLogPlusCounterNew(14));
return hllcAggregator;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
index 18680ec..cd1aa96 100644
--- a/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/MeasureCodecTest.java
@@ -26,7 +26,7 @@ import java.nio.ByteBuffer;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.bitmap.BitmapCounter;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.datatype.LongMutable;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -57,7 +57,7 @@ public class MeasureCodecTest extends LocalFileMetadataTestCase {
DoubleMutable d = new DoubleMutable(1.0);
LongMutable l = new LongMutable(2);
BigDecimal b = new BigDecimal("333.1234");
- HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(16);
+ HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew(16);
hllc.add("1234567");
hllc.add("abcdefg");
BitmapCounter bitmap = new BitmapCounter();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
index de1b442..031636e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureType.java
@@ -36,7 +36,7 @@ import org.apache.kylin.metadata.tuple.TupleInfo;
* MeasureType captures how a kind of aggregation is defined, how it is calculated
* during cube build, and how it is involved in query and storage scan.
*
- * @param <T> the Java type of aggregation data object, e.g. HyperLogLogPlusCounter
+ * @param <T> the Java type of aggregation data object, e.g. HyperLogLogPlusCounterOld
*/
abstract public class MeasureType<T> {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
index c5bd482..d94dec9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -62,7 +62,7 @@ import com.google.common.collect.Maps;
}
</pre>
*
- * @param <T> the Java type of aggregation data object, e.g. HyperLogLogPlusCounter
+ * @param <T> the Java type of aggregation data object, e.g. HyperLogLogPlusCounterOld
*/
abstract public class MeasureTypeFactory<T> {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
new file mode 100644
index 0000000..26ee6ab
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/DenseRegister.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.measure.hllc;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Created by xiefan on 16-12-9.
+ */
+public class DenseRegister implements Register {
+ private int p;
+
+ private int m;
+
+ private byte[] register;
+
+ public DenseRegister(int p) {
+ this.m = 1 << p;
+ this.register = new byte[m];
+ }
+
+ public void set(int pos, byte value) {
+ register[pos] = value;
+ }
+
+ @Override
+ public Byte get(int pos) {
+ return register[pos];
+ }
+
+ @Override
+ public void merge(Register another) {
+ if (another instanceof DenseRegister) {
+ DenseRegister dr = (DenseRegister) another;
+ for (int i = 0; i < register.length; i++) {
+ if (dr.register[i] > register[i])
+ register[i] = dr.register[i];
+ }
+ } else {
+ SparseRegister sr = (SparseRegister) another;
+ Collection<Map.Entry<Integer, Byte>> allValue = sr.getAllValue();
+ for (Map.Entry<Integer, Byte> entry : allValue) {
+ if (entry.getValue() > register[entry.getKey()])
+ register[entry.getKey()] = entry.getValue();
+ }
+ }
+ }
+
+ @Override
+ public void clear() {
+ byte zero = (byte) 0;
+ Arrays.fill(register, zero);
+ }
+
+ @Override
+ public int getSize() {
+ int size = 0;
+ for (int i = 0; i < m; i++) {
+ if (register[i] > 0)
+ size++;
+ }
+ return size;
+ }
+
+ @Override
+ public int getHashCode() {
+ return Arrays.hashCode(register);
+ }
+
+ public byte[] getRawRegister() {
+ return this.register;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
index aea2df1..ca73285 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCAggregator.java
@@ -23,10 +23,10 @@ import org.apache.kylin.measure.MeasureAggregator;
/**
*/
@SuppressWarnings("serial")
-public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounter> {
+public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounterNew> {
final int precision;
- HyperLogLogPlusCounter sum = null;
+ HyperLogLogPlusCounterNew sum = null;
public HLLCAggregator(int precision) {
this.precision = precision;
@@ -38,15 +38,15 @@ public class HLLCAggregator extends MeasureAggregator<HyperLogLogPlusCounter> {
}
@Override
- public void aggregate(HyperLogLogPlusCounter value) {
+ public void aggregate(HyperLogLogPlusCounterNew value) {
if (sum == null)
- sum = new HyperLogLogPlusCounter(value);
+ sum = new HyperLogLogPlusCounterNew(value);
else
sum.merge(value);
}
@Override
- public HyperLogLogPlusCounter getState() {
+ public HyperLogLogPlusCounterNew getState() {
return sum;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
index 0e58dca..481fa4e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCMeasureType.java
@@ -33,15 +33,15 @@ import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.collect.ImmutableMap;
-public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounter> {
+public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounterNew> {
public static final String FUNC_COUNT_DISTINCT = FunctionDesc.FUNC_COUNT_DISTINCT;
public static final String DATATYPE_HLLC = "hllc";
- public static class Factory extends MeasureTypeFactory<HyperLogLogPlusCounter> {
+ public static class Factory extends MeasureTypeFactory<HyperLogLogPlusCounterNew> {
@Override
- public MeasureType<HyperLogLogPlusCounter> createMeasureType(String funcName, DataType dataType) {
+ public MeasureType<HyperLogLogPlusCounterNew> createMeasureType(String funcName, DataType dataType) {
return new HLLCMeasureType(funcName, dataType);
}
@@ -56,7 +56,7 @@ public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounter> {
}
@Override
- public Class<? extends DataTypeSerializer<HyperLogLogPlusCounter>> getAggrDataTypeSerializer() {
+ public Class<? extends DataTypeSerializer<HyperLogLogPlusCounterNew>> getAggrDataTypeSerializer() {
return HLLCSerializer.class;
}
}
@@ -91,13 +91,13 @@ public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounter> {
}
@Override
- public MeasureIngester<HyperLogLogPlusCounter> newIngester() {
- return new MeasureIngester<HyperLogLogPlusCounter>() {
- HyperLogLogPlusCounter current = new HyperLogLogPlusCounter(dataType.getPrecision());
+ public MeasureIngester<HyperLogLogPlusCounterNew> newIngester() {
+ return new MeasureIngester<HyperLogLogPlusCounterNew>() {
+ HyperLogLogPlusCounterNew current = new HyperLogLogPlusCounterNew(dataType.getPrecision());
@Override
- public HyperLogLogPlusCounter valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
- HyperLogLogPlusCounter hllc = current;
+ public HyperLogLogPlusCounterNew valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ HyperLogLogPlusCounterNew hllc = current;
hllc.clear();
for (String v : values) {
if (v != null)
@@ -109,7 +109,7 @@ public class HLLCMeasureType extends MeasureType<HyperLogLogPlusCounter> {
}
@Override
- public MeasureAggregator<HyperLogLogPlusCounter> newAggregator() {
+ public MeasureAggregator<HyperLogLogPlusCounterNew> newAggregator() {
return new HLLCAggregator(dataType.getPrecision());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
index 4d08b6f..1d01abc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLCSerializer.java
@@ -28,10 +28,10 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
* @author yangli9
*
*/
-public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
+public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounterNew> {
// be thread-safe and avoid repeated obj creation
- private ThreadLocal<HyperLogLogPlusCounter> current = new ThreadLocal<HyperLogLogPlusCounter>();
+ private ThreadLocal<HyperLogLogPlusCounterNew> current = new ThreadLocal<HyperLogLogPlusCounterNew>();
private int precision;
@@ -40,7 +40,7 @@ public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
}
@Override
- public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
+ public void serialize(HyperLogLogPlusCounterNew value, ByteBuffer out) {
try {
value.writeRegisters(out);
} catch (IOException e) {
@@ -48,18 +48,18 @@ public class HLLCSerializer extends DataTypeSerializer<HyperLogLogPlusCounter> {
}
}
- private HyperLogLogPlusCounter current() {
- HyperLogLogPlusCounter hllc = current.get();
+ private HyperLogLogPlusCounterNew current() {
+ HyperLogLogPlusCounterNew hllc = current.get();
if (hllc == null) {
- hllc = new HyperLogLogPlusCounter(precision);
+ hllc = new HyperLogLogPlusCounterNew(precision);
current.set(hllc);
}
return hllc;
}
@Override
- public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
- HyperLogLogPlusCounter hllc = current();
+ public HyperLogLogPlusCounterNew deserialize(ByteBuffer in) {
+ HyperLogLogPlusCounterNew hllc = current();
try {
hllc.readRegisters(in);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
index 8f2a0fa..a72ad09 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
@@ -31,21 +31,21 @@ public class HLLDistinctCountAggFunc {
private static final Logger logger = LoggerFactory.getLogger(HLLDistinctCountAggFunc.class);
- public static HyperLogLogPlusCounter init() {
+ public static HyperLogLogPlusCounterNew init() {
return null;
}
- public static HyperLogLogPlusCounter initAdd(Object v) {
+ public static HyperLogLogPlusCounterNew initAdd(Object v) {
if (v instanceof Long) { // holistic case
long l = (Long) v;
return new FixedValueHLLCMockup(l);
} else {
- HyperLogLogPlusCounter c = (HyperLogLogPlusCounter) v;
- return new HyperLogLogPlusCounter(c);
+ HyperLogLogPlusCounterNew c = (HyperLogLogPlusCounterNew) v;
+ return new HyperLogLogPlusCounterNew(c);
}
}
- public static HyperLogLogPlusCounter add(HyperLogLogPlusCounter counter, Object v) {
+ public static HyperLogLogPlusCounterNew add(HyperLogLogPlusCounterNew counter, Object v) {
if (v instanceof Long) { // holistic case
long l = (Long) v;
if (counter == null) {
@@ -58,9 +58,9 @@ public class HLLDistinctCountAggFunc {
return counter;
}
} else {
- HyperLogLogPlusCounter c = (HyperLogLogPlusCounter) v;
+ HyperLogLogPlusCounterNew c = (HyperLogLogPlusCounterNew) v;
if (counter == null) {
- return new HyperLogLogPlusCounter(c);
+ return new HyperLogLogPlusCounterNew(c);
} else {
counter.merge(c);
return counter;
@@ -68,16 +68,16 @@ public class HLLDistinctCountAggFunc {
}
}
- public static HyperLogLogPlusCounter merge(HyperLogLogPlusCounter counter0, Object counter1) {
+ public static HyperLogLogPlusCounterNew merge(HyperLogLogPlusCounterNew counter0, Object counter1) {
return add(counter0, counter1);
}
- public static long result(HyperLogLogPlusCounter counter) {
+ public static long result(HyperLogLogPlusCounterNew counter) {
return counter == null ? 0L : counter.getCountEstimate();
}
@SuppressWarnings("serial")
- private static class FixedValueHLLCMockup extends HyperLogLogPlusCounter {
+ private static class FixedValueHLLCMockup extends HyperLogLogPlusCounterNew {
private Long value = null;
@@ -107,7 +107,7 @@ public class HLLDistinctCountAggFunc {
}
@Override
- public void merge(HyperLogLogPlusCounter another) {
+ public void merge(HyperLogLogPlusCounterNew another) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounter.java
deleted file mode 100644
index 00407f9..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounter.java
+++ /dev/null
@@ -1,392 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.hllc;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.BytesUtil;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-
-/**
- * About compression, test on HLLC data shows
- *
- * - LZF compression ratio is around 65%-80%, fast
- * - GZIP compression ratio is around 41%-46%, very slow
- *
- * @author yangli9
- */
-@SuppressWarnings("serial")
-public class HyperLogLogPlusCounter implements Serializable, Comparable<HyperLogLogPlusCounter> {
-
- private final int p;
- private final int m;
- private final HashFunction hashFunc;
- byte[] registers;
- int singleBucket;
-
- public HyperLogLogPlusCounter() {
- this(10);
- }
-
- public HyperLogLogPlusCounter(int p) {
- this(p, Hashing.murmur3_128());
- }
-
- public HyperLogLogPlusCounter(HyperLogLogPlusCounter another) {
- this(another.p, another.hashFunc);
- merge(another);
- }
-
- /** The larger p is, the more storage (2^p bytes), the better accuracy */
- private HyperLogLogPlusCounter(int p, HashFunction hashFunc) {
- this.p = p;
- this.m = 1 << p;//(int) Math.pow(2, p);
- this.hashFunc = hashFunc;
- this.registers = new byte[m];
- this.singleBucket = -1;
- }
-
- public void clear() {
- byte zero = (byte) 0;
- if (singleBucket == -1) {
- //nothing
- } else if (singleBucket >= 0) {
- registers[singleBucket] = 0;
- } else {
- Arrays.fill(registers, zero);
- }
- singleBucket = -1;
- }
-
- public void add(int value) {
- add(hashFunc.hashInt(value).asLong());
- }
-
- public void add(String value) {
- add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
- }
-
- public void add(byte[] value) {
- add(hashFunc.hashBytes(value).asLong());
- }
-
- public void add(byte[] value, int offset, int length) {
- add(hashFunc.hashBytes(value, offset, length).asLong());
- }
-
- protected void add(long hash) {
- int bucketMask = m - 1;
- int bucket = (int) (hash & bucketMask);
- int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
-
- if (firstOnePos > registers[bucket])
- registers[bucket] = (byte) firstOnePos;
-
- if (singleBucket == -1)
- singleBucket = bucket;
- else
- singleBucket = Integer.MIN_VALUE;
- }
-
- public void merge(HyperLogLogPlusCounter another) {
- assert this.p == another.p;
- assert this.hashFunc == another.hashFunc;
-
- // quick path for single value HLLC
- if (another.singleBucket == -1) {
- return;
- } else if (another.singleBucket >= 0) {
- int b = another.singleBucket;
- if (registers[b] < another.registers[b])
- registers[b] = another.registers[b];
- } else {
- // normal path
- for (int i = 0; i < m; i++) {
- if (registers[i] < another.registers[i])
- registers[i] = another.registers[i];
- }
- }
- singleBucket = Integer.MIN_VALUE;
- }
-
- public long getCountEstimate() {
- return new HLLCSnapshot(this).getCountEstimate();
- }
-
- public int getPrecision() {
- return this.p;
- }
-
- public double getErrorRate() {
- return 1.04 / Math.sqrt(m);
- }
-
- private int size() {
- if (singleBucket == -1) {
- return 0;
- } else if (singleBucket >= 0) {
- return 1;
- } else {
- int size = 0;
- for (int i = 0; i < m; i++) {
- if (registers[i] > 0)
- size++;
- }
- return size;
- }
- }
-
- @Override
- public String toString() {
- return "" + getCountEstimate();
- }
-
- // ============================================================================
-
- // a memory efficient snapshot of HLL registers which can yield count
- // estimate later
- public static class HLLCSnapshot {
- byte p;
- double registerSum;
- int zeroBuckets;
-
- public HLLCSnapshot(HyperLogLogPlusCounter hllc) {
- p = (byte) hllc.p;
- registerSum = 0;
- zeroBuckets = 0;
-
- byte[] registers = hllc.registers;
- for (int i = 0; i < hllc.m; i++) {
- if (registers[i] == 0) {
- registerSum++;
- zeroBuckets++;
- } else {
- registerSum += 1.0 / (1L << registers[i]);
- }
- }
- }
-
- public long getCountEstimate() {
- int m = 1 << p;
- double alpha = 0.7213 / (1 + 1.079 / m);
- double estimate = alpha * m * m / registerSum;
-
- // small cardinality adjustment
- if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
- estimate = m * Math.log(m * 1.0 / zeroBuckets);
- } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
- estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
- }
-
- return Math.round(estimate);
- }
- }
-
- // ============================================================================
-
- public void writeRegisters(final ByteBuffer out) throws IOException {
-
- final int indexLen = getRegisterIndexSize();
- int size = size();
-
- // decide output scheme -- map (3*size bytes) or array (2^p bytes)
- byte scheme;
- if (5 + (indexLen + 1) * size < m) // 5 is max len of vint
- scheme = 0; // map
- else
- scheme = 1; // array
- out.put(scheme);
-
- if (scheme == 0) { // map scheme
- BytesUtil.writeVInt(size, out);
- if (singleBucket == -1) {
- // no non-zero register
- } else if (singleBucket >= 0) {
- writeUnsigned(singleBucket, indexLen, out);
- out.put(registers[singleBucket]);
- } else {
- for (int i = 0; i < m; i++) {
- if (registers[i] > 0) {
- writeUnsigned(i, indexLen, out);
- out.put(registers[i]);
- }
- }
- }
- } else if (scheme == 1) { // array scheme
- out.put(registers);
- } else
- throw new IllegalStateException();
- }
-
- public void readRegisters(ByteBuffer in) throws IOException {
- byte scheme = in.get();
-
- if (scheme == 0) { // map scheme
- clear();
- int size = BytesUtil.readVInt(in);
- if (size > m)
- throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
- int indexLen = getRegisterIndexSize();
- int key = 0;
- for (int i = 0; i < size; i++) {
- key = readUnsigned(in, indexLen);
- registers[key] = in.get();
- }
-
- if (size == 0)
- singleBucket = -1;
- else if (size == 1)
- singleBucket = key;
- else
- singleBucket = Integer.MIN_VALUE;
-
- } else if (scheme == 1) { // array scheme
- in.get(registers);
- singleBucket = Integer.MIN_VALUE;
- } else
- throw new IllegalStateException();
- }
-
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
- int len;
-
- byte scheme = in.get();
- if (scheme == 0) { // map scheme
- int size = BytesUtil.readVInt(in);
- int indexLen = getRegisterIndexSize();
- len = in.position() - mark + (indexLen + 1) * size;
- } else {
- len = in.position() - mark + m;
- }
-
- in.position(mark);
- return len;
- }
-
- public int maxLength() {
- return 1 + m;
- }
-
- public void writeRegistersArray(final ByteBuffer out) {
- out.put(this.registers);
- }
-
- public void readRegistersArray(ByteBuffer in) {
- in.get(registers, 0, m);
- singleBucket = Integer.MIN_VALUE;
- }
-
- private int getRegisterIndexSize() {
- return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
- result = prime * result + p;
- result = prime * result + Arrays.hashCode(registers);
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- HyperLogLogPlusCounter other = (HyperLogLogPlusCounter) obj;
- if (hashFunc == null) {
- if (other.hashFunc != null)
- return false;
- } else if (!hashFunc.equals(other.hashFunc))
- return false;
- if (p != other.p)
- return false;
- if (!Arrays.equals(registers, other.registers))
- return false;
- return true;
- }
-
- @Override
- public int compareTo(HyperLogLogPlusCounter o) {
- if (o == null)
- return 1;
-
- long e1 = this.getCountEstimate();
- long e2 = o.getCountEstimate();
-
- if (e1 == e2)
- return 0;
- else if (e1 > e2)
- return 1;
- else
- return -1;
- }
-
- public static void main(String[] args) throws IOException {
- dumpErrorRates();
- }
-
- static void dumpErrorRates() {
- for (int p = 10; p <= 18; p++) {
- double rate = new HyperLogLogPlusCounter(p).getErrorRate();
- double er = Math.round(rate * 10000) / 100D;
- double er2 = Math.round(rate * 2 * 10000) / 100D;
- double er3 = Math.round(rate * 3 * 10000) / 100D;
- long size = Math.round(Math.pow(2, p));
- System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
- }
- }
-
- /**
- *
- * @param num
- * @param size
- * @param out
- */
- public static void writeUnsigned(int num, int size, ByteBuffer out) {
- for (int i = 0; i < size; i++) {
- out.put((byte) num);
- num >>>= 8;
- }
- }
-
- public static int readUnsigned(ByteBuffer in, int size) {
- int integer = 0;
- int mask = 0xff;
- int shift = 0;
- for (int i = 0; i < size; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
- }
- return integer;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterNew.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterNew.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterNew.java
new file mode 100644
index 0000000..d7329f6
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterNew.java
@@ -0,0 +1,388 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import org.apache.kylin.common.util.BytesUtil;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Collection;
+import java.util.Map;
+
+@SuppressWarnings("serial")
+public class HyperLogLogPlusCounterNew implements Serializable, Comparable<HyperLogLogPlusCounterNew> {
+
+ private int p;
+
+ private int m;
+
+ private HashFunction hashFunc = Hashing.murmur3_128();
+
+ private Register register;
+
+ public static double overflowFactor = 0.01;
+
+ public HyperLogLogPlusCounterNew(int p, RegisterType type, HashFunction hashFunc) {
+ this.p = p;
+ this.m = 1 << p;//(int) Math.pow(2, p);
+ this.hashFunc = hashFunc;
+ if (type == RegisterType.SPARSE) {
+ double over = overflowFactor * m;
+ this.register = new SparseRegister((int) over);
+ } else {
+ this.register = new DenseRegister(p);
+ }
+ }
+
+ public HyperLogLogPlusCounterNew() {
+ this(10, RegisterType.SPARSE, Hashing.murmur3_128());
+ }
+
+ public HyperLogLogPlusCounterNew(int p) {
+ this(p, RegisterType.SPARSE, Hashing.murmur3_128());
+ }
+
+ public HyperLogLogPlusCounterNew(int p, RegisterType type) {
+ this(p, type, Hashing.murmur3_128());
+ }
+
+ public HyperLogLogPlusCounterNew(int p, HashFunction hashFunc) {
+ this(p, RegisterType.SPARSE, hashFunc);
+ }
+
+ public HyperLogLogPlusCounterNew(HyperLogLogPlusCounterNew another) {
+ this(another.p, another.hashFunc);
+ merge(another);
+ }
+
+ public void add(int value) {
+ add(hashFunc.hashInt(value).asLong());
+ }
+
+ public void add(String value) {
+ add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
+ }
+
+ public void add(byte[] value) {
+ add(hashFunc.hashBytes(value).asLong());
+ }
+
+ public void add(byte[] value, int offset, int length) {
+ add(hashFunc.hashBytes(value, offset, length).asLong());
+ }
+
+ protected void add(long hash) {
+ int bucketMask = m - 1;
+ int bucket = (int) (hash & bucketMask);
+ int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
+ Byte b = register.get(bucket);
+ if (b == null || (byte) firstOnePos > b) {
+ register.set(bucket, (byte) firstOnePos);
+ }
+ if (register instanceof SparseRegister) {
+ if (((SparseRegister) register).isOverThreshold()) {
+ register = ((SparseRegister) register).toDense(p);
+ }
+ }
+ }
+
+ public void merge(HyperLogLogPlusCounterNew another) {
+ assert this.p == another.p;
+ assert this.hashFunc == another.hashFunc;
+ if (register instanceof SparseRegister && another.register instanceof SparseRegister) {
+ register.merge(another.register);
+ if (((SparseRegister) register).isOverThreshold()) {
+ register = ((SparseRegister) register).toDense(p);
+ }
+ } else if (register instanceof SparseRegister && another.register instanceof DenseRegister) {
+ register = ((SparseRegister) register).toDense(p);
+ register.merge(another.register);
+ } else {
+ register.merge(another.register);
+ }
+ }
+
+ public long getCountEstimate() {
+ return new HLLCSnapshot(this).getCountEstimate();
+ }
+
+ public int getPrecision() {
+ return this.p;
+ }
+
+ public double getErrorRate() {
+ return 1.04 / Math.sqrt(m);
+ }
+
+ @Override
+ public String toString() {
+ return "" + getCountEstimate();
+ }
+
+ // ============================================================================
+
+ // a memory efficient snapshot of HLL registers which can yield count
+ // estimate later
+ public static class HLLCSnapshot {
+ byte p;
+ double registerSum;
+ int zeroBuckets;
+
+ public HLLCSnapshot(HyperLogLogPlusCounterNew hllc) {
+ p = (byte) hllc.p;
+ registerSum = 0;
+ zeroBuckets = 0;
+ Register register = hllc.getRegister();
+ DenseRegister dr;
+ if (register instanceof SparseRegister) {
+ dr = ((SparseRegister) register).toDense(p);
+ } else {
+ dr = (DenseRegister) register;
+ }
+ byte[] registers = dr.getRawRegister();
+ for (int i = 0; i < hllc.m; i++) {
+ if (registers[i] == 0) {
+ registerSum++;
+ zeroBuckets++;
+ } else {
+ registerSum += 1.0 / (1L << registers[i]);
+ }
+ }
+ }
+
+ public long getCountEstimate() {
+ int m = 1 << p;
+ double alpha = 0.7213 / (1 + 1.079 / m);
+ double estimate = alpha * m * m / registerSum;
+
+ // small cardinality adjustment
+ if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
+ estimate = m * Math.log(m * 1.0 / zeroBuckets);
+ } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
+ estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
+ }
+
+ return Math.round(estimate);
+ }
+ }
+
+ public static void main(String[] args) throws IOException {
+ dumpErrorRates();
+ }
+
+ static void dumpErrorRates() {
+ for (int p = 10; p <= 18; p++) {
+ double rate = new HyperLogLogPlusCounterNew(p, RegisterType.SPARSE).getErrorRate();
+ double er = Math.round(rate * 10000) / 100D;
+ double er2 = Math.round(rate * 2 * 10000) / 100D;
+ double er3 = Math.round(rate * 3 * 10000) / 100D;
+ long size = Math.round(Math.pow(2, p));
+ System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
+ }
+ }
+
+ public Register getRegister() {
+ return register;
+ }
+
+ public void clear() {
+ register.clear();
+ }
+
+ public RegisterType getRegisterType() {
+ if (register instanceof SparseRegister)
+ return RegisterType.SPARSE;
+ else
+ return RegisterType.DENSE;
+ }
+
+ // ============================================================================
+
+ public void writeRegisters(final ByteBuffer out) throws IOException {
+
+ final int indexLen = getRegisterIndexSize();
+ int size = size();
+
+ // decide output scheme -- map (3*size bytes) or array (2^p bytes)
+ byte scheme;
+ //byte type;
+ if (register instanceof SparseRegister || 5 + (indexLen + 1) * size < m) {
+ scheme = 0; //map
+ } else {
+ scheme = 1; // array
+ }
+ out.put(scheme);
+ if (scheme == 0) { // map scheme
+ BytesUtil.writeVInt(size, out);
+ if (register instanceof SparseRegister) { //sparse\u3000register
+ Collection<Map.Entry<Integer, Byte>> allValue = ((SparseRegister) register).getAllValue();
+ for (Map.Entry<Integer, Byte> entry : allValue) {
+ writeUnsigned(entry.getKey(), indexLen, out);
+ out.put(entry.getValue());
+ }
+ } else { //dense register
+ byte[] registers = ((DenseRegister) register).getRawRegister();
+ for (int i = 0; i < m; i++) {
+ if (registers[i] > 0) {
+ writeUnsigned(i, indexLen, out);
+ out.put(registers[i]);
+ }
+ }
+ }
+ } else if (scheme == 1) { // array scheme
+ out.put(((DenseRegister) register).getRawRegister());
+ } else
+ throw new IllegalStateException();
+ }
+
+ public void readRegisters(ByteBuffer in) throws IOException {
+ byte scheme = in.get();
+ if (scheme == 0) { // map scheme
+ clear();
+ int size = BytesUtil.readVInt(in);
+ if (size > m)
+ throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
+ double over = overflowFactor * m;
+ if (size > (int) over) {
+ this.register = new DenseRegister(p);
+ } else {
+ this.register = new SparseRegister((int) over);//default is sparse
+ }
+ int indexLen = getRegisterIndexSize();
+ int key = 0;
+ for (int i = 0; i < size; i++) {
+ key = readUnsigned(in, indexLen);
+ register.set(key, in.get());
+ }
+ } else if (scheme == 1) { // array scheme
+ this.register = new DenseRegister(p);
+ for (int i = 0; i < m; i++) {
+ register.set(i, in.get());
+ }
+ } else
+ throw new IllegalStateException();
+ }
+
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ int len;
+ byte scheme = in.get();
+ if (scheme == 0) { // map scheme
+ int size = BytesUtil.readVInt(in);
+ int indexLen = getRegisterIndexSize();
+ len = in.position() - mark + (indexLen + 1) * size;
+ } else {
+ len = in.position() - mark + m;
+ }
+
+ in.position(mark);
+ return len;
+ }
+
+ public int maxLength() {
+ return 1 + m;
+ }
+
+ private int getRegisterIndexSize() {
+ return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
+ result = prime * result + p;
+ result = prime * result + register.getHashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ HyperLogLogPlusCounterNew other = (HyperLogLogPlusCounterNew) obj;
+ if (hashFunc == null) {
+ if (other.hashFunc != null)
+ return false;
+ } else if (!hashFunc.equals(other.hashFunc))
+ return false;
+ if (p != other.p)
+ return false;
+ if (this.getRegisterType() != other.getRegisterType())
+ return false;
+ if (register.getHashCode() != other.register.getHashCode())
+ return false;
+ return true;
+ }
+
+ @Override
+ public int compareTo(HyperLogLogPlusCounterNew o) {
+ if (o == null)
+ return 1;
+
+ long e1 = this.getCountEstimate();
+ long e2 = o.getCountEstimate();
+
+ if (e1 == e2)
+ return 0;
+ else if (e1 > e2)
+ return 1;
+ else
+ return -1;
+ }
+
+ /**
+ *
+ * @param num
+ * @param size
+ * @param out
+ */
+ public static void writeUnsigned(int num, int size, ByteBuffer out) {
+ for (int i = 0; i < size; i++) {
+ out.put((byte) num);
+ num >>>= 8;
+ }
+ }
+
+ public static int readUnsigned(ByteBuffer in, int size) {
+ int integer = 0;
+ int mask = 0xff;
+ int shift = 0;
+ for (int i = 0; i < size; i++) {
+ integer |= (in.get() << shift) & mask;
+ mask = mask << 8;
+ shift += 8;
+ }
+ return integer;
+ }
+
+ private int size() {
+ return register.getSize();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterOld.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterOld.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterOld.java
new file mode 100644
index 0000000..cb5533e
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HyperLogLogPlusCounterOld.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+import org.apache.kylin.common.util.BytesUtil;
+
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+
+/**
+ * About compression, test on HLLC data shows
+ *
+ * - LZF compression ratio is around 65%-80%, fast
+ * - GZIP compression ratio is around 41%-46%, very slow
+ *
+ * @author yangli9
+ */
+@SuppressWarnings("serial")
+public class HyperLogLogPlusCounterOld implements Serializable, Comparable<HyperLogLogPlusCounterOld> {
+
+ private final int p;
+ private final int m;
+ private final HashFunction hashFunc;
+ byte[] registers;
+ int singleBucket;
+
+ public HyperLogLogPlusCounterOld() {
+ this(10);
+ }
+
+ public HyperLogLogPlusCounterOld(int p) {
+ this(p, Hashing.murmur3_128());
+ }
+
+ public HyperLogLogPlusCounterOld(HyperLogLogPlusCounterOld another) {
+ this(another.p, another.hashFunc);
+ merge(another);
+ }
+
+ /** The larger p is, the more storage (2^p bytes), the better accuracy */
+ private HyperLogLogPlusCounterOld(int p, HashFunction hashFunc) {
+ this.p = p;
+ this.m = 1 << p;//(int) Math.pow(2, p);
+ this.hashFunc = hashFunc;
+ this.registers = new byte[m];
+ this.singleBucket = -1;
+ }
+
+ public void clear() {
+ byte zero = (byte) 0;
+ if (singleBucket == -1) {
+ //nothing
+ } else if (singleBucket >= 0) {
+ registers[singleBucket] = 0;
+ } else {
+ Arrays.fill(registers, zero);
+ }
+ singleBucket = -1;
+ }
+
+ public void add(int value) {
+ add(hashFunc.hashInt(value).asLong());
+ }
+
+ public void add(String value) {
+ add(hashFunc.hashString(value, Charset.defaultCharset()).asLong());
+ }
+
+ public void add(byte[] value) {
+ add(hashFunc.hashBytes(value).asLong());
+ }
+
+ public void add(byte[] value, int offset, int length) {
+ add(hashFunc.hashBytes(value, offset, length).asLong());
+ }
+
+ protected void add(long hash) {
+ int bucketMask = m - 1;
+ int bucket = (int) (hash & bucketMask);
+ int firstOnePos = Long.numberOfLeadingZeros(hash | bucketMask) + 1;
+
+ if (firstOnePos > registers[bucket])
+ registers[bucket] = (byte) firstOnePos;
+
+ if (singleBucket == -1)
+ singleBucket = bucket;
+ else
+ singleBucket = Integer.MIN_VALUE;
+ }
+
+ public void merge(HyperLogLogPlusCounterOld another) {
+ assert this.p == another.p;
+ assert this.hashFunc == another.hashFunc;
+
+ // quick path for single value HLLC
+ if (another.singleBucket == -1) {
+ return;
+ } else if (another.singleBucket >= 0) {
+ int b = another.singleBucket;
+ if (registers[b] < another.registers[b])
+ registers[b] = another.registers[b];
+ } else {
+ // normal path
+ for (int i = 0; i < m; i++) {
+ if (registers[i] < another.registers[i])
+ registers[i] = another.registers[i];
+ }
+ }
+ singleBucket = Integer.MIN_VALUE;
+ }
+
+ public long getCountEstimate() {
+ return new HLLCSnapshot(this).getCountEstimate();
+ }
+
+ public int getPrecision() {
+ return this.p;
+ }
+
+ public double getErrorRate() {
+ return 1.04 / Math.sqrt(m);
+ }
+
+ private int size() {
+ if (singleBucket == -1) {
+ return 0;
+ } else if (singleBucket >= 0) {
+ return 1;
+ } else {
+ int size = 0;
+ for (int i = 0; i < m; i++) {
+ if (registers[i] > 0)
+ size++;
+ }
+ return size;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "" + getCountEstimate();
+ }
+
+ // ============================================================================
+
+ // a memory efficient snapshot of HLL registers which can yield count
+ // estimate later
+ public static class HLLCSnapshot {
+ byte p;
+ double registerSum;
+ int zeroBuckets;
+
+ public HLLCSnapshot(HyperLogLogPlusCounterOld hllc) {
+ p = (byte) hllc.p;
+ registerSum = 0;
+ zeroBuckets = 0;
+
+ byte[] registers = hllc.registers;
+ for (int i = 0; i < hllc.m; i++) {
+ if (registers[i] == 0) {
+ registerSum++;
+ zeroBuckets++;
+ } else {
+ registerSum += 1.0 / (1L << registers[i]);
+ }
+ }
+ }
+
+ public long getCountEstimate() {
+ int m = 1 << p;
+ double alpha = 0.7213 / (1 + 1.079 / m);
+ double estimate = alpha * m * m / registerSum;
+
+ // small cardinality adjustment
+ if (zeroBuckets >= m * 0.07) { // (reference presto's HLL impl)
+ estimate = m * Math.log(m * 1.0 / zeroBuckets);
+ } else if (HyperLogLogPlusTable.isBiasCorrection(m, estimate)) {
+ estimate = HyperLogLogPlusTable.biasCorrection(p, estimate);
+ }
+
+ return Math.round(estimate);
+ }
+ }
+
+ // ============================================================================
+
+ public void writeRegisters(final ByteBuffer out) throws IOException {
+
+ final int indexLen = getRegisterIndexSize();
+ int size = size();
+
+ // decide output scheme -- map (3*size bytes) or array (2^p bytes)
+ byte scheme;
+ if (5 + (indexLen + 1) * size < m) // 5 is max len of vint
+ scheme = 0; // map
+ else
+ scheme = 1; // array
+ out.put(scheme);
+
+ if (scheme == 0) { // map scheme
+ BytesUtil.writeVInt(size, out);
+ if (singleBucket == -1) {
+ // no non-zero register
+ } else if (singleBucket >= 0) {
+ writeUnsigned(singleBucket, indexLen, out);
+ out.put(registers[singleBucket]);
+ } else {
+ for (int i = 0; i < m; i++) {
+ if (registers[i] > 0) {
+ writeUnsigned(i, indexLen, out);
+ out.put(registers[i]);
+ }
+ }
+ }
+ } else if (scheme == 1) { // array scheme
+ out.put(registers);
+ } else
+ throw new IllegalStateException();
+ }
+
+ public void readRegisters(ByteBuffer in) throws IOException {
+ byte scheme = in.get();
+
+ if (scheme == 0) { // map scheme
+ clear();
+ int size = BytesUtil.readVInt(in);
+ if (size > m)
+ throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
+ int indexLen = getRegisterIndexSize();
+ int key = 0;
+ for (int i = 0; i < size; i++) {
+ key = readUnsigned(in, indexLen);
+ registers[key] = in.get();
+ }
+
+ if (size == 0)
+ singleBucket = -1;
+ else if (size == 1)
+ singleBucket = key;
+ else
+ singleBucket = Integer.MIN_VALUE;
+
+ } else if (scheme == 1) { // array scheme
+ in.get(registers);
+ singleBucket = Integer.MIN_VALUE;
+ } else
+ throw new IllegalStateException();
+ }
+
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ int len;
+
+ byte scheme = in.get();
+ if (scheme == 0) { // map scheme
+ int size = BytesUtil.readVInt(in);
+ int indexLen = getRegisterIndexSize();
+ len = in.position() - mark + (indexLen + 1) * size;
+ } else {
+ len = in.position() - mark + m;
+ }
+
+ in.position(mark);
+ return len;
+ }
+
+ public int maxLength() {
+ return 1 + m;
+ }
+
+ /*public void writeRegistersArray(final ByteBuffer out) {
+ out.put(this.registers);
+ }
+
+ public void readRegistersArray(ByteBuffer in) {
+ in.get(registers, 0, m);
+ singleBucket = Integer.MIN_VALUE;
+ }*/
+
+ private int getRegisterIndexSize() {
+ return (p - 1) / 8 + 1; // 2 when p=16, 3 when p=17
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((hashFunc == null) ? 0 : hashFunc.hashCode());
+ result = prime * result + p;
+ result = prime * result + Arrays.hashCode(registers);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ HyperLogLogPlusCounterOld other = (HyperLogLogPlusCounterOld) obj;
+ if (hashFunc == null) {
+ if (other.hashFunc != null)
+ return false;
+ } else if (!hashFunc.equals(other.hashFunc))
+ return false;
+ if (p != other.p)
+ return false;
+ if (!Arrays.equals(registers, other.registers))
+ return false;
+ return true;
+ }
+
+ @Override
+ public int compareTo(HyperLogLogPlusCounterOld o) {
+ if (o == null)
+ return 1;
+
+ long e1 = this.getCountEstimate();
+ long e2 = o.getCountEstimate();
+
+ if (e1 == e2)
+ return 0;
+ else if (e1 > e2)
+ return 1;
+ else
+ return -1;
+ }
+
+ public static void main(String[] args) throws IOException {
+ dumpErrorRates();
+ }
+
+ static void dumpErrorRates() {
+ for (int p = 10; p <= 18; p++) {
+ double rate = new HyperLogLogPlusCounterOld(p).getErrorRate();
+ double er = Math.round(rate * 10000) / 100D;
+ double er2 = Math.round(rate * 2 * 10000) / 100D;
+ double er3 = Math.round(rate * 3 * 10000) / 100D;
+ long size = Math.round(Math.pow(2, p));
+ System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
+ }
+ }
+
+ /**
+ *
+ * @param num
+ * @param size
+ * @param out
+ */
+ public static void writeUnsigned(int num, int size, ByteBuffer out) {
+ for (int i = 0; i < size; i++) {
+ out.put((byte) num);
+ num >>>= 8;
+ }
+ }
+
+ public static int readUnsigned(ByteBuffer in, int size) {
+ int integer = 0;
+ int mask = 0xff;
+ int shift = 0;
+ for (int i = 0; i < size; i++) {
+ integer |= (in.get() << shift) & mask;
+ mask = mask << 8;
+ shift += 8;
+ }
+ return integer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/Register.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/Register.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/Register.java
new file mode 100644
index 0000000..79c4bba
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/Register.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.measure.hllc;
+
+/**
+ * Created by xiefan on 16-12-9.
+ */
+public interface Register {
+
+ void set(int pos, byte value);
+
+ Byte get(int pos);
+
+ void merge(Register another);
+
+ void clear();
+
+ int getSize();
+
+ int getHashCode();
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/RegisterType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/RegisterType.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/RegisterType.java
new file mode 100644
index 0000000..fec9939
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/RegisterType.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.measure.hllc;
+
+/**
+ * Created by xiefan on 16-12-9.
+ */
+public enum RegisterType {
+ SPARSE, DENSE
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
new file mode 100644
index 0000000..d241e81
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/SparseRegister.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.measure.hllc;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Created by xiefan on 16-12-9.
+ */
+public class SparseRegister implements Register {
+
+ private int overThreshold;
+
+ private Map<Integer, Byte> sparseRegister = new TreeMap<>();
+
+ public SparseRegister(int overThreshold) {
+ this.overThreshold = overThreshold;
+ }
+
+ public DenseRegister toDense(int p) {
+ DenseRegister dr = new DenseRegister(p);
+ for (Map.Entry<Integer, Byte> entry : sparseRegister.entrySet()) {
+ dr.set(entry.getKey(), entry.getValue());
+ }
+ return dr;
+ }
+
+ @Override
+ public void set(int pos, byte value) {
+ sparseRegister.put(pos, value);
+ }
+
+ @Override
+ public Byte get(int pos) {
+ return sparseRegister.get(pos);
+ }
+
+ @Override
+ public void merge(Register another) {
+ assert another instanceof SparseRegister;
+ SparseRegister sr = (SparseRegister) another;
+ for (Map.Entry<Integer, Byte> entry : sr.sparseRegister.entrySet()) {
+ Byte v = sparseRegister.get(entry.getKey());
+ if (v == null || entry.getValue() > v)
+ sparseRegister.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ @Override
+ public void clear() {
+ sparseRegister.clear();
+ }
+
+ @Override
+ public int getSize() {
+ return sparseRegister.size();
+ }
+
+ @Override
+ public int getHashCode() {
+ final int prime = 31;
+ int result = 1;
+ for (Map.Entry<Integer, Byte> entry : sparseRegister.entrySet()) {
+ result = prime * result + entry.getKey();
+ result = prime * result + entry.getValue();
+ }
+ return result;
+ }
+
+ public boolean isOverThreshold() {
+ if (this.sparseRegister.size() > overThreshold)
+ return true;
+ return false;
+ }
+
+ public Collection<Map.Entry<Integer, Byte>> getAllValue() {
+ return Collections.unmodifiableCollection(sparseRegister.entrySet());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
index 3adec73..103e721 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/AggregatorMemEstimateTest.java
@@ -26,7 +26,7 @@ import org.apache.kylin.measure.bitmap.BitmapAggregator;
import org.apache.kylin.measure.bitmap.BitmapCounter;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.measure.hllc.HLLCAggregator;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DoubleMutable;
import org.apache.kylin.metadata.datatype.LongMutable;
@@ -94,7 +94,7 @@ public class AggregatorMemEstimateTest extends LocalFileMetadataTestCase {
@Test
public void testAggregatorEstimate() {
HLLCAggregator hllcAggregator = new HLLCAggregator(14);
- hllcAggregator.aggregate(new HyperLogLogPlusCounter(14));
+ hllcAggregator.aggregate(new HyperLogLogPlusCounterNew(14));
BitmapAggregator bitmapAggregator = new BitmapAggregator();
BitmapCounter bitmapCounter = new BitmapCounter();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterOldTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterOldTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterOldTest.java
new file mode 100644
index 0000000..5d17fea
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterOldTest.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hll;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterOld;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @author yangli9
+ *
+ */
+public class HyperLogLogCounterOldTest {
+
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ Random rand1 = new Random(1);
+ Random rand2 = new Random(2);
+ Random rand3 = new Random(3);
+ int errorCount1 = 0;
+ int errorCount2 = 0;
+ int errorCount3 = 0;
+
+ @Test
+ public void testOneAdd() throws IOException {
+ HyperLogLogPlusCounterOld hllc = new HyperLogLogPlusCounterOld(14);
+ HyperLogLogPlusCounterOld one = new HyperLogLogPlusCounterOld(14);
+ for (int i = 0; i < 1000000; i++) {
+ one.clear();
+ one.add(rand1.nextInt());
+ hllc.merge(one);
+ }
+ assertTrue(hllc.getCountEstimate() > 1000000 * 0.9);
+ }
+
+ @Test
+ public void testPeekLength() throws IOException {
+ HyperLogLogPlusCounterOld hllc = new HyperLogLogPlusCounterOld(10);
+ HyperLogLogPlusCounterOld copy = new HyperLogLogPlusCounterOld(10);
+ byte[] value = new byte[10];
+ for (int i = 0; i < 200000; i++) {
+ rand1.nextBytes(value);
+ hllc.add(value);
+
+ buf.clear();
+ hllc.writeRegisters(buf);
+
+ int len = buf.position();
+ buf.position(0);
+ assertEquals(len, hllc.peekLength(buf));
+
+ copy.readRegisters(buf);
+ assertEquals(len, buf.position());
+ assertEquals(hllc, copy);
+ }
+ buf.clear();
+ }
+
+ private Set<String> generateTestData(int n) {
+ Set<String> testData = new HashSet<String>();
+ for (int i = 0; i < n; i++) {
+ String[] samples = generateSampleData();
+ for (String sample : samples) {
+ testData.add(sample);
+ }
+ }
+ return testData;
+ }
+
+ // simulate the visit (=visitor+id)
+ private String[] generateSampleData() {
+
+ StringBuilder buf = new StringBuilder();
+ for (int i = 0; i < 19; i++) {
+ buf.append(Math.abs(rand1.nextInt()) % 10);
+ }
+ String header = buf.toString();
+
+ int size = Math.abs(rand3.nextInt()) % 9 + 1;
+ String[] samples = new String[size];
+ for (int k = 0; k < size; k++) {
+ buf = new StringBuilder(header);
+ buf.append("-");
+ for (int i = 0; i < 10; i++) {
+ buf.append(Math.abs(rand3.nextInt()) % 10);
+ }
+ samples[k] = buf.toString();
+ }
+
+ return samples;
+ }
+
+ @Test
+ public void countTest() throws IOException {
+ int n = 10;
+ for (int i = 0; i < 5; i++) {
+ count(n);
+ n *= 10;
+ }
+ }
+
+ private void count(int n) throws IOException {
+ Set<String> testSet = generateTestData(n);
+
+ HyperLogLogPlusCounterOld hllc = newHLLC();
+ for (String testData : testSet) {
+ hllc.add(Bytes.toBytes(testData));
+ }
+ long estimate = hllc.getCountEstimate();
+ double errorRate = hllc.getErrorRate();
+ double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size();
+ System.out.println(estimate);
+ System.out.println(testSet.size());
+ System.out.println(errorRate);
+ System.out.println("=" + actualError);
+ Assert.assertTrue(actualError < errorRate * 3.0);
+
+ checkSerialize(hllc);
+ }
+
+ private void checkSerialize(HyperLogLogPlusCounterOld hllc) throws IOException {
+ long estimate = hllc.getCountEstimate();
+ buf.clear();
+ hllc.writeRegisters(buf);
+ buf.flip();
+ hllc.readRegisters(buf);
+ Assert.assertEquals(estimate, hllc.getCountEstimate());
+ }
+
+ @Test
+ public void mergeTest() throws IOException {
+ double error = 0;
+ int n = 100;
+ for (int i = 0; i < n; i++) {
+ double e = merge(i);
+ error += e;
+ }
+ System.out.println("Total average error is " + error / n);
+
+ System.out.println(" errorRateCount1 is " + errorCount1 + "!");
+ System.out.println(" errorRateCount2 is " + errorCount2 + "!");
+ System.out.println(" errorRateCount3 is " + errorCount3 + "!");
+
+ Assert.assertTrue(errorCount1 <= n * 0.30);
+ Assert.assertTrue(errorCount2 <= n * 0.05);
+ Assert.assertTrue(errorCount3 <= n * 0.02);
+ }
+
+ private double merge(int round) throws IOException {
+ int ln = 20;
+ int dn = 100 * (round + 1);
+ Set<String> testSet = new HashSet<String>();
+ HyperLogLogPlusCounterOld[] hllcs = new HyperLogLogPlusCounterOld[ln];
+ for (int i = 0; i < ln; i++) {
+ hllcs[i] = newHLLC();
+ for (int k = 0; k < dn; k++) {
+ String[] samples = generateSampleData();
+ for (String data : samples) {
+ testSet.add(data);
+ hllcs[i].add(Bytes.toBytes(data));
+ }
+ }
+ }
+ HyperLogLogPlusCounterOld mergeHllc = newHLLC();
+ for (HyperLogLogPlusCounterOld hllc : hllcs) {
+ mergeHllc.merge(serDes(hllc));
+ }
+
+ double errorRate = mergeHllc.getErrorRate();
+ long estimate = mergeHllc.getCountEstimate();
+ double actualError = Math.abs((double) (testSet.size() - estimate) / testSet.size());
+
+ System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
+ Assert.assertTrue(actualError < 0.1);
+
+ if (actualError > errorRate) {
+ errorCount1++;
+ }
+ if (actualError > 2 * errorRate) {
+ errorCount2++;
+ }
+ if (actualError > 3 * errorRate) {
+ errorCount3++;
+ }
+
+ return actualError;
+ }
+
+ private HyperLogLogPlusCounterOld serDes(HyperLogLogPlusCounterOld hllc) throws IOException {
+ buf.clear();
+ hllc.writeRegisters(buf);
+ buf.flip();
+ HyperLogLogPlusCounterOld copy = new HyperLogLogPlusCounterOld(hllc.getPrecision());
+ copy.readRegisters(buf);
+ Assert.assertEquals(copy.getCountEstimate(), hllc.getCountEstimate());
+ return copy;
+ }
+
+ @Test
+ public void testPerformance() throws IOException {
+ int N = 3; // reduce N HLLC into one
+ int M = 1000; // for M times, use 100000 for real perf test
+
+ HyperLogLogPlusCounterOld samples[] = new HyperLogLogPlusCounterOld[N];
+ for (int i = 0; i < N; i++) {
+ samples[i] = newHLLC();
+ for (String str : generateTestData(10000))
+ samples[i].add(str);
+ }
+
+ System.out.println("Perf test running ... ");
+ long start = System.currentTimeMillis();
+ HyperLogLogPlusCounterOld sum = newHLLC();
+ for (int i = 0; i < M; i++) {
+ sum.clear();
+ for (int j = 0; j < N; j++) {
+ sum.merge(samples[j]);
+ checkSerialize(sum);
+ }
+ }
+ long duration = System.currentTimeMillis() - start;
+ System.out.println("Perf test result: " + duration / 1000 + " seconds");
+ }
+
+ @Test
+ public void testEquivalence() {
+ byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 };
+ byte[] b = new byte[] { 3, 4, 42 };
+ HyperLogLogPlusCounterOld ha = new HyperLogLogPlusCounterOld();
+ HyperLogLogPlusCounterOld hb = new HyperLogLogPlusCounterOld();
+ ha.add(a, 1, 3);
+ hb.add(b);
+
+ Assert.assertTrue(ha.getCountEstimate() == hb.getCountEstimate());
+ }
+
+ private HyperLogLogPlusCounterOld newHLLC() {
+ return new HyperLogLogPlusCounterOld(16);
+ }
+}
[26/50] [abbrv] kylin git commit: KYLIN 1875 update modify alias
Posted by li...@apache.org.
KYLIN 1875 update modify alias
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f55cc5c4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f55cc5c4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f55cc5c4
Branch: refs/heads/master-cdh5.7
Commit: f55cc5c446e8878d059019561894cc70bd0ab89b
Parents: e7cc152
Author: chenzhx <34...@qq.com>
Authored: Wed Dec 14 16:33:31 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 15 18:57:36 2016 +0800
----------------------------------------------------------------------
webapp/app/css/AdminLTE.css | 5 +
webapp/app/js/controllers/modelDataModel.js | 141 ++++++++++++-------
webapp/app/js/controllers/modelEdit.js | 8 +-
webapp/app/js/services/tree.js | 3 +
.../app/partials/modelDesigner/data_model.html | 25 ++--
5 files changed, 119 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f55cc5c4/webapp/app/css/AdminLTE.css
----------------------------------------------------------------------
diff --git a/webapp/app/css/AdminLTE.css b/webapp/app/css/AdminLTE.css
index 6688457..857dbf7 100644
--- a/webapp/app/css/AdminLTE.css
+++ b/webapp/app/css/AdminLTE.css
@@ -4744,6 +4744,11 @@ Gradient Background colors
opacity: 1;
filter: alpha(opacity=100);
}
+.model_graph svg{
+ width:1100px !important;
+ height:600px !important;
+}
+
/*
* Misc: print
http://git-wip-us.apache.org/repos/asf/kylin/blob/f55cc5c4/webapp/app/js/controllers/modelDataModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelDataModel.js b/webapp/app/js/controllers/modelDataModel.js
index a09101b..b6edd43 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -22,7 +22,7 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
$scope.modelsManager = modelsManager;
angular.forEach($scope.modelsManager.selectedModel.lookups,function(joinTable){
if(!joinTable.alias){
- joinTable.alias=VdmUtil.removeNameSpace(joinTable.table);
+ joinTable.alias=VdmUtil.removeNameSpace(joinTable.table);
}
});
$scope.init = function (){
@@ -110,10 +110,21 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
$scope.editLookup = function (lookup) {
$scope.lookupState.editingIndex = lookupList.indexOf(lookup);
$scope.lookupState.editing = true;
-
// Make a copy of model will be editing.
$scope.newLookup = angular.copy(lookup);
+ $scope.newLookup.join.pk_type = [];
+ $scope.newLookup.join.fk_type = [];
+ $scope.newLookup.join.isCompatible=[];
$scope.newLookup.joinTable=VdmUtil.getNameSpaceTopName($scope.newLookup.join.foreign_key[0]);
+ angular.forEach($scope.newLookup.join.primary_key,function(pk,index){
+ $scope.newLookup.join.pk_type[index] = TableModel.getColumnType(VdmUtil.removeNameSpace(pk),$scope.newLookup.table);
+ $scope.newLookup.join.fk_type[index] = TableModel.getColumnType(VdmUtil.removeNameSpace($scope.newLookup.join.foreign_key[index]),$scope.aliasTableMap[$scope.newLookup.joinTable]);
+ if($scope.newLookup.join.pk_type[index]!==$scope.newLookup.join.fk_type[index]){
+ $scope.newLookup.join.isCompatible[index]=false;
+ }else{
+ $scope.newLookup.join.isCompatible[index]=true;
+ }
+ });
$scope.openLookupModal();
};
@@ -129,24 +140,51 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
$scope.doneEditLookup = function () {
// Copy edited model to destination model.
- $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)]=$scope.modelsManager.selectedModel.fact_table;
- $scope.aliasName=[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)];
angular.copy($scope.newLookup, lookupList[$scope.lookupState.editingIndex]);
- angular.forEach(lookupList,function(joinTable){
- $scope.aliasName.push(joinTable.alias);
- $scope.aliasTableMap[joinTable.alias]=joinTable.table;
- // $scope.tableAliasMap[joinTable.alias]=joinTable.table;
- });
+ var oldAlias=$scope.aliasName[$scope.lookupState.editingIndex+1];
+ var newAlias=$scope.newLookup.alias;
+ if(oldAlias!=newAlias){
+ $scope.aliasName[$scope.lookupState.editingIndex+1]=newAlias;
+ for(var i=0;i<$scope.newLookup.join.primary_key.length;i++){
+ $scope.newLookup.join.primary_key[i]=$scope.newLookup.join.primary_key[i].replace(oldAlias+'.',newAlias+'.');
+ }
+
+ delete $scope.aliasTableMap[oldAlias];
+ $scope.aliasTableMap[newAlias]=$scope.newLookup.table;
+
+ for(var i=0;i<lookupList.length;i++){
+ for(var j=0;j<lookupList[i].join.foreign_key.length;j++){
+ lookupList[i].join.foreign_key[j]=lookupList[i].join.foreign_key[j].replace(oldAlias+'.',newAlias+'.');
+ }
+ }
+
+ for(var i=0;i<modelsManager.selectedModel.dimensions.length;i++){
+ if(modelsManager.selectedModel.dimensions[i].table==oldAlias){
+ modelsManager.selectedModel.dimensions[i].table=newAlias;
+ }
+ }
+
+ if($scope.newLookup.kind=="FACT"){
+ for(var i=0;i< modelsManager.selectedModel.metrics.length;i++){
+ modelsManager.selectedModel.metrics[i]= modelsManager.selectedModel.metrics[i].replace(oldAlias+'.',newAlias+'.');
+ }
+ modelsManager.selectedModel.partition_desc.partition_date_column = modelsManager.selectedModel.partition_desc.partition_date_column.replace(oldAlias+'.',newAlias+'.');
+ }
+ }
+ angular.copy($scope.newLookup,lookupList[$scope.lookupState.editingIndex]);
$scope.resetParams();
};
$scope.changeFactTable = function () {
- delete $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)];
+ $scope.aliasTableMap={};
$scope.aliasTableMap[VdmUtil.removeNameSpace($scope.FactTable.root)]=$scope.FactTable.root;
- delete $scope.tableAliasMap[$scope.modelsManager.selectedModel.fact_table];
+ $scope.tableAliasMap={};
$scope.tableAliasMap[$scope.FactTable.root]=VdmUtil.removeNameSpace($scope.FactTable.root);
- $scope.aliasName.splice($scope.aliasName.indexOf(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)),1);
- $scope.aliasName.push(VdmUtil.removeNameSpace($scope.FactTable.root));
+ $scope.aliasName=[VdmUtil.removeNameSpace($scope.FactTable.root)];
+ modelsManager.selectedModel.lookups = [];
+ modelsManager.selectedModel.dimensions = [];
+ modelsManager.selectedModel.metrics= [];
+ modelsManager.selectedModel.partition_desc.partition_date_column = null;
$scope.modelsManager.selectedModel.fact_table=$scope.FactTable.root;
}
$scope.changeJoinTable = function () {
@@ -158,7 +196,7 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
$scope.removeLookup = function (lookup) {
var dimExist = _.some(modelsManager.selectedModel.dimensions,function(item,index){
- return item.alias===lookup.alias;
+ return item.table===lookup.alias;
});
if(dimExist) {
SweetAlert.swal({
@@ -172,7 +210,7 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
}, function (isConfirm) {
if (isConfirm) {
for (var i = modelsManager.selectedModel.dimensions.length - 1; i >= 0; i--) {
- if (modelsManager.selectedModel.dimensions[i].alias === lookup.alias) {
+ if (modelsManager.selectedModel.dimensions[i].table === lookup.alias) {
modelsManager.selectedModel.dimensions.splice(i, 1);
}
}
@@ -187,13 +225,13 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
}
$scope.changeKey = function(index){
- var join_table = $scope.newLookup.jointable;
+ var join_table = $scope.newLookup.joinTable;
var lookup_table = $scope.newLookup.table;
var pk_column = $scope.newLookup.join.primary_key[index];
var fk_column = $scope.newLookup.join.foreign_key[index];
if(pk_column!=='null'&&fk_column!=='null'){
- $scope.newLookup.join.pk_type[index] = TableModel.getColumnType(pk_column,lookup_table);
- $scope.newLookup.join.fk_type[index] = TableModel.getColumnType(fk_column,join_table);
+ $scope.newLookup.join.pk_type[index] = TableModel.getColumnType(VdmUtil.removeNameSpace(pk_column),$scope.newLookup.table);
+ $scope.newLookup.join.fk_type[index] = TableModel.getColumnType(VdmUtil.removeNameSpace(fk_column),$scope.aliasTableMap[$scope.newLookup.joinTable]);
if($scope.newLookup.join.pk_type[index]!==$scope.newLookup.join.fk_type[index]){
$scope.newLookup.join.isCompatible[index]=false;
}else{
@@ -226,34 +264,43 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
};
$scope.checkLookupForm = function(){
- var errors = [];
- // null validate
- for(var i = 0;i<$scope.newLookup.join.primary_key.length;i++){
- if($scope.newLookup.join.primary_key[i]==='null'){
- errors.push("Primary Key can't be null.");
- break;
- }
- }
- for(var i = 0;i<$scope.newLookup.join.foreign_key.length;i++){
- if($scope.newLookup.join.foreign_key[i]==='null'){
- errors.push("Foreign Key can't be null.");
- break;
- }
- }
-
- var errorInfo = "";
- angular.forEach(errors,function(item){
- errorInfo+="\n"+item;
- });
- if(errors.length){
-// SweetAlert.swal('Warning!', errorInfo, '');
- SweetAlert.swal('', errorInfo, 'warning');
- return false;
- }else{
- return true;
- }
-
+ var errors = [];
+ for(var i = 0;i<$scope.newLookup.join.primary_key.length;i++){
+ if($scope.newLookup.join.primary_key[i]==='null'){
+ errors.push("Primary Key can't be null.");
+ break;
+ }
+ }
+ for(var i = 0;i<$scope.newLookup.join.foreign_key.length;i++){
+ if($scope.newLookup.join.foreign_key[i]==='null'){
+ errors.push("Foreign Key can't be null.");
+ break;
+ }
+ }
+ if($scope.aliasName.indexOf($scope.newLookup.alias)!=-1&&$scope.lookupState.editing == false){
+ errors.push("Table Alias ["+$scope.newLookup.alias+"] already exist!");
+ }
+ if($scope.aliasName.indexOf($scope.newLookup.alias)!=-1&&$scope.aliasName[$scope.lookupState.editingIndex+1] != $scope.newLookup.alias){
+ errors.push("Table Alias ["+$scope.newLookup.alias+"] already exist!");
+ }
+ var errorInfo = "";
+ angular.forEach(errors,function(item){
+ errorInfo+="\n"+item;
+ });
+ if(errors.length){
+ SweetAlert.swal('', errorInfo, 'warning');
+ return false;
+ }else{
+ return true;
+ }
};
-
-
+ $scope.filterNotRoot = function (item) {
+ return item.name!==modelsManager.selectedModel.fact_table;
+ };
+/* $scope.$watch('$scope.newLookup.alias', function (newValue, oldValue) {
+ if (newValue&&$scope.lookupState.editing ) {
+ console.log(newValue);
+ console.log(oldValue);
+ }
+ });*/
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/f55cc5c4/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index 4d1e12c..fd78d03 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -19,7 +19,7 @@
'use strict';
-KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $location, $templateCache, $interpolate, MessageService, TableService, CubeDescService, ModelService, loadingRequest, SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager,VdmUtil) {
+KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $location, $templateCache, $interpolate, MessageService, TableService, CubeDescService, ModelService, loadingRequest, SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager, CubeService, VdmUtil) {
//add or edit ?
var absUrl = $location.absUrl();
$scope.tableAliasMap={};
@@ -93,9 +93,9 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
ModelDescService.query({model_name: modelName}, function (model) {
if (model) {
modelsManager.selectedModel = model;
- if($scope.modelsManager.selectedModel.partition_desc.partition_time_column){
- $scope.partitionColumn.hasSeparateTimeColumn = true;
- }
+ CubeService.list({modelName:model.name}, function (_cubes) {
+ $scope.cubesLength = _cubes.length;
+ });
modelsManager.selectedModel.project = ProjectModel.getProjectByCubeModel(modelName);
if(!ProjectModel.getSelectedProject()){
ProjectModel.setSelectedProject(modelsManager.selectedModel.project);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f55cc5c4/webapp/app/js/services/tree.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/tree.js b/webapp/app/js/services/tree.js
index 08e290b..20de19b 100755
--- a/webapp/app/js/services/tree.js
+++ b/webapp/app/js/services/tree.js
@@ -47,6 +47,9 @@ KylinApp.service('ModelGraphService', function (VdmUtil) {
model.graph = (!!model.graph) ? model.graph : {};
angular.forEach(model.lookups,function (lookup, index) {
+ if(!lookup.alias){
+ lookup.alias=VdmUtil.removeNameSpace(lookup.table);
+ }
if (lookup.join && lookup.join.primary_key.length > 0) {
var dimensionNode={
"type": lookup.kind,
http://git-wip-us.apache.org/repos/asf/kylin/blob/f55cc5c4/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index f1192f4..e032cf9 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -29,7 +29,7 @@
<select chosen ng-model="FactTable.root" ng-if="state.mode=='edit'"
ng-options="table.name as table.name for table in tableModel.selectProjectTables"
style="width:100%;" ng-change="changeFactTable()"
- name="table_name"
+ name="table_name" ng-disabled="cubesLength>0"
ng-required="true"
data-placeholder="Fact Table Name"
class="chosen-select">
@@ -92,17 +92,18 @@
<ul class="list-unstyled">
<li ng-repeat="pk in lookup.join.primary_key track by $index">
<code>{{lookup.join.foreign_key[$index]}} = {{pk}}</code>
+
</li>
</ul>
</td>
<td ng-if="state.mode=='edit'">
<!-- edit button -->
- <button class="btn btn-xs btn-info" ng-disabled="lookupState.editing"
- ng-click="editLookup(lookup)" tooltip="Edit Lookup"><i class="fa fa-pencil"></i>
+ <button class="btn btn-xs btn-info" ng-disabled="lookupState.editing||cubesLength>0"
+ ng-click="editLookup(lookup)" ><i class="fa fa-pencil"></i>
</button>
<!-- remove button -->
- <button class="btn btn-xs btn-danger" ng-disabled="lookupState.editing"
- ng-click="removeLookup(lookup)" tooltip="Remove Lookup"><i class="fa fa-trash-o"></i>
+ <button class="btn btn-xs btn-danger" ng-disabled="lookupState.editing||cubesLength>0"
+ ng-click="removeLookup(lookup)" ><i class="fa fa-trash-o"></i>
</button>
</td>
</tr>
@@ -126,17 +127,17 @@
<div>
<select chosen ng-model="newLookup.joinTable" style="width: 45%"
ng-options="table as table for table in aliasName"
- name="table_name"
+ name="table_name" ng-disabled="lookupState.editing"
ng-required="true"
data-placeholder="Join Table Name"
class="chosen-select">
<option value=""> -- Select Join Table -- </option>
</select>
<small class="help-block" ng-show="lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$submitted)">Table name required</small>
- <b> </b>
+ <b> </b>
<select chosen ng-model="newLookup.table" style="width: 45%"
- ng-options="table.name as table.name for table in tableModel.selectProjectTables"
- name="table_name"
+ ng-options="table.name as table.name for table in tableModel.selectProjectTables|filter:filterNotRoot"
+ name="table_name" ng-disabled="lookupState.editing"
ng-required="true" ng-change="changeJoinTable()"
data-placeholder="Join Table Name"
class="chosen-select">
@@ -146,7 +147,6 @@
</div>
<div class="space-4"></div>
- <small class="help-block red" ng-show="newLookup.join.isCompatible[$index]==false"><i class="fa fa-exclamation-triangle"></i> <b>Column Type incompatible {{newLookup.join.foreign_key[$index]}}[{{newLookup.join.fk_type[$index]}}], {{newLookup.join.primary_key[$index]}} [{{newLookup.join.pk_type[$index]}}]</b></small>
</div>
</div>
</div>
@@ -154,8 +154,9 @@
<div class="row">
<label class="col-sm-3 control-label font-color-default"><b>Alias</b></label>
<div class="col-sm-6">
- <input type="text" class="form-control " placeholder="Input Table Alias" ng-required="true"
- ng-model="newLookup.alias" ng-change="changeAlias()" ng-disabled="">
+ <input type="text" class="form-control " name="joinTable_alias" placeholder="Input Table Alias" ng-required="true"
+ ng-model="newLookup.alias" ng-change="changeAlias()" ng-pattern="/^\w+$/">
+ <small class="help-block red" ng-show="!lookup_form.joinTable_alias.$error.required&&lookup_form.joinTable_alias.$invalid && (lookup_form.joinTable_alias.$dirty||lookup_form.$submitted)"><i class="fa fa-exclamation-triangle"></i> Alias is invalid.</small>
</div>
</div>
</div>
[08/50] [abbrv] kylin git commit: minor, fix KylinConfig load BCC
Posted by li...@apache.org.
minor, fix KylinConfig load BCC
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e3933381
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e3933381
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e3933381
Branch: refs/heads/master-cdh5.7
Commit: e393338124302f60ff6e85e5fdc2e8cb806696f5
Parents: d11d019
Author: Billy Liu <bi...@apache.org>
Authored: Tue Dec 13 08:03:22 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Tue Dec 13 08:03:22 2016 +0800
----------------------------------------------------------------------
core-common/src/main/java/org/apache/kylin/common/KylinConfig.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e3933381/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index d888fd3..acd4398 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -215,6 +215,7 @@ public class KylinConfig extends KylinConfigBase {
FileInputStream is = new FileInputStream(propFile);
conf.load(is);
IOUtils.closeQuietly(is);
+ conf = BCC.check(conf);
File propOverrideFile = new File(propFile.getParentFile(), propFile.getName() + ".override");
if (propOverrideFile.exists()) {
@@ -222,7 +223,6 @@ public class KylinConfig extends KylinConfigBase {
Properties propOverride = new Properties();
propOverride.load(ois);
IOUtils.closeQuietly(ois);
- conf = BCC.check(conf);
conf.putAll(BCC.check(propOverride));
}
} catch (IOException e) {
[11/50] [abbrv] kylin git commit: KYLIN-1832 HyperLogLog performance
optimization
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterTest.java
deleted file mode 100644
index 5b7c565..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterTest.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.hll;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class HyperLogLogCounterTest {
-
- ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
- Random rand1 = new Random(1);
- Random rand2 = new Random(2);
- Random rand3 = new Random(3);
- int errorCount1 = 0;
- int errorCount2 = 0;
- int errorCount3 = 0;
-
- @Test
- public void testOneAdd() throws IOException {
- HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(14);
- HyperLogLogPlusCounter one = new HyperLogLogPlusCounter(14);
- for (int i = 0; i < 1000000; i++) {
- one.clear();
- one.add(rand1.nextInt());
- hllc.merge(one);
- }
- assertTrue(hllc.getCountEstimate() > 1000000 * 0.9);
- }
-
- @Test
- public void testPeekLength() throws IOException {
- HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter(10);
- HyperLogLogPlusCounter copy = new HyperLogLogPlusCounter(10);
- byte[] value = new byte[10];
- for (int i = 0; i < 200000; i++) {
- rand1.nextBytes(value);
- hllc.add(value);
-
- buf.clear();
- hllc.writeRegisters(buf);
-
- int len = buf.position();
- buf.position(0);
- assertEquals(len, hllc.peekLength(buf));
-
- copy.readRegisters(buf);
- assertEquals(len, buf.position());
- assertEquals(hllc, copy);
- }
- buf.clear();
- }
-
- private Set<String> generateTestData(int n) {
- Set<String> testData = new HashSet<String>();
- for (int i = 0; i < n; i++) {
- String[] samples = generateSampleData();
- for (String sample : samples) {
- testData.add(sample);
- }
- }
- return testData;
- }
-
- // simulate the visit (=visitor+id)
- private String[] generateSampleData() {
-
- StringBuilder buf = new StringBuilder();
- for (int i = 0; i < 19; i++) {
- buf.append(Math.abs(rand1.nextInt()) % 10);
- }
- String header = buf.toString();
-
- int size = Math.abs(rand3.nextInt()) % 9 + 1;
- String[] samples = new String[size];
- for (int k = 0; k < size; k++) {
- buf = new StringBuilder(header);
- buf.append("-");
- for (int i = 0; i < 10; i++) {
- buf.append(Math.abs(rand3.nextInt()) % 10);
- }
- samples[k] = buf.toString();
- }
-
- return samples;
- }
-
- @Test
- public void countTest() throws IOException {
- int n = 10;
- for (int i = 0; i < 5; i++) {
- count(n);
- n *= 10;
- }
- }
-
- private void count(int n) throws IOException {
- Set<String> testSet = generateTestData(n);
-
- HyperLogLogPlusCounter hllc = newHLLC();
- for (String testData : testSet) {
- hllc.add(Bytes.toBytes(testData));
- }
- long estimate = hllc.getCountEstimate();
- double errorRate = hllc.getErrorRate();
- double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size();
- System.out.println(estimate);
- System.out.println(testSet.size());
- System.out.println(errorRate);
- System.out.println("=" + actualError);
- Assert.assertTrue(actualError < errorRate * 3.0);
-
- checkSerialize(hllc);
- }
-
- private void checkSerialize(HyperLogLogPlusCounter hllc) throws IOException {
- long estimate = hllc.getCountEstimate();
- buf.clear();
- hllc.writeRegisters(buf);
- buf.flip();
- hllc.readRegisters(buf);
- Assert.assertEquals(estimate, hllc.getCountEstimate());
- }
-
- @Test
- public void mergeTest() throws IOException {
- double error = 0;
- int n = 100;
- for (int i = 0; i < n; i++) {
- double e = merge(i);
- error += e;
- }
- System.out.println("Total average error is " + error / n);
-
- System.out.println(" errorRateCount1 is " + errorCount1 + "!");
- System.out.println(" errorRateCount2 is " + errorCount2 + "!");
- System.out.println(" errorRateCount3 is " + errorCount3 + "!");
-
- Assert.assertTrue(errorCount1 <= n * 0.30);
- Assert.assertTrue(errorCount2 <= n * 0.05);
- Assert.assertTrue(errorCount3 <= n * 0.02);
- }
-
- private double merge(int round) throws IOException {
- int ln = 20;
- int dn = 100 * (round + 1);
- Set<String> testSet = new HashSet<String>();
- HyperLogLogPlusCounter[] hllcs = new HyperLogLogPlusCounter[ln];
- for (int i = 0; i < ln; i++) {
- hllcs[i] = newHLLC();
- for (int k = 0; k < dn; k++) {
- String[] samples = generateSampleData();
- for (String data : samples) {
- testSet.add(data);
- hllcs[i].add(Bytes.toBytes(data));
- }
- }
- }
- HyperLogLogPlusCounter mergeHllc = newHLLC();
- for (HyperLogLogPlusCounter hllc : hllcs) {
- mergeHllc.merge(serDes(hllc));
- }
-
- double errorRate = mergeHllc.getErrorRate();
- long estimate = mergeHllc.getCountEstimate();
- double actualError = Math.abs((double) (testSet.size() - estimate) / testSet.size());
-
- System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
- Assert.assertTrue(actualError < 0.1);
-
- if (actualError > errorRate) {
- errorCount1++;
- }
- if (actualError > 2 * errorRate) {
- errorCount2++;
- }
- if (actualError > 3 * errorRate) {
- errorCount3++;
- }
-
- return actualError;
- }
-
- private HyperLogLogPlusCounter serDes(HyperLogLogPlusCounter hllc) throws IOException {
- buf.clear();
- hllc.writeRegisters(buf);
- buf.flip();
- HyperLogLogPlusCounter copy = new HyperLogLogPlusCounter(hllc.getPrecision());
- copy.readRegisters(buf);
- Assert.assertEquals(copy.getCountEstimate(), hllc.getCountEstimate());
- return copy;
- }
-
- @Test
- public void testPerformance() throws IOException {
- int N = 3; // reduce N HLLC into one
- int M = 1000; // for M times, use 100000 for real perf test
-
- HyperLogLogPlusCounter samples[] = new HyperLogLogPlusCounter[N];
- for (int i = 0; i < N; i++) {
- samples[i] = newHLLC();
- for (String str : generateTestData(10000))
- samples[i].add(str);
- }
-
- System.out.println("Perf test running ... ");
- long start = System.currentTimeMillis();
- HyperLogLogPlusCounter sum = newHLLC();
- for (int i = 0; i < M; i++) {
- sum.clear();
- for (int j = 0; j < N; j++) {
- sum.merge(samples[j]);
- checkSerialize(sum);
- }
- }
- long duration = System.currentTimeMillis() - start;
- System.out.println("Perf test result: " + duration / 1000 + " seconds");
- }
-
- @Test
- public void testEquivalence() {
- byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 };
- byte[] b = new byte[] { 3, 4, 42 };
- HyperLogLogPlusCounter ha = new HyperLogLogPlusCounter();
- HyperLogLogPlusCounter hb = new HyperLogLogPlusCounter();
- ha.add(a, 1, 3);
- hb.add(b);
-
- Assert.assertTrue(ha.getCountEstimate() == hb.getCountEstimate());
- }
-
- private HyperLogLogPlusCounter newHLLC() {
- return new HyperLogLogPlusCounter(16);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/test/java/org/apache/kylin/measure/hll2/HyperLogLogCounterNewTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hll2/HyperLogLogCounterNewTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hll2/HyperLogLogCounterNewTest.java
new file mode 100644
index 0000000..feb8c8e
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hll2/HyperLogLogCounterNewTest.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.measure.hll2;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterOld;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.RegisterType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Created by xiefan on 16-12-12.
+ */
+public class HyperLogLogCounterNewTest {
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ Random rand1 = new Random(1);
+ Random rand2 = new Random(2);
+ Random rand3 = new Random(3);
+ int errorCount1 = 0;
+ int errorCount2 = 0;
+ int errorCount3 = 0;
+
+ @Test
+ public void testOneAdd() throws IOException {
+ HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew(14);
+ HyperLogLogPlusCounterNew one = new HyperLogLogPlusCounterNew(14);
+ for (int i = 0; i < 1000000; i++) {
+ one.clear();
+ one.add(rand1.nextInt());
+ hllc.merge(one);
+ }
+ System.out.println(hllc.getCountEstimate());
+ assertTrue(hllc.getCountEstimate() > 1000000 * 0.9);
+ }
+
+ @Test
+ public void tesSparseEstimate() throws IOException {
+ HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew(14);
+ for (int i = 0; i < 10; i++) {
+ hllc.add(i);
+ }
+ System.out.println(hllc.getCountEstimate());
+ assertTrue(hllc.getCountEstimate() > 10 * 0.9);
+ }
+
+ @Test
+ public void countTest() throws IOException {
+ int n = 10;
+ for (int i = 0; i < 5; i++) {
+ count(n);
+ n *= 10;
+ }
+ }
+
+ @Test
+ public void mergeTest() throws IOException {
+ double error = 0;
+ int n = 100;
+ for (int i = 0; i < n; i++) {
+ double e = merge(i);
+ error += e;
+ }
+ System.out.println("Total average error is " + error / n);
+
+ System.out.println(" errorRateCount1 is " + errorCount1 + "!");
+ System.out.println(" errorRateCount2 is " + errorCount2 + "!");
+ System.out.println(" errorRateCount3 is " + errorCount3 + "!");
+
+ Assert.assertTrue(errorCount1 <= n * 0.30);
+ Assert.assertTrue(errorCount2 <= n * 0.05);
+ Assert.assertTrue(errorCount3 <= n * 0.02);
+ }
+
+ /*
+ compare the result of two different hll counter
+ */
+ @Test
+ public void compareResult() {
+ int p = 12; //4096
+ int m = 1 << p;
+
+ for (int t = 0; t < 5; t++) {
+ //compare sparse
+ HyperLogLogPlusCounterOld oldCounter = new HyperLogLogPlusCounterOld(p);
+ HyperLogLogPlusCounterNew newCounter = new HyperLogLogPlusCounterNew(p);
+
+ for (int i = 0; i < 20; i++) {
+ //int r = rand1.nextInt();
+ oldCounter.add(i);
+ newCounter.add(i);
+ }
+ assertEquals(RegisterType.SPARSE, newCounter.getRegisterType());
+ assertEquals(oldCounter.getCountEstimate(), newCounter.getCountEstimate());
+ //compare dense
+ for (int i = 0; i < m; i++) {
+ oldCounter.add(i);
+ newCounter.add(i);
+ }
+ assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
+ assertEquals(oldCounter.getCountEstimate(), newCounter.getCountEstimate());
+ }
+
+ }
+
+ @Test
+ public void testPeekLength() throws IOException {
+ HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew(10);
+ HyperLogLogPlusCounterNew copy = new HyperLogLogPlusCounterNew(10);
+ byte[] value = new byte[10];
+ for (int i = 0; i < 200000; i++) {
+ rand1.nextBytes(value);
+ hllc.add(value);
+
+ buf.clear();
+ hllc.writeRegisters(buf);
+
+ int len = buf.position();
+ buf.position(0);
+ assertEquals(len, hllc.peekLength(buf));
+
+ copy.readRegisters(buf);
+ assertEquals(len, buf.position());
+ assertEquals(hllc, copy);
+ }
+ buf.clear();
+ }
+
+ @Test
+ public void testEquivalence() {
+ byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 };
+ byte[] b = new byte[] { 3, 4, 42 };
+ HyperLogLogPlusCounterNew ha = new HyperLogLogPlusCounterNew();
+ HyperLogLogPlusCounterNew hb = new HyperLogLogPlusCounterNew();
+ ha.add(a, 1, 3);
+ hb.add(b);
+
+ Assert.assertTrue(ha.getCountEstimate() == hb.getCountEstimate());
+ }
+
+ @Test
+ public void testAutoChangeToSparse() {
+ int p = 15;
+ int m = 1 << p;
+ HyperLogLogPlusCounterNew counter = new HyperLogLogPlusCounterNew(p);
+ assertEquals(RegisterType.SPARSE, counter.getRegisterType());
+ double over = HyperLogLogPlusCounterNew.overflowFactor * m;
+ int overFlow = (int) over + 1000;
+ for (int i = 0; i < overFlow; i++)
+ counter.add(i);
+ assertEquals(RegisterType.DENSE, counter.getRegisterType());
+ }
+
+ @Test
+ public void testSerialilze() throws Exception {
+ //test sparse serialize
+ int p = 15;
+ int m = 1 << p;
+ HyperLogLogPlusCounterNew counter = new HyperLogLogPlusCounterNew(p);
+ counter.add(123);
+ assertEquals(RegisterType.SPARSE, counter.getRegisterType());
+ checkSerialize(counter);
+ //test dense serialize
+ double over = HyperLogLogPlusCounterNew.overflowFactor * m;
+ int overFlow = (int) over + 1000;
+ for (int i = 0; i < overFlow; i++)
+ counter.add(i);
+ assertEquals(RegisterType.DENSE, counter.getRegisterType());
+ checkSerialize(counter);
+ }
+
+ private Set<String> generateTestData(int n) {
+ Set<String> testData = new HashSet<String>();
+ for (int i = 0; i < n; i++) {
+ String[] samples = generateSampleData();
+ for (String sample : samples) {
+ testData.add(sample);
+ }
+ }
+ return testData;
+ }
+
+ // simulate the visit (=visitor+id)
+ private String[] generateSampleData() {
+
+ StringBuilder buf = new StringBuilder();
+ for (int i = 0; i < 19; i++) {
+ buf.append(Math.abs(rand1.nextInt()) % 10);
+ }
+ String header = buf.toString();
+
+ int size = Math.abs(rand3.nextInt()) % 9 + 1;
+ String[] samples = new String[size];
+ for (int k = 0; k < size; k++) {
+ buf = new StringBuilder(header);
+ buf.append("-");
+ for (int i = 0; i < 10; i++) {
+ buf.append(Math.abs(rand3.nextInt()) % 10);
+ }
+ samples[k] = buf.toString();
+ }
+
+ return samples;
+ }
+
+ private double merge(int round) throws IOException {
+ int ln = 20;
+ int dn = 100 * (round + 1);
+ Set<String> testSet = new HashSet<String>();
+ HyperLogLogPlusCounterNew[] hllcs = new HyperLogLogPlusCounterNew[ln];
+ for (int i = 0; i < ln; i++) {
+ hllcs[i] = newHLLC();
+ for (int k = 0; k < dn; k++) {
+ String[] samples = generateSampleData();
+ for (String data : samples) {
+ testSet.add(data);
+ hllcs[i].add(Bytes.toBytes(data));
+ }
+ }
+ }
+ HyperLogLogPlusCounterNew mergeHllc = newHLLC();
+ for (HyperLogLogPlusCounterNew hllc : hllcs) {
+ mergeHllc.merge(hllc);
+ }
+
+ double errorRate = mergeHllc.getErrorRate();
+ long estimate = mergeHllc.getCountEstimate();
+ double actualError = Math.abs((double) (testSet.size() - estimate) / testSet.size());
+
+ System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
+ Assert.assertTrue(actualError < 0.1);
+
+ if (actualError > errorRate) {
+ errorCount1++;
+ }
+ if (actualError > 2 * errorRate) {
+ errorCount2++;
+ }
+ if (actualError > 3 * errorRate) {
+ errorCount3++;
+ }
+
+ return actualError;
+ }
+
+ private HyperLogLogPlusCounterNew newHLLC() {
+ return new HyperLogLogPlusCounterNew(16);
+ }
+
+ private void count(int n) throws IOException {
+ Set<String> testSet = generateTestData(n);
+
+ HyperLogLogPlusCounterNew hllc = newHLLC();
+ for (String testData : testSet) {
+ hllc.add(Bytes.toBytes(testData));
+ }
+ long estimate = hllc.getCountEstimate();
+ double errorRate = hllc.getErrorRate();
+ double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size();
+ System.out.println(estimate);
+ System.out.println(testSet.size());
+ System.out.println(errorRate);
+ System.out.println("=" + actualError);
+ Assert.assertTrue(actualError < errorRate * 3.0);
+
+ checkSerialize(hllc);
+ }
+
+ private void checkSerialize(HyperLogLogPlusCounterNew hllc) throws IOException {
+ long estimate = hllc.getCountEstimate();
+ buf.clear();
+ hllc.writeRegisters(buf);
+ buf.flip();
+ hllc.readRegisters(buf);
+ Assert.assertEquals(estimate, hllc.getCountEstimate());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/core-metadata/src/test/java/org/apache/kylin/measure/hll2/NewHyperLogLogBenchmarkTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hll2/NewHyperLogLogBenchmarkTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hll2/NewHyperLogLogBenchmarkTest.java
new file mode 100644
index 0000000..bfb87f9
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hll2/NewHyperLogLogBenchmarkTest.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.measure.hll2;
+
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterOld;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.RegisterType;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Created by xiefan on 16-12-12.
+ */
+public class NewHyperLogLogBenchmarkTest {
+
+ public static final Random rand = new Random(1);
+
+ final int testTimes = 10000;
+
+ @Test
+ public void denseToDenseRegisterMergeBenchmark() throws Exception {
+ final int p = 15;
+ int m = 1 << p;
+
+ System.out.println("m : " + m);
+ double oldFactor = HyperLogLogPlusCounterNew.overflowFactor;
+ HyperLogLogPlusCounterNew.overflowFactor = 1.1; //keep sparse
+ for (int cardinality : getTestDataDivide(m)) {
+ final HyperLogLogPlusCounterOld oldCounter = new HyperLogLogPlusCounterOld(p);
+ final HyperLogLogPlusCounterOld oldCounter2 = getRandOldCounter(p, cardinality);
+ long oldTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+
+ for (int i = 0; i < testTimes; i++) {
+ oldCounter.merge(oldCounter2);
+ }
+ }
+ });
+ final HyperLogLogPlusCounterNew newCounter = new HyperLogLogPlusCounterNew(p, RegisterType.DENSE);
+ final HyperLogLogPlusCounterNew newCounter2 = new HyperLogLogPlusCounterNew(p, RegisterType.DENSE);
+ for (int i = 0; i < testTimes; i++)
+ newCounter2.add(i);
+ long newTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+ for (int i = 0; i < testTimes; i++) {
+ newCounter.merge(newCounter2);
+ }
+ }
+ });
+ assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
+ assertEquals(RegisterType.DENSE, newCounter2.getRegisterType());
+ System.out.println("----------------------------");
+ System.out.println("cardinality : " + cardinality);
+ System.out.println("old time : " + oldTime);
+ System.out.println("new time : " + newTime);
+ }
+ HyperLogLogPlusCounterNew.overflowFactor = oldFactor;
+ }
+
+ @Test
+ public void sparseToSparseMergeBenchmark() throws Exception {
+ final int p = 15;
+ int m = 1 << p;
+ System.out.println("m : " + m);
+ double oldFactor = HyperLogLogPlusCounterNew.overflowFactor;
+ HyperLogLogPlusCounterNew.overflowFactor = 1.1; //keep sparse
+ for (int cardinality : getTestDataDivide(m)) {
+ final HyperLogLogPlusCounterOld oldCounter = new HyperLogLogPlusCounterOld(p);
+ final HyperLogLogPlusCounterOld oldCounter2 = getRandOldCounter(p, cardinality);
+ long oldTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+
+ for (int i = 0; i < testTimes; i++) {
+ oldCounter.merge(oldCounter2);
+ }
+ }
+ });
+ final HyperLogLogPlusCounterNew newCounter = new HyperLogLogPlusCounterNew(p);
+ final HyperLogLogPlusCounterNew newCounter2 = getRandNewCounter(p, cardinality);
+ long newTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+ for (int i = 0; i < testTimes; i++) {
+ newCounter.merge(newCounter2);
+ }
+ }
+ });
+ assertEquals(RegisterType.SPARSE, newCounter.getRegisterType());
+ assertEquals(RegisterType.SPARSE, newCounter2.getRegisterType());
+ System.out.println("----------------------------");
+ System.out.println("cardinality : " + cardinality);
+ System.out.println("old time : " + oldTime);
+ System.out.println("new time : " + newTime);
+ }
+ HyperLogLogPlusCounterNew.overflowFactor = oldFactor;
+ }
+
+ @Test
+ public void sparseToDenseRegisterMergeBenchmark() throws Exception {
+ final int p = 15;
+ int m = 1 << p;
+ System.out.println("m : " + m);
+ double oldFactor = HyperLogLogPlusCounterNew.overflowFactor;
+ HyperLogLogPlusCounterNew.overflowFactor = 1.1; //keep sparse
+ for (int cardinality : getTestDataDivide(m)) {
+ System.out.println("----------------------------");
+ System.out.println("cardinality : " + cardinality);
+ final HyperLogLogPlusCounterOld oldCounter = new HyperLogLogPlusCounterOld(p);
+ final HyperLogLogPlusCounterOld oldCounter2 = getRandOldCounter(p, cardinality);
+ long oldTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+ for (int i = 0; i < testTimes; i++) {
+ oldCounter.merge(oldCounter2);
+ }
+ }
+ });
+ final HyperLogLogPlusCounterNew newCounter = new HyperLogLogPlusCounterNew(p, RegisterType.DENSE);
+ final HyperLogLogPlusCounterNew newCounter2 = getRandNewCounter(p, cardinality);
+ long newTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+ for (int i = 0; i < testTimes; i++) {
+ newCounter.merge(newCounter2);
+ }
+ }
+ });
+ assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
+ assertEquals(RegisterType.SPARSE, newCounter2.getRegisterType());
+ System.out.println("old time : " + oldTime);
+ System.out.println("new time : " + newTime);
+ }
+ HyperLogLogPlusCounterNew.overflowFactor = oldFactor;
+ }
+
+ @Test
+ public void sparseSerializeBenchmark() throws Exception {
+ final int p = 15;
+ int m = 1 << p;
+ double oldFactor = HyperLogLogPlusCounterNew.overflowFactor;
+ HyperLogLogPlusCounterNew.overflowFactor = 1.1; //keep sparse
+ for (int cardinality : getTestDataDivide(m)) {
+ System.out.println("----------------------------");
+ System.out.println("cardinality : " + cardinality);
+ final HyperLogLogPlusCounterOld oldCounter = getRandOldCounter(p, cardinality);
+ long oldTime = runTestCase(new TestCase() {
+ @Override
+ public void run() throws Exception {
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ long totalBytes = 0;
+ for (int i = 0; i < testTimes; i++) {
+ buf.clear();
+ oldCounter.writeRegisters(buf);
+ totalBytes += buf.position();
+ buf.flip();
+ oldCounter.readRegisters(buf);
+ }
+ System.out.println("old serialize bytes : " + totalBytes / testTimes + "B");
+ }
+ });
+ final HyperLogLogPlusCounterNew newCounter = getRandNewCounter(p, cardinality);
+ long newTime = runTestCase(new TestCase() {
+ @Override
+ public void run() throws Exception {
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ long totalBytes = 0;
+ for (int i = 0; i < testTimes; i++) {
+ buf.clear();
+ newCounter.writeRegisters(buf);
+ totalBytes += buf.position();
+ buf.flip();
+ newCounter.readRegisters(buf);
+ }
+ System.out.println("new serialize bytes : " + totalBytes / testTimes + "B");
+ }
+ });
+ assertEquals(RegisterType.SPARSE, newCounter.getRegisterType());
+ System.out.println("old serialize time : " + oldTime);
+ System.out.println("new serialize time : " + newTime);
+ }
+ HyperLogLogPlusCounterNew.overflowFactor = oldFactor;
+ }
+
+ @Test
+ public void denseSerializeBenchmark() throws Exception {
+ final int p = 15;
+ int m = 1 << p;
+ double oldFactor = HyperLogLogPlusCounterNew.overflowFactor;
+ HyperLogLogPlusCounterNew.overflowFactor = 0; //keep sparse
+ for (int cardinality : getTestDataDivide(m)) {
+ System.out.println("----------------------------");
+ System.out.println("cardinality : " + cardinality);
+ final HyperLogLogPlusCounterOld oldCounter = getRandOldCounter(p, cardinality);
+ long oldTime = runTestCase(new TestCase() {
+ @Override
+ public void run() throws Exception {
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ long totalBytes = 0;
+ for (int i = 0; i < testTimes; i++) {
+ buf.clear();
+ oldCounter.writeRegisters(buf);
+ totalBytes += buf.position();
+ buf.flip();
+ oldCounter.readRegisters(buf);
+ }
+ System.out.println("old serialize bytes : " + totalBytes / testTimes + "B");
+ }
+ });
+ final HyperLogLogPlusCounterNew newCounter = getRandNewCounter(p, cardinality, RegisterType.DENSE);
+ long newTime = runTestCase(new TestCase() {
+ @Override
+ public void run() throws Exception {
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ long totalBytes = 0;
+ for (int i = 0; i < testTimes; i++) {
+ buf.clear();
+ newCounter.writeRegisters(buf);
+ totalBytes += buf.position();
+ buf.flip();
+ newCounter.readRegisters(buf);
+ }
+ System.out.println("new serialize bytes : " + totalBytes / testTimes + "B");
+ }
+ });
+ assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
+ System.out.println("old serialize time : " + oldTime);
+ System.out.println("new serialize time : " + newTime);
+ }
+ HyperLogLogPlusCounterNew.overflowFactor = oldFactor;
+ }
+
+ interface TestCase {
+ void run() throws Exception;
+ }
+
+ public long runTestCase(TestCase testCase) throws Exception {
+ long startTime = System.currentTimeMillis();
+ testCase.run();
+ return System.currentTimeMillis() - startTime;
+ }
+
+ public HyperLogLogPlusCounterOld getRandOldCounter(int p, int num) {
+ HyperLogLogPlusCounterOld c = new HyperLogLogPlusCounterOld(p);
+ for (int i = 0; i < num; i++)
+ c.add(i);
+ return c;
+ }
+
+ public HyperLogLogPlusCounterNew getRandNewCounter(int p, int num) {
+ HyperLogLogPlusCounterNew c = new HyperLogLogPlusCounterNew(p);
+ for (int i = 0; i < num; i++)
+ c.add(i);
+ return c;
+ }
+
+ public HyperLogLogPlusCounterNew getRandNewCounter(int p, int num, RegisterType type) {
+ HyperLogLogPlusCounterNew c = new HyperLogLogPlusCounterNew(p, type);
+ for (int i = 0; i < num; i++)
+ c.add(i);
+ return c;
+ }
+
+ public static int[] getTestDataDivide(int m) {
+ return new int[] { 1, 5, 10, 100, m / 200, m / 100, m / 50, m / 20, m / 10, m };
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 21af1e6..5445491 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -53,7 +53,7 @@ import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.kv.RowKeyEncoder;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -76,7 +76,7 @@ public class CubeStatsReader {
final int samplingPercentage;
final int mapperNumberOfFirstBuild; // becomes meaningless after merge
final double mapperOverlapRatioOfFirstBuild; // becomes meaningless after merge
- final Map<Long, HyperLogLogPlusCounter> cuboidRowEstimatesHLL;
+ final Map<Long, HyperLogLogPlusCounterNew> cuboidRowEstimatesHLL;
final CuboidScheduler cuboidScheduler;
public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) throws IOException {
@@ -96,7 +96,7 @@ public class CubeStatsReader {
int percentage = 100;
int mapperNumber = 0;
double mapperOverlapRatio = 0;
- Map<Long, HyperLogLogPlusCounter> counterMap = Maps.newHashMap();
+ Map<Long, HyperLogLogPlusCounterNew> counterMap = Maps.newHashMap();
LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), hadoopConf);
BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), hadoopConf);
@@ -108,7 +108,7 @@ public class CubeStatsReader {
} else if (key.get() == -2) {
mapperNumber = Bytes.toInt(value.getBytes());
} else if (key.get() > 0) {
- HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(kylinConfig.getCubeStatsHLLPrecision());
+ HyperLogLogPlusCounterNew hll = new HyperLogLogPlusCounterNew(kylinConfig.getCubeStatsHLLPrecision());
ByteArray byteArray = new ByteArray(value.getBytes());
hll.readRegisters(byteArray.asBuffer());
counterMap.put(key.get(), hll);
@@ -161,9 +161,9 @@ public class CubeStatsReader {
return mapperOverlapRatioOfFirstBuild;
}
- public static Map<Long, Long> getCuboidRowCountMapFromSampling(Map<Long, HyperLogLogPlusCounter> hllcMap, int samplingPercentage) {
+ public static Map<Long, Long> getCuboidRowCountMapFromSampling(Map<Long, HyperLogLogPlusCounterNew> hllcMap, int samplingPercentage) {
Map<Long, Long> cuboidRowCountMap = Maps.newHashMap();
- for (Map.Entry<Long, HyperLogLogPlusCounter> entry : hllcMap.entrySet()) {
+ for (Map.Entry<Long, HyperLogLogPlusCounterNew> entry : hllcMap.entrySet()) {
// No need to adjust according sampling percentage. Assumption is that data set is far
// more than cardinality. Even a percentage of the data should already see all cardinalities.
cuboidRowCountMap.put(entry.getKey(), entry.getValue().getCountEstimate());
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
index 74a2107..219cdf2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
@@ -33,17 +33,17 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
public class CubeStatsWriter {
public static void writeCuboidStatistics(Configuration conf, Path outputPath, //
- Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage) throws IOException {
+ Map<Long, HyperLogLogPlusCounterNew> cuboidHLLMap, int samplingPercentage) throws IOException {
writeCuboidStatistics(conf, outputPath, cuboidHLLMap, samplingPercentage, 0, 0);
}
public static void writeCuboidStatistics(Configuration conf, Path outputPath, //
- Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage, int mapperNumber, double mapperOverlapRatio) throws IOException {
+ Map<Long, HyperLogLogPlusCounterNew> cuboidHLLMap, int samplingPercentage, int mapperNumber, double mapperOverlapRatio) throws IOException {
Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
List<Long> allCuboids = new ArrayList<Long>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 776d750..0d388c7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -47,7 +47,7 @@ import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CubeStatsWriter;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,7 +64,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private List<TblColRef> columnList;
private String statisticsOutput = null;
private List<Long> baseCuboidRowCountInMappers;
- protected Map<Long, HyperLogLogPlusCounter> cuboidHLLMap = null;
+ protected Map<Long, HyperLogLogPlusCounterNew> cuboidHLLMap = null;
protected long baseCuboidId;
protected CubeDesc cubeDesc;
private long totalRowsBeforeMerge = 0;
@@ -156,7 +156,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
// for hll
long cuboidId = Bytes.toLong(key.getBytes(), 1, Bytes.SIZEOF_LONG);
for (Text value : values) {
- HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(cubeConfig.getCubeStatsHLLPrecision());
+ HyperLogLogPlusCounterNew hll = new HyperLogLogPlusCounterNew(cubeConfig.getCubeStatsHLLPrecision());
ByteBuffer bf = ByteBuffer.wrap(value.getBytes(), 0, value.getLength());
hll.readRegisters(bf);
@@ -270,7 +270,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
if (isStatistics) {
// output the hll info
long grandTotal = 0;
- for (HyperLogLogPlusCounter hll : cuboidHLLMap.values()) {
+ for (HyperLogLogPlusCounterNew hll : cuboidHLLMap.values()) {
grandTotal += hll.getCountEstimate();
}
double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
index a5c8fc0..c0575f1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
@@ -29,7 +29,7 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.collect.Lists;
@@ -45,7 +45,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
protected CuboidScheduler cuboidScheduler = null;
protected int nRowKey;
private Integer[][] allCuboidsBitSet = null;
- private HyperLogLogPlusCounter[] allCuboidsHLL = null;
+ private HyperLogLogPlusCounterNew[] allCuboidsHLL = null;
private Long[] cuboidIds;
private HashFunction hf = null;
private int rowCount = 0;
@@ -76,9 +76,9 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
allCuboidsBitSet = allCuboidsBitSetList.toArray(new Integer[cuboidIdList.size()][]);
cuboidIds = cuboidIdList.toArray(new Long[cuboidIdList.size()]);
- allCuboidsHLL = new HyperLogLogPlusCounter[cuboidIds.length];
+ allCuboidsHLL = new HyperLogLogPlusCounterNew[cuboidIds.length];
for (int i = 0; i < cuboidIds.length; i++) {
- allCuboidsHLL[i] = new HyperLogLogPlusCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision());
+ allCuboidsHLL[i] = new HyperLogLogPlusCounterNew(cubeDesc.getConfig().getCubeStatsHLLPrecision());
}
hf = Hashing.murmur3_32();
@@ -207,7 +207,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
if (collectStatistics) {
ByteBuffer hllBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
// output each cuboid's hll to reducer, key is 0 - cuboidId
- HyperLogLogPlusCounter hll;
+ HyperLogLogPlusCounterNew hll;
for (int i = 0; i < cuboidIds.length; i++) {
hll = allCuboidsHLL[i];
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index 88f6ba2..e839989 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -47,7 +47,7 @@ import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,7 +56,7 @@ import com.google.common.collect.Maps;
public class MergeStatisticsStep extends AbstractExecutable {
private static final Logger logger = LoggerFactory.getLogger(MergeStatisticsStep.class);
- protected Map<Long, HyperLogLogPlusCounter> cuboidHLLMap = Maps.newHashMap();
+ protected Map<Long, HyperLogLogPlusCounterNew> cuboidHLLMap = Maps.newHashMap();
public MergeStatisticsStep() {
super();
@@ -100,7 +100,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
// sampling percentage;
averageSamplingPercentage += Bytes.toInt(value.getBytes());
} else if (key.get() > 0) {
- HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(kylinConf.getCubeStatsHLLPrecision());
+ HyperLogLogPlusCounterNew hll = new HyperLogLogPlusCounterNew(kylinConf.getCubeStatsHLLPrecision());
ByteArray byteArray = new ByteArray(value.getBytes());
hll.readRegisters(byteArray.asBuffer());
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
index 89d23fa..cae3b62 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.junit.Before;
import org.junit.Test;
@@ -45,7 +45,7 @@ public class CubeSamplingTest {
private Integer[][] allCuboidsBitSet;
private HashFunction hf = null;
private long baseCuboidId;
- private HyperLogLogPlusCounter[] allCuboidsHLL = null;
+ private HyperLogLogPlusCounterNew[] allCuboidsHLL = null;
private final byte[] seperator = Bytes.toBytes(",");
@Before
@@ -61,9 +61,9 @@ public class CubeSamplingTest {
allCuboidsBitSet = allCuboidsBitSetList.toArray(new Integer[allCuboidsBitSetList.size()][]);
System.out.println("Totally have " + allCuboidsBitSet.length + " cuboids.");
- allCuboidsHLL = new HyperLogLogPlusCounter[allCuboids.size()];
+ allCuboidsHLL = new HyperLogLogPlusCounterNew[allCuboids.size()];
for (int i = 0; i < allCuboids.size(); i++) {
- allCuboidsHLL[i] = new HyperLogLogPlusCounter(14);
+ allCuboidsHLL[i] = new HyperLogLogPlusCounterNew(14);
}
// hf = Hashing.goodFastHash(32);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
index ca8684f..a00db94 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.CubeStatsWriter;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.junit.Test;
import com.google.common.collect.Maps;
@@ -48,7 +48,7 @@ public class FactDistinctColumnsReducerTest {
}
System.out.println(outputPath);
- Map<Long, HyperLogLogPlusCounter> cuboidHLLMap = Maps.newHashMap();
+ Map<Long, HyperLogLogPlusCounterNew> cuboidHLLMap = Maps.newHashMap();
CubeStatsWriter.writeCuboidStatistics(conf, outputPath, cuboidHLLMap, 100);
FileSystem.getLocal(conf).delete(outputPath, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 10c74f3..76212c8 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -83,7 +83,7 @@ import org.apache.kylin.engine.spark.cube.DefaultTupleConverter;
import org.apache.kylin.engine.spark.util.IteratorUtils;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.MeasureAggregators;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -241,15 +241,15 @@ public class SparkCubing extends AbstractApplication {
}
}
- private Map<Long, HyperLogLogPlusCounter> sampling(final JavaRDD<List<String>> rowJavaRDD, final String cubeName, String segmentId) throws Exception {
+ private Map<Long, HyperLogLogPlusCounterNew> sampling(final JavaRDD<List<String>> rowJavaRDD, final String cubeName, String segmentId) throws Exception {
CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).reloadCubeLocal(cubeName);
CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
CubeDesc cubeDesc = cubeInstance.getDescriptor();
CuboidScheduler cuboidScheduler = new CuboidScheduler(cubeDesc);
List<Long> allCuboidIds = cuboidScheduler.getAllCuboidIds();
- final HashMap<Long, HyperLogLogPlusCounter> zeroValue = Maps.newHashMap();
+ final HashMap<Long, HyperLogLogPlusCounterNew> zeroValue = Maps.newHashMap();
for (Long id : allCuboidIds) {
- zeroValue.put(id, new HyperLogLogPlusCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
+ zeroValue.put(id, new HyperLogLogPlusCounterNew(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
}
CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
@@ -278,12 +278,12 @@ public class SparkCubing extends AbstractApplication {
row_hashcodes[i] = new ByteArray();
}
- final HashMap<Long, HyperLogLogPlusCounter> samplingResult = rowJavaRDD.aggregate(zeroValue, new Function2<HashMap<Long, HyperLogLogPlusCounter>, List<String>, HashMap<Long, HyperLogLogPlusCounter>>() {
+ final HashMap<Long, HyperLogLogPlusCounterNew> samplingResult = rowJavaRDD.aggregate(zeroValue, new Function2<HashMap<Long, HyperLogLogPlusCounterNew>, List<String>, HashMap<Long, HyperLogLogPlusCounterNew>>() {
final HashFunction hashFunction = Hashing.murmur3_128();
@Override
- public HashMap<Long, HyperLogLogPlusCounter> call(HashMap<Long, HyperLogLogPlusCounter> v1, List<String> v2) throws Exception {
+ public HashMap<Long, HyperLogLogPlusCounterNew> call(HashMap<Long, HyperLogLogPlusCounterNew> v1, List<String> v2) throws Exception {
for (int i = 0; i < nRowKey; i++) {
Hasher hc = hashFunction.newHasher();
String colValue = v2.get(rowKeyColumnIndexes[i]);
@@ -296,7 +296,7 @@ public class SparkCubing extends AbstractApplication {
for (Map.Entry<Long, Integer[]> entry : allCuboidsBitSet.entrySet()) {
Hasher hc = hashFunction.newHasher();
- HyperLogLogPlusCounter counter = v1.get(entry.getKey());
+ HyperLogLogPlusCounterNew counter = v1.get(entry.getKey());
final Integer[] cuboidBitSet = entry.getValue();
for (int position = 0; position < cuboidBitSet.length; position++) {
hc.putBytes(row_hashcodes[cuboidBitSet[position]].array());
@@ -305,14 +305,14 @@ public class SparkCubing extends AbstractApplication {
}
return v1;
}
- }, new Function2<HashMap<Long, HyperLogLogPlusCounter>, HashMap<Long, HyperLogLogPlusCounter>, HashMap<Long, HyperLogLogPlusCounter>>() {
+ }, new Function2<HashMap<Long, HyperLogLogPlusCounterNew>, HashMap<Long, HyperLogLogPlusCounterNew>, HashMap<Long, HyperLogLogPlusCounterNew>>() {
@Override
- public HashMap<Long, HyperLogLogPlusCounter> call(HashMap<Long, HyperLogLogPlusCounter> v1, HashMap<Long, HyperLogLogPlusCounter> v2) throws Exception {
+ public HashMap<Long, HyperLogLogPlusCounterNew> call(HashMap<Long, HyperLogLogPlusCounterNew> v1, HashMap<Long, HyperLogLogPlusCounterNew> v2) throws Exception {
Preconditions.checkArgument(v1.size() == v2.size());
Preconditions.checkArgument(v1.size() > 0);
- for (Map.Entry<Long, HyperLogLogPlusCounter> entry : v1.entrySet()) {
- final HyperLogLogPlusCounter counter1 = entry.getValue();
- final HyperLogLogPlusCounter counter2 = v2.get(entry.getKey());
+ for (Map.Entry<Long, HyperLogLogPlusCounterNew> entry : v1.entrySet()) {
+ final HyperLogLogPlusCounterNew counter1 = entry.getValue();
+ final HyperLogLogPlusCounterNew counter2 = v2.get(entry.getKey());
counter1.merge(Preconditions.checkNotNull(counter2, "counter cannot be null"));
}
return v1;
@@ -470,7 +470,7 @@ public class SparkCubing extends AbstractApplication {
ClassUtil.addClasspath(confPath);
}
- private byte[][] createHTable(String cubeName, String segmentId, Map<Long, HyperLogLogPlusCounter> samplingResult) throws Exception {
+ private byte[][] createHTable(String cubeName, String segmentId, Map<Long, HyperLogLogPlusCounterNew> samplingResult) throws Exception {
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName);
final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
@@ -614,7 +614,7 @@ public class SparkCubing extends AbstractApplication {
}
});
- final Map<Long, HyperLogLogPlusCounter> samplingResult = sampling(rowJavaRDD, cubeName, segmentId);
+ final Map<Long, HyperLogLogPlusCounterNew> samplingResult = sampling(rowJavaRDD, cubeName, segmentId);
final byte[][] splitKeys = createHTable(cubeName, segmentId, samplingResult);
final String hfile = build(rowJavaRDD, cubeName, segmentId, splitKeys);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
index 06a07ca..230249f 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
@@ -35,18 +35,18 @@ import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
/**
* @author Jack
- *
+ *
*/
public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritable, BytesWritable> {
- private Map<Integer, HyperLogLogPlusCounter> hllcMap = new HashMap<Integer, HyperLogLogPlusCounter>();
+ private Map<Integer, HyperLogLogPlusCounterNew> hllcMap = new HashMap<Integer, HyperLogLogPlusCounterNew>();
public static final String DEFAULT_DELIM = ",";
private int counter = 0;
@@ -87,9 +87,9 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab
counter++;
}
- private HyperLogLogPlusCounter getHllc(Integer key) {
+ private HyperLogLogPlusCounterNew getHllc(Integer key) {
if (!hllcMap.containsKey(key)) {
- hllcMap.put(key, new HyperLogLogPlusCounter());
+ hllcMap.put(key, new HyperLogLogPlusCounterNew());
}
return hllcMap.get(key);
}
@@ -100,7 +100,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab
ByteBuffer buf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
while (it.hasNext()) {
int key = it.next();
- HyperLogLogPlusCounter hllc = hllcMap.get(key);
+ HyperLogLogPlusCounterNew hllc = hllcMap.get(key);
buf.clear();
hllc.writeRegisters(buf);
buf.flip();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
index ea66999..32cc6d9 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
/**
* @author Jack
@@ -41,7 +41,7 @@ import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWritable, IntWritable, LongWritable> {
public static final int ONE = 1;
- private Map<Integer, HyperLogLogPlusCounter> hllcMap = new HashMap<Integer, HyperLogLogPlusCounter>();
+ private Map<Integer, HyperLogLogPlusCounterNew> hllcMap = new HashMap<Integer, HyperLogLogPlusCounterNew>();
@Override
protected void setup(Context context) throws IOException {
@@ -53,16 +53,16 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri
int skey = key.get();
for (BytesWritable v : values) {
ByteBuffer buffer = ByteBuffer.wrap(v.getBytes());
- HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter();
+ HyperLogLogPlusCounterNew hll = new HyperLogLogPlusCounterNew();
hll.readRegisters(buffer);
getHllc(skey).merge(hll);
hll.clear();
}
}
- private HyperLogLogPlusCounter getHllc(Integer key) {
+ private HyperLogLogPlusCounterNew getHllc(Integer key) {
if (!hllcMap.containsKey(key)) {
- hllcMap.put(key, new HyperLogLogPlusCounter());
+ hllcMap.put(key, new HyperLogLogPlusCounterNew());
}
return hllcMap.get(key);
}
@@ -78,7 +78,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri
it = keys.iterator();
while (it.hasNext()) {
int key = it.next();
- HyperLogLogPlusCounter hllc = hllcMap.get(key);
+ HyperLogLogPlusCounterNew hllc = hllcMap.get(key);
ByteBuffer buf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
buf.clear();
hllc.writeRegisters(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f05404d5/source-hive/src/test/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducerTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducerTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducerTest.java
index d27860a..410543a 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducerTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducerTest.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.apache.hadoop.mrunit.types.Pair;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
+import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
import org.junit.Before;
import org.junit.Test;
@@ -57,7 +57,7 @@ public class ColumnCardinalityReducerTest {
}
private byte[] getBytes(String str) throws IOException {
- HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
+ HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew();
StringTokenizer tokenizer = new StringTokenizer(str, ColumnCardinalityMapper.DEFAULT_DELIM);
int i = 0;
while (tokenizer.hasMoreTokens()) {
[32/50] [abbrv] kylin git commit: KYLIN-2283 A new data gen tool for
CI
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index 2f8991b..df6eb2f 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -217,7 +217,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact " + "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME");
- statement.setString(1, "FP-GTC_A");
+ statement.setString(1, "FP-GTC");
ResultSet rs = statement.executeQuery();
@@ -225,7 +225,7 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
String format_name = rs.getString(1);
- Assert.assertTrue("FP-GTC_A".equals(format_name));
+ Assert.assertTrue("FP-GTC".equals(format_name));
rs.close();
statement.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/kylin-it/src/test/resources/query/sql_timeout/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_timeout/query02.sql b/kylin-it/src/test/resources/query/sql_timeout/query02.sql
index 2f187a4..a00096c 100644
--- a/kylin-it/src/test/resources/query/sql_timeout/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_timeout/query02.sql
@@ -16,4 +16,4 @@
-- limitations under the License.
--
-select seller_id,lstg_format_name,sum(price) from test_kylin_fact group by seller_id,lstg_format_name
\ No newline at end of file
+select seller_id,cal_dt,lstg_format_name,sum(price) from test_kylin_fact group by seller_id,cal_dt,lstg_format_name
\ No newline at end of file
[46/50] [abbrv] kylin git commit: KYLIN-2287 Speed up model and cube
list load in Web
Posted by li...@apache.org.
KYLIN-2287 Speed up model and cube list load in Web
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64a0a594
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64a0a594
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64a0a594
Branch: refs/heads/master-cdh5.7
Commit: 64a0a5943c034ea6f9cdc6d179d8be9da64fe5ac
Parents: f50c0c8
Author: kangkaisen <ka...@live.com>
Authored: Wed Dec 14 20:08:14 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Tue Dec 20 15:12:54 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeSchema.js | 25 +-
webapp/app/js/controllers/cubes.js | 318 ++++++++++-------------
webapp/app/js/controllers/models.js | 54 ++--
webapp/app/js/model/cubeListModel.js | 4 -
webapp/app/js/model/modelsManager.js | 49 +---
webapp/app/partials/cubes/cubes.html | 12 +-
webapp/app/partials/models/models_tree.html | 16 +-
7 files changed, 215 insertions(+), 263 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/js/controllers/cubeSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js
index 357b6af..af8ee7c 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -45,18 +45,6 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic
$scope.state = {mode: "view"};
}
- var queryParam = {offset: 0, limit: 65535};
-
- CubeService.list(queryParam, function (all_cubes) {
- if($scope.allCubes.length > 0){
- $scope.allCubes.splice(0,$scope.allCubes.length);
- }
-
- for (var i = 0; i < all_cubes.length; i++) {
- $scope.allCubes.push(all_cubes[i].name.toUpperCase());
- }
- });
-
$scope.$watch('cubeMetaFrame', function (newValue, oldValue) {
if(!newValue){
return;
@@ -198,6 +186,19 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic
};
$scope.check_cube_info = function(){
+
+ var queryParam = {offset: 0, limit: 65535};
+
+ CubeService.list(queryParam, function (all_cubes) {
+ if($scope.allCubes.length > 0){
+ $scope.allCubes.splice(0,$scope.allCubes.length);
+ }
+
+ for (var i = 0; i < all_cubes.length; i++) {
+ $scope.allCubes.push(all_cubes[i].name.toUpperCase());
+ }
+ });
+
if(($scope.state.mode === "edit") &&$scope.cubeMode=="addNewCube"&&($scope.allCubes.indexOf($scope.cubeMetaFrame.name.toUpperCase()) >= 0)){
SweetAlert.swal('Oops...', "The cube named [" + $scope.cubeMetaFrame.name.toUpperCase() + "] already exists", 'warning');
return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/js/controllers/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
index b672d3f..cbdbb7d 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -79,43 +79,6 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
return CubeList.list(queryParam).then(function (resp) {
angular.forEach($scope.cubeList.cubes,function(cube,index){
- cube.streaming = false;
- CubeDescService.query({cube_name: cube.name}, {}, function (detail) {
- if (detail.length > 0 && detail[0].hasOwnProperty("name")) {
- cube.detail = detail[0];
- ModelService.list({projectName:$scope.projectModel.selectedProject,modelName:cube.detail.model_name}, function (_models) {
- if(_models && _models.length){
- for(var i=0;i<=_models.length;i++){
- if(_models[i].name == cube.detail.model_name){
- cube.model = _models[i];
- var factTable = cube.model.fact_table;
- TableService.get({tableName:factTable},function(table){
- if(table && table.source_type == 1){
- cube.streaming = true;
- }
- })
- break;
- }
- }
- }
-
- })
- //cube.model = modelsManager.getModel(cube.detail.model_name);
-
- defer.resolve(cube.detail);
-
- } else {
- SweetAlert.swal('Oops...', "No cube detail info loaded.", 'error');
- }
- }, function (e) {
- if (e.data && e.data.exception) {
- var message = e.data.exception;
- var msg = !!(message) ? message : 'Failed to take action.';
- SweetAlert.swal('Oops...', msg, 'error');
- } else {
- SweetAlert.swal('Oops...', "Failed to take action.", 'error');
- }
- });
})
$scope.loading = false;
@@ -331,140 +294,139 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
};
$scope.startJobSubmit = function (cube) {
- $scope.loadDetail(cube);
- // for streaming cube build tip
- if(cube.streaming){
-
- SweetAlert.swal({
- title: '',
- text: "Are you sure to start the build?",
- type: '',
- showCancelButton: true,
- confirmButtonColor: '#DD6B55',
- confirmButtonText: "Yes",
- closeOnConfirm: true
- }, function(isConfirm) {
- if(isConfirm){
- loadingRequest.show();
- CubeService.rebuildStreamingCube(
- {
- cubeId: cube.name
- },
- {
- sourceOffsetStart:0,
- sourceOffsetEnd:'9223372036854775807',
- buildType:'BUILD'
- }, function (job) {
- loadingRequest.hide();
- SweetAlert.swal('Success!', 'Rebuild job was submitted successfully', 'success');
- },function(e){
-
- loadingRequest.hide();
- if(e.data&& e.data.exception){
- var message =e.data.exception;
- var msg = !!(message) ? message : 'Failed to take action.';
- SweetAlert.swal('Oops...', msg, 'error');
- }else{
- SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+
+ $scope.loadDetail(cube).then(function () {
+ $scope.metaModel={
+ model:cube.model
+ };
+
+ TableService.get({tableName:$scope.metaModel.model.fact_table},function(table){
+ if(table && table.source_type == 1){
+ cube.streaming = true;
+ }
+
+ // for streaming cube build tip
+ if(cube.streaming){
+ $modal.open({
+ templateUrl: 'streamingBuild.html',
+ controller: streamingBuildCtrl,
+ resolve: {
+ cube: function () {
+ return cube;
+ },
+ metaModel:function(){
+ return $scope.metaModel;
+ },
+ buildType: function () {
+ return 'BUILD';
+ },
+ scope:function(){
+
+ return $scope;
}
+ }
});
+ return;
}
- })
- return;
- }
- $scope.metaModel={
- model:modelsManager.getModelByCube(cube.name)
- }
- if ($scope.metaModel.model.name) {
- if ($scope.metaModel.model.partition_desc.partition_date_column) {
-
- $modal.open({
- templateUrl: 'jobSubmit.html',
- controller: jobSubmitCtrl,
- resolve: {
- cube: function () {
- return cube;
- },
- metaModel:function(){
- return $scope.metaModel;
- },
- buildType: function () {
- return 'BUILD';
- },
- scope:function(){
-
- return $scope;
- }
+ //for batch cube build tip
+ if ($scope.metaModel.model.name) {
+
+ //for partition cube build tip
+ if ($scope.metaModel.model.partition_desc.partition_date_column) {
+ $modal.open({
+ templateUrl: 'jobSubmit.html',
+ controller: jobSubmitCtrl,
+ resolve: {
+ cube: function () {
+ return cube;
+ },
+ metaModel:function(){
+ return $scope.metaModel;
+ },
+ buildType: function () {
+ return 'BUILD';
+ },
+ scope:function(){
+ return $scope;
+ }
+ }
+ });
}
- });
- }
- else {
- SweetAlert.swal({
- title: '',
- text: "Are you sure to start the build ?",
- type: '',
- showCancelButton: true,
- confirmButtonColor: '#DD6B55',
- confirmButtonText: "Yes",
- closeOnConfirm: true
- }, function(isConfirm) {
- if(isConfirm){
+ //for not partition cube build tip
+ else {
+ SweetAlert.swal({
+ title: '',
+ text: "Are you sure to start the build ?",
+ type: '',
+ showCancelButton: true,
+ confirmButtonColor: '#DD6B55',
+ confirmButtonText: "Yes",
+ closeOnConfirm: true
+ }, function(isConfirm) {
+ if(isConfirm){
+
+ loadingRequest.show();
+ CubeService.rebuildCube(
+ {
+ cubeId: cube.name
+ },
+ {
+ buildType: 'BUILD',
+ startTime: 0,
+ endTime: 0
+ }, function (job) {
+
+ loadingRequest.hide();
+ SweetAlert.swal('Success!', 'Rebuild job was submitted successfully', 'success');
+ },function(e){
+
+ loadingRequest.hide();
+ if(e.data&& e.data.exception){
+ var message =e.data.exception;
+ var msg = !!(message) ? message : 'Failed to take action.';
+ SweetAlert.swal('Oops...', msg, 'error');
+ }else{
+ SweetAlert.swal('Oops...', "Failed to take action.", 'error');
+ }
+ });
+ }
- loadingRequest.show();
- CubeService.rebuildCube(
- {
- cubeId: cube.name
- },
- {
- buildType: 'BUILD',
- startTime: 0,
- endTime: 0
- }, function (job) {
-
- loadingRequest.hide();
- SweetAlert.swal('Success!', 'Rebuild job was submitted successfully', 'success');
- },function(e){
-
- loadingRequest.hide();
- if(e.data&& e.data.exception){
- var message =e.data.exception;
- var msg = !!(message) ? message : 'Failed to take action.';
- SweetAlert.swal('Oops...', msg, 'error');
- }else{
- SweetAlert.swal('Oops...', "Failed to take action.", 'error');
- }
- });
+ });
}
+ }
+ })
+ })
- });
- }
- }
};
$scope.startRefresh = function (cube) {
- $scope.metaModel={
- model:modelsManager.getModelByCube(cube.name)
- };
- $modal.open({
- templateUrl: 'jobRefresh.html',
- controller: jobSubmitCtrl,
- resolve: {
- cube: function () {
- return cube;
- },
- metaModel:function(){
- return $scope.metaModel;
- },
- buildType: function () {
- return 'REFRESH';
- },
- scope:function(){
- return $scope;
+
+ $scope.loadDetail(cube).then(function () {
+ $scope.metaModel={
+ model:cube.model
+ };
+ $modal.open({
+ templateUrl: 'jobRefresh.html',
+ controller: jobSubmitCtrl,
+ resolve: {
+ cube: function () {
+ return cube;
+ },
+ metaModel:function(){
+ return $scope.metaModel;
+ },
+ buildType: function () {
+ return 'REFRESH';
+ },
+ scope:function(){
+ return $scope;
+ }
}
+ });
}
- });
+ )
};
@@ -493,27 +455,29 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
$location.path("cubes/edit/" + cube.name);
}
$scope.startMerge = function (cube) {
- $scope.metaModel={
- model:modelsManager.getModelByCube(cube.name)
- };
- $modal.open({
- templateUrl: 'jobMerge.html',
- controller: jobSubmitCtrl,
- resolve: {
- cube: function () {
- return cube;
- },
- metaModel:function(){
- return $scope.metaModel;
- },
- buildType: function () {
- return 'MERGE';
- },
- scope:function(){
- return $scope;
+ $scope.loadDetail(cube).then(function () {
+ $scope.metaModel={
+ model:cube.model
+ };
+ $modal.open({
+ templateUrl: 'jobMerge.html',
+ controller: jobSubmitCtrl,
+ resolve: {
+ cube: function () {
+ return cube;
+ },
+ metaModel:function(){
+ return $scope.metaModel;
+ },
+ buildType: function () {
+ return 'MERGE';
+ },
+ scope:function(){
+ return $scope;
+ }
}
- }
- });
+ });
+ })
}
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/js/controllers/models.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/models.js b/webapp/app/js/controllers/models.js
index fb2c6d1..e3ee038 100644
--- a/webapp/app/js/controllers/models.js
+++ b/webapp/app/js/controllers/models.js
@@ -18,7 +18,7 @@
'use strict';
-KylinApp.controller('ModelsCtrl', function ($scope, $q, $routeParams, $location, $window, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService, SweetAlert, loadingRequest, $log, modelConfig, ProjectModel, ModelService, MetaModel, modelsManager, cubesManager, TableModel, $animate) {
+KylinApp.controller('ModelsCtrl', function ($scope, $q, $routeParams, $location, $window, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService, SweetAlert, loadingRequest, $log, modelConfig, ProjectModel, ModelService, MetaModel, modelsManager, cubesManager, TableModel, AccessService) {
//tree data
@@ -131,21 +131,28 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, $routeParams, $location,
var cubename = [];
var modelstate=false;
var i=0;
- if (model.cubes.length != 0) {
- angular.forEach(model.cubes,function(cube){
- if (cube.status=="READY"){
- modelstate=true;
- cubename[i] =cube.name;
- i++;
- }
- })
- }
- if(modelstate==false){
- $location.path("/models/edit/"+model.name);
- }
- else{
- SweetAlert.swal('Sorry','This model is still used by '+ cubename.join(','));
- }
+
+ CubeService.list({modelName:model.name}, function (_cubes) {
+ model.cubes = _cubes;
+
+ if (model.cubes.length != 0) {
+ angular.forEach(model.cubes,function(cube){
+ if (cube.status=="READY"){
+ modelstate=true;
+ cubename[i] =cube.name;
+ i++;
+ }
+ })
+ }
+
+ if(modelstate==false){
+ $location.path("/models/edit/"+model.name);
+ }
+ else{
+ SweetAlert.swal('Sorry','This model is still used by '+ cubename.join(','));
+ }
+ })
+
};
$scope.cloneModel = function(model){
@@ -176,6 +183,21 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, $routeParams, $location,
});
};
+ $scope.listModelAccess = function (model) {
+ if(model.uuid){
+ AccessService.list({type: "DataModelDesc", uuid: model.uuid}, function (accessEntities) {
+ model.accessEntities = accessEntities;
+ try {
+ if (!model.owner) {
+ model.owner = accessEntities[0].sid.principal;
+ }
+ } catch (error) {
+ $log.error("No acl info.");
+ }
+ })
+ }
+ };
+
var ModelDetailModalCtrl = function ($scope, $location, $modalInstance, scope) {
modelsManager.selectedModel.visiblePage='metadata';
$scope.cancel = function () {
http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/js/model/cubeListModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeListModel.js b/webapp/app/js/model/cubeListModel.js
index 13c924c..39853c0 100755
--- a/webapp/app/js/model/cubeListModel.js
+++ b/webapp/app/js/model/cubeListModel.js
@@ -25,10 +25,6 @@ KylinApp.service('CubeList',function(CubeService,$q,AccessService){
var defer = $q.defer();
CubeService.list(queryParam, function (_cubes) {
angular.forEach(_cubes, function (cube, index) {
- AccessService.list({type: "CubeInstance", uuid: cube.uuid}, function (accessEntities) {
- cube.accessEntities = accessEntities;
- });
-
if(cube.name){
if (cube.segments && cube.segments.length > 0) {
for(var i= cube.segments.length-1;i>=0;i--){
http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/js/model/modelsManager.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/modelsManager.js b/webapp/app/js/model/modelsManager.js
index c104405..faa8d39 100644
--- a/webapp/app/js/model/modelsManager.js
+++ b/webapp/app/js/model/modelsManager.js
@@ -25,56 +25,22 @@ KylinApp.service('modelsManager',function(ModelService,CubeService,$q,AccessServ
this.loading = false;
this.selectedModel={};
- this.cubeModel={};
- this.cubeSelected = false;
-
- //list models and complemete cube,access info
+ //list models
this.list = function(queryParam){
_this.loading = true;
var defer = $q.defer();
- var cubeDetail = [];
- var modelPermission = [];
ModelService.list(queryParam, function (_models) {
- //_this.removeAll();
angular.forEach(_models, function (model, index) {
- $log.info("Add model permission info");
- if(model.uuid){
- modelPermission.push(
- AccessService.list({type: "DataModelDesc", uuid: model.uuid}, function (accessEntities) {
- model.accessEntities = accessEntities;
- try{
- if(!model.owner){
- model.owner = accessEntities[0].sid.principal;
- }
- } catch(error){
- $log.error("No acl info.");
- }
-
- }).$promise
- )
- }
-
- $log.info("Add cube info to model ,not detail info");
- cubeDetail.push(
- CubeService.list({modelName:model.name}, function (_cubes) {
- model.cubes = _cubes;
- }).$promise
- );
-
_this.modelNameList.push(model.name);
-
- model.project = ProjectModel.getProjectByCubeModel(model.name);
+ model.project = ProjectModel.getProjectByCubeModel(model.name);
});
- $q.all(cubeDetail,modelPermission).then(
- function(result){
- _models = _.filter(_models,function(models){return models.name!=undefined});
- _this.models = _models;
- _this.loading = false;
- defer.resolve(_this.models);
- }
- );
+
+ _models = _.filter(_models,function(models){return models.name!=undefined});
+ _this.models = _models;
+ _this.loading = false;
+
},function(){
defer.reject("Failed to load models");
});
@@ -127,5 +93,4 @@ KylinApp.service('modelsManager',function(ModelService,CubeService,$q,AccessServ
return defer.promise;
};
-
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/partials/cubes/cubes.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html
index 0187523..4be7ecd 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -54,7 +54,6 @@
</th>
<th>Actions</th>
<th ng-if="userService.hasRole('ROLE_ADMIN')">Admins</th>
- <th>Streaming</th>
</tr>
</thead>
<!--Body-->
@@ -83,12 +82,12 @@
<td>{{ cube.owner}}</td>
<td>{{ cube.create_time_utc | utcToConfigTimeZone}}</td>
<td>
- <div ng-click="$event.stopPropagation();" class="btn-group" ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask)">
+ <div ng-click="$event.stopPropagation();" class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle"
data-toggle="dropdown" ng-click="listAccess(cube, 'CubeInstance')">
Action <span class="ace-icon fa fa-caret-down icon-on-right"></span>
</button>
- <ul class="dropdown-menu" role="menu">
+ <ul ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask)" class="dropdown-menu" role="menu">
<li ng-if="cube.status!='READY' && userService.hasRole('ROLE_ADMIN') ">
<a ng-click="dropCube(cube)" tooltip="Drop the cube, related jobs and data permanently.">Drop</a></li>
<li ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))">
@@ -108,20 +107,17 @@
N/A
</span>
</td>
- <td ng-if="userService.hasRole('ROLE_ADMIN')">
+ <td>
<div ng-click="$event.stopPropagation();" class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" ng-click="listAccess(cube, 'CubeInstance')">
Action <span class="ace-icon fa fa-caret-down icon-on-right"></span>
</button>
- <ul class="dropdown-menu" role="menu">
+ <ul class="dropdown-menu" role="menu" ng-if="userService.hasRole('ROLE_ADMIN')">
<li ng-if="cube.status!='READY'"><a href="cubes/edit/{{cube.name}}/descriptionjson">Edit CubeDesc</a></li>
<li><a href="cubes/view/{{cube.name}}/instancejson">View Cube</a></li>
</ul>
</div>
</td>
- <td>
- <label class="badge" ng-class="{'label-info':cube.streaming==true}" style="cursor:pointer;">{{cube.streaming}}</label>
- </td>
</tr>
<tr ng-show="cube.showDetail">
<td colspan="10" style="padding: 10px 30px 10px 30px;">
http://git-wip-us.apache.org/repos/asf/kylin/blob/64a0a594/webapp/app/partials/models/models_tree.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/models/models_tree.html b/webapp/app/partials/models/models_tree.html
index 530139e..944d2dc 100644
--- a/webapp/app/partials/models/models_tree.html
+++ b/webapp/app/partials/models/models_tree.html
@@ -41,15 +41,23 @@
<div>
<h3 class="text-info">Models</h3>
</div>
+
<div style="width:100%; height:{{window}}px; overflow:auto;margin-top: 20px;" class="cube_model_trees">
<ul class="list-group models-tree">
<li class="list-group-item" ng-repeat="model in modelsManager.models">
- <div class="pull-right" showonhoverparent style="display:none;">
- <a ng-click="editModel(model)" title="Edit Model" style="cursor:pointer;margin-right: 8px;" ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission(model, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask))"><span class="fa fa-pencil fa-lg fa-fw"></span></a>
- <a ng-click="cloneModel(model)" title="Clone Model" style="cursor:pointer;margin-right: 8px;" ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission(model, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask))"><span class="fa fa-copy fa-lg fa-fw"></span></a>
- <a ng-click="dropModel(model)" title="Drop Model" style="cursor:pointer;margin-right: 8px;" ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission(model, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask))"><span class="fa fa-trash-o fa-lg fa-fw"></span></a>
+ <div class="pull-right" showonhoverparent style="display:none;" >
+ <div ng-click="$event.stopPropagation();" class="btn-group">
+ <button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown" ng-click="listModelAccess(model)">
+ Action <span class="ace-icon fa fa-caret-down icon-on-right"></span>
+ </button>
+ <ul class="dropdown-menu" role="menu" ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission(model, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask))">
+ <li><a ng-click="editModel(model)" title="Edit Model" style="cursor:pointer;margin-right: 8px;" >Edit</a></li>
+ <li><a ng-click="cloneModel(model)" title="Clone Model" style="cursor:pointer;margin-right: 8px;" >Clone </a></li>
+ <li><a ng-click="dropModel(model)" title="Drop Model" style="cursor:pointer;margin-right: 8px;">Drop</a></li>
+ </ul>
+ </div>
</div>
<span class="strong"><a style="cursor: pointer;word-break:break-all;" ng-click="openModal(model)">{{model.name}}</a></span>
[36/50] [abbrv] kylin git commit: KYLIN-2288 recogonize measure empty
string as zero
Posted by li...@apache.org.
KYLIN-2288 recogonize measure empty string as zero
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d53ce9a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d53ce9a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d53ce9a
Branch: refs/heads/master-cdh5.7
Commit: 1d53ce9a0183ec369a681dec29b412f77f681f3e
Parents: d1175d2
Author: NiChuanlei <ni...@360.cn>
Authored: Fri Dec 16 16:10:37 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Dec 19 11:50:14 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/measure/basic/BigDecimalIngester.java | 2 +-
.../main/java/org/apache/kylin/measure/basic/DoubleIngester.java | 2 +-
.../src/main/java/org/apache/kylin/measure/basic/LongIngester.java | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d53ce9a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
index b51917c..c7541ab 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalIngester.java
@@ -33,7 +33,7 @@ public class BigDecimalIngester extends MeasureIngester<BigDecimal> {
if (values.length > 1)
throw new IllegalArgumentException();
- if (values[0] == null)
+ if (values[0] == null || values[0].length() == 0)
return new BigDecimal(0);
else
return new BigDecimal(values[0]);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d53ce9a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
index 8356faa..a2f3980 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/DoubleIngester.java
@@ -37,7 +37,7 @@ public class DoubleIngester extends MeasureIngester<DoubleMutable> {
throw new IllegalArgumentException();
DoubleMutable l = current;
- if (values[0] == null)
+ if (values[0] == null || values[0].length() == 0)
l.set(0L);
else
l.set(Double.parseDouble(values[0]));
http://git-wip-us.apache.org/repos/asf/kylin/blob/1d53ce9a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
index bfe6fe8..45a1634 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/LongIngester.java
@@ -37,7 +37,7 @@ public class LongIngester extends MeasureIngester<LongMutable> {
throw new IllegalArgumentException();
LongMutable l = current;
- if (values[0] == null)
+ if (values[0] == null || values[0].length() == 0)
l.set(0L);
else
l.set(Long.parseLong(values[0]));
[43/50] [abbrv] kylin git commit: KYLIN-2295 Refactor CI,
blend view cubes into the rest
Posted by li...@apache.org.
KYLIN-2295 Refactor CI, blend view cubes into the rest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f563df4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f563df4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f563df4
Branch: refs/heads/master-cdh5.7
Commit: 6f563df4afa46af61ffcbbc3ad599098b737d07a
Parents: 4ae4333
Author: Yang Li <li...@apache.org>
Authored: Sun Dec 18 20:24:25 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Mon Dec 19 16:03:22 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 22 +-
.../kylin/source/datagen/ColumnGenConfig.java | 18 +-
.../source/datagen/ModelDataGenerator.java | 10 +-
.../kylin/source/datagen/TableGenConfig.java | 5 +-
...t_kylin_cube_with_view_inner_join_empty.json | 10 -
...st_kylin_cube_with_view_left_join_empty.json | 10 -
...st_kylin_cube_with_view_inner_join_desc.json | 169 ----
...est_kylin_cube_with_view_left_join_desc.json | 169 ----
.../localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv | 8 -
.../data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv | 8 +
.../test_kylin_inner_join_view_model_desc.json | 119 ---
.../test_kylin_left_join_view_model_desc.json | 119 ---
.../localmeta/project/default.json | 14 +-
.../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json | 302 +++----
.../table/DEFAULT.TEST_KYLIN_COUNTRY.json | 46 +-
.../table/DEFAULT.TEST_KYLIN_FACT.json | 133 +--
.../localmeta/table/EDW.TEST_CAL_DT.json | 814 +++++++++----------
.../table/EDW.TEST_SELLER_TYPE_DIM.json | 86 +-
.../table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json | 43 +
.../localmeta/table/EDW.TEST_SITES.json | 94 +--
.../localmeta/table/EDW.V_TEST_CAL_DT.json | 409 ----------
.../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json | 814 +++++++++----------
.../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json | 298 +++----
.../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json | 46 +-
.../localmeta/table/SNOWTEST.KYLIN_SALES.json | 112 +--
.../kylin/provision/BuildCubeWithEngine.java | 83 +-
.../java/org/apache/kylin/query/H2Database.java | 14 +-
.../apache/kylin/query/ITKylinQueryTest.java | 2 +-
.../org/apache/kylin/query/KylinTestBase.java | 1 -
29 files changed, 1498 insertions(+), 2480 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 8fc583d..c8b0d43 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -122,13 +122,13 @@ public class DeployUtil {
// ============================================================================
static final String TABLE_CAL_DT = "edw.test_cal_dt";
- static final String VIEW_CAL_DT = "edw.v_test_cal_dt";
static final String TABLE_CATEGORY_GROUPINGS = "default.test_category_groupings";
static final String TABLE_KYLIN_FACT = "default.test_kylin_fact";
- static final String TABLE_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
+ static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
+ static final String TABLE_SELLER_TYPE_DIM_TABLE = "edw.test_seller_type_dim_table";
static final String TABLE_SITES = "edw.test_sites";
- static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM, TABLE_SITES };
+ static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES };
public static void prepareTestDataForNormalCubes(String cubeName) throws Exception {
@@ -230,7 +230,7 @@ public class DeployUtil {
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase())));
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase())));
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM.toUpperCase())));
+ hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase())));
hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase())));
// load data to hive tables
@@ -238,11 +238,11 @@ public class DeployUtil {
hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir));
hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, tableFileDir));
hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM, tableFileDir));
+ hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM_TABLE, tableFileDir));
hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES, tableFileDir));
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
- hiveCmdBuilder.addStatements(generateCreateViewHql(VIEW_CAL_DT, metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase())));
+ hiveCmdBuilder.addStatements(generateCreateViewHql(VIEW_SELLER_TYPE_DIM, TABLE_SELLER_TYPE_DIM_TABLE));
config().getCliCommandExecutor().execute(hiveCmdBuilder.build());
}
@@ -274,14 +274,14 @@ public class DeployUtil {
return new String[] { dropsql, ddl.toString() };
}
- private static String[] generateCreateViewHql(String viewName, TableDesc tableDesc) {
+ private static String[] generateCreateViewHql(String viewName, String tableName) {
- String dropsql = "DROP VIEW IF EXISTS " + viewName + ";";
- StringBuilder ddl = new StringBuilder();
+ String dropView = "DROP VIEW IF EXISTS " + viewName + ";\n";
+ String dropTable = "DROP TABLE IF EXISTS " + viewName + ";\n";
- ddl.append("CREATE VIEW " + viewName + " AS SELECT * FROM " + tableDesc.getIdentity() + ";\n");
+ String createSql = ("CREATE VIEW " + viewName + " AS SELECT * FROM " + tableName + ";\n");
- return new String[] { dropsql, ddl.toString() };
+ return new String[] { dropView, dropTable, createSql };
}
private static String getHiveDataType(String javaDataType) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
index 6387873..62fe46c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
@@ -23,6 +23,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
public class ColumnGenConfig {
@@ -69,7 +71,7 @@ public class ColumnGenConfig {
if (FK.equals(values.get(0)) || (values.get(0).isEmpty() && pkValues != null)) {
isFK = true;
- values = pkValues;
+ values = getPkValues(modelGen, config, pkValues);
} else if (ID.equals(values.get(0))) {
isID = true;
idStart = (values.size() > 1) ? Integer.parseInt(values.get(1)) : 0;
@@ -90,6 +92,20 @@ public class ColumnGenConfig {
unique = Util.parseBoolean(config, "uniq", modelGen.isPK(col));
}
+ private List<String> getPkValues(ModelDataGenerator modelGen, Map<String, String> config, List<String> dftPkValues) throws IOException {
+ String pkColName = config.get("pk");
+ if (pkColName == null)
+ return dftPkValues;
+
+ int cut = pkColName.lastIndexOf('.');
+ String pkTableName = pkColName.substring(0, cut);
+ pkColName = pkColName.substring(cut + 1);
+
+ KylinConfig kylinConfig = modelGen.getModle().getConfig();
+ ColumnDesc pkcol = MetadataManager.getInstance(kylinConfig).getTableDesc(pkTableName).findColumnByName(pkColName);
+ return modelGen.getPkValues(pkcol);
+ }
+
private int guessCardinality(String col) {
for (String s : col.split("_")) {
if (s.startsWith("C")) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
index c325ab0..60f2736 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
@@ -166,6 +166,9 @@ public class ModelDataGenerator {
private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) {
for (TableDesc t : tables) {
+ if (t.isView())
+ continue;
+
out.print("DROP TABLE IF EXISTS " + t.getIdentity() + ";\n");
out.print("CREATE TABLE " + t.getIdentity() + "(" + "\n");
@@ -198,6 +201,11 @@ public class ModelDataGenerator {
private void generateLoadDataDDL(Set<TableDesc> tables, PrintWriter out) {
for (TableDesc t : tables) {
+ if (t.isView()) {
+ out.print("-- " + t.getIdentity() + " is view \n");
+ continue;
+ }
+
out.print("LOAD DATA LOCAL INPATH '" + t.getIdentity() + ".csv' OVERWRITE INTO TABLE " + t.getIdentity() + ";\n");
}
}
@@ -241,7 +249,7 @@ public class ModelDataGenerator {
return null;
}
- private List<String> getPkValues(ColumnDesc pk) throws IOException {
+ public List<String> getPkValues(ColumnDesc pk) throws IOException {
if (existsInStore(pk.getTable()) == false)
return null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java
index be948c1..a0f19e7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java
@@ -34,7 +34,10 @@ public class TableGenConfig {
dataGen = "";
}
- if (dataGen == null)
+ if (dataGen == null || "no".equals(dataGen) || "false".equals(dataGen) || "skip".equals(dataGen))
+ return;
+
+ if (table.isView())
return;
needGen = true;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json
deleted file mode 100644
index 6215174..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_inner_join_empty.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "uuid" : "ffff3e80-41be-49a5-90ca-9fb7294db186",
- "last_modified" : 0,
- "name" : "test_kylin_cube_with_view_inner_join_empty",
- "owner" : null,
- "descriptor" : "test_kylin_cube_with_view_inner_join_desc",
- "segments" : [ ],
- "status" : "DISABLED",
- "create_time" : null
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json
deleted file mode 100644
index 62b7432..0000000
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_view_left_join_empty.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "uuid" : "eeee3e80-41be-49a5-90ca-9fb7294db186",
- "last_modified" : 0,
- "name" : "test_kylin_cube_with_view_left_join_empty",
- "owner" : null,
- "descriptor" : "test_kylin_cube_with_view_left_join_desc",
- "segments" : [ ],
- "status" : "DISABLED",
- "create_time" : null
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
deleted file mode 100644
index d4c64b5..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_inner_join_desc.json
+++ /dev/null
@@ -1,169 +0,0 @@
-{
- "uuid" : "9876b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name" : "test_kylin_cube_with_view_inner_join_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "EDW.V_TEST_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
- "column" : "{FK}",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "column" : "{FK}",
- "derived" : [ "SELLER_TYPE_DESC" ]
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
- },
- "returntype" : "bigint"
- },
- "dependent_measure_ref" : null
- }],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "cal_dt",
- "encoding" : "dict"
- }, {
- "column" : "leaf_categ_id",
- "encoding" : "dict"
- }, {
- "column" : "meta_categ_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl2_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl3_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_format_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_site_id",
- "encoding" : "dict"
- }, {
- "column" : "slr_segment_cd",
- "encoding" : "dict"
- } ]
- },
- "signature" : null,
- "last_modified" : 1448959801311,
- "model_name" : "test_kylin_inner_join_view_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
- } ]
- }]
- },
- "aggregation_groups" : [ {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
- }
- }, {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ ]
- }
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
- "partition_date_start": 0
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
deleted file mode 100644
index 0388c0e..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_view_left_join_desc.json
+++ /dev/null
@@ -1,169 +0,0 @@
-{
- "uuid" : "6789b7a8-3929-4dff-b59d-2100aadc8dbf",
- "name" : "test_kylin_cube_with_view_left_join_desc",
- "description" : null,
- "dimensions" : [ {
- "name" : "CAL_DT",
- "table" : "EDW.V_TEST_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
- "column" : "{FK}",
- "derived" : [ "SITE_NAME", "CRE_USER" ]
- }, {
- "name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
- "column" : "{FK}",
- "derived" : [ "SELLER_TYPE_DESC" ]
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
- },
- "dependent_measure_ref" : null
- }, {
- "name" : "ITEM_COUNT_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "ITEM_COUNT",
- "next_parameter" : null
- },
- "returntype" : "bigint"
- },
- "dependent_measure_ref" : null
- }],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "cal_dt",
- "encoding" : "dict"
- }, {
- "column" : "leaf_categ_id",
- "encoding" : "dict"
- }, {
- "column" : "meta_categ_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl2_name",
- "encoding" : "dict"
- }, {
- "column" : "categ_lvl3_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_format_name",
- "encoding" : "dict"
- }, {
- "column" : "lstg_site_id",
- "encoding" : "dict"
- }, {
- "column" : "slr_segment_cd",
- "encoding" : "dict"
- } ]
- },
- "signature" : null,
- "last_modified" : 1448959801311,
- "model_name" : "test_kylin_left_join_view_model_desc",
- "null_string" : null,
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "f1",
- "columns" : [ {
- "qualifier" : "m",
- "measure_refs" : [ "gmv_sum", "gmv_min", "gmv_max", "trans_cnt", "item_count_sum" ]
- } ]
- }]
- },
- "aggregation_groups" : [ {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "lstg_format_name", "lstg_site_id", "meta_categ_name"],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ [ "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ] ]
- }
- }, {
- "includes" : [ "cal_dt", "categ_lvl2_name", "categ_lvl3_name", "leaf_categ_id", "meta_categ_name" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" ] ],
- "mandatory_dims" : [ "cal_dt" ],
- "joint_dims" : [ ]
- }
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
- "partition_date_start": 0
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv b/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv
deleted file mode 100644
index 73abe26..0000000
--- a/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM.csv
+++ /dev/null
@@ -1,8 +0,0 @@
-16,Consumer-Other,16,102,Consumer,2012-08-09,USER_P,,USER_A
-5,Occasional,5,102,Consumer,2010-05-11,USER_H,2011-03-01 11:30:52,USER_A
-15,Professional-Other,15,101,Professional,2012-08-09,USER_P,,USER_A
--99,Not Applicable,-99,-99,Not Applicable,2010-05-11,USER_H,,USER_A
-14,Regular,14,102,Consumer,2011-03-01,USER_H,,USER_A
-12,Merchant,12,101,Professional,2011-03-01,USER_H,,USER_A
-11,Large Merchant,11,101,Professional,2011-03-01,USER_H,,USER_A
-13,Entrepreneur,13,101,Professional,2011-03-01,USER_H,,USER_A
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv b/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv
new file mode 100644
index 0000000..d653ae2
--- /dev/null
+++ b/examples/test_case_data/localmeta/data/EDW.TEST_SELLER_TYPE_DIM_TABLE.csv
@@ -0,0 +1,8 @@
+16,Consumer-Other,16,102,Consumer,2012-08-09,USER_P,,USER_A
+5,Occasional,5,102,Consumer,2010-05-11,USER_H,2011-03-01 11:30:52,USER_A
+15,Professional-Other,15,101,Professional,2012-08-09,USER_P,,USER_A
+-99,Not Applicable,-99,-99,Not Applicable,2010-05-11,USER_H,,USER_A
+14,Regular,14,102,Consumer,2011-03-01,USER_H,,USER_A
+12,Merchant,12,101,Professional,2011-03-01,USER_H,,USER_A
+11,Large Merchant,11,101,Professional,2011-03-01,USER_H,,USER_A
+13,Entrepreneur,13,101,Professional,2011-03-01,USER_H,,USER_A
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
deleted file mode 100644
index d7b7227..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_view_model_desc.json
+++ /dev/null
@@ -1,119 +0,0 @@
-{
- "uuid": "8b184ee2-1ccb-4b07-a38e-4c298563e0f7",
- "name": "test_kylin_inner_join_view_model_desc",
- "lookups": [
- {
- "table": "EDW.V_TEST_CAL_DT",
- "join": {
- "type": "inner",
- "primary_key": [
- "CAL_DT"
- ],
- "foreign_key": [
- "CAL_DT"
- ]
- }
- },
- {
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join": {
- "type": "inner",
- "primary_key": [
- "LEAF_CATEG_ID",
- "SITE_ID"
- ],
- "foreign_key": [
- "LEAF_CATEG_ID",
- "LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "EDW.TEST_SITES",
- "join": {
- "type": "inner",
- "primary_key": [
- "SITE_ID"
- ],
- "foreign_key": [
- "LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "join": {
- "type": "inner",
- "primary_key": [
- "SELLER_TYPE_CD"
- ],
- "foreign_key": [
- "SLR_SEGMENT_CD"
- ]
- }
- }
- ],
- "dimensions": [
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "TRANS_ID",
- "CAL_DT",
- "lstg_format_name",
- "LSTG_SITE_ID",
- "SLR_SEGMENT_CD",
- "SELLER_ID"
- ]
- },
- {
- "table": "default.test_category_groupings",
- "columns": [
- "leaf_categ_id",
- "site_id",
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER",
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name"
- ]
- },
- {
- "table": "edw.test_sites",
- "columns": [
- "site_id",
- "site_name",
- "cre_user"
- ]
- },
- {
- "table": "edw.test_seller_type_dim",
- "columns": [
- "seller_type_cd",
- "seller_type_desc"
- ]
- },
- {
- "table": "edw.v_test_cal_dt",
- "columns": [
- "cal_dt",
- "week_beg_dt"
- ]
- }
- ],
- "metrics": [
- "PRICE",
- "ITEM_COUNT",
- "SELLER_ID",
- "USER_ID"
- ],
- "last_modified": 1422435345352,
- "fact_table": "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition": null,
- "partition_desc": {
- "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start": 0,
- "partition_type": "APPEND"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
deleted file mode 100644
index 2ff55d3..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_view_model_desc.json
+++ /dev/null
@@ -1,119 +0,0 @@
-{
- "uuid": "9c184ee2-1ccb-4b07-a38e-4c298563e0f7",
- "name": "test_kylin_left_join_view_model_desc",
- "lookups": [
- {
- "table": "EDW.V_TEST_CAL_DT",
- "join": {
- "type": "left",
- "primary_key": [
- "CAL_DT"
- ],
- "foreign_key": [
- "CAL_DT"
- ]
- }
- },
- {
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
- "join": {
- "type": "left",
- "primary_key": [
- "LEAF_CATEG_ID",
- "SITE_ID"
- ],
- "foreign_key": [
- "LEAF_CATEG_ID",
- "LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "EDW.TEST_SITES",
- "join": {
- "type": "left",
- "primary_key": [
- "SITE_ID"
- ],
- "foreign_key": [
- "LSTG_SITE_ID"
- ]
- }
- },
- {
- "table": "EDW.TEST_SELLER_TYPE_DIM",
- "join": {
- "type": "left",
- "primary_key": [
- "SELLER_TYPE_CD"
- ],
- "foreign_key": [
- "SLR_SEGMENT_CD"
- ]
- }
- }
- ],
- "dimensions": [
- {
- "table": "default.test_kylin_fact",
- "columns": [
- "TRANS_ID",
- "CAL_DT",
- "lstg_format_name",
- "LSTG_SITE_ID",
- "SLR_SEGMENT_CD",
- "SELLER_ID"
- ]
- },
- {
- "table": "default.test_category_groupings",
- "columns": [
- "leaf_categ_id",
- "site_id",
- "USER_DEFINED_FIELD1",
- "USER_DEFINED_FIELD3",
- "UPD_DATE",
- "UPD_USER",
- "meta_categ_name",
- "categ_lvl2_name",
- "categ_lvl3_name"
- ]
- },
- {
- "table": "edw.test_sites",
- "columns": [
- "site_id",
- "site_name",
- "cre_user"
- ]
- },
- {
- "table": "edw.test_seller_type_dim",
- "columns": [
- "seller_type_cd",
- "seller_type_desc"
- ]
- },
- {
- "table": "edw.v_test_cal_dt",
- "columns": [
- "cal_dt",
- "week_beg_dt"
- ]
- }
- ],
- "metrics": [
- "PRICE",
- "ITEM_COUNT",
- "SELLER_ID",
- "USER_ID"
- ],
- "last_modified": 1422435345352,
- "fact_table": "DEFAULT.TEST_KYLIN_FACT",
- "filter_condition": null,
- "partition_desc": {
- "partition_date_column": "DEFAULT.TEST_KYLIN_FACT.cal_dt",
- "partition_date_start": 0,
- "partition_type": "APPEND"
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index d24a22a..5eefce7 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -32,16 +32,6 @@
"type": "HYBRID",
"realization": "test_kylin_hybrid_inner_join"
},
- {
- "name": "test_kylin_cube_with_view_left_join_empty",
- "type": "CUBE",
- "realization": "test_kylin_cube_with_view_left_join_empty"
- },
- {
- "name": "test_kylin_cube_with_view_inner_join_empty",
- "type": "CUBE",
- "realization": "test_kylin_cube_with_view_inner_join_empty"
- },
{
"name": "ssb",
"type": "CUBE",
@@ -51,18 +41,16 @@
"tables": [
"DEFAULT.TEST_KYLIN_FACT",
"EDW.TEST_CAL_DT",
- "EDW.V_TEST_CAL_DT",
"DEFAULT.TEST_CATEGORY_GROUPINGS",
"EDW.TEST_SITES",
"EDW.TEST_SELLER_TYPE_DIM",
+ "EDW.TEST_SELLER_TYPE_DIM_TABLE",
"DEFAULT.STREAMING_TABLE"
],
"models": [
"ssb",
"test_kylin_inner_join_model_desc",
- "test_kylin_inner_join_view_model_desc",
"test_kylin_left_join_model_desc",
- "test_kylin_left_join_view_model_desc",
"test_streaming_table_model_desc"
],
"override_kylin_properties" :{
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
index 069f41b..d7ab580 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json
@@ -1,152 +1,152 @@
-{
-
- "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
- "name" : "TEST_CATEGORY_GROUPINGS",
- "columns" : [ {
- "id" : "1",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "LEAF_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "3",
- "name" : "SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "4",
- "name" : "CATEG_BUSN_MGR",
- "datatype" : "string"
- }, {
- "id" : "5",
- "name" : "CATEG_BUSN_UNIT",
- "datatype" : "string"
- }, {
- "id" : "6",
- "name" : "REGN_CATEG",
- "datatype" : "string"
- }, {
- "id" : "7",
- "name" : "USER_DEFINED_FIELD1",
- "datatype" : "string"
- }, {
- "id" : "8",
- "name" : "USER_DEFINED_FIELD3",
- "datatype" : "string"
- }, {
- "id" : "9",
- "name" : "GROUPINGS_CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "10",
- "name" : "UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "11",
- "name" : "GROUPINGS_CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "12",
- "name" : "UPD_USER",
- "datatype" : "string"
- }, {
- "id" : "13",
- "name" : "META_CATEG_ID",
- "datatype" : "decimal"
- }, {
- "id" : "14",
- "name" : "META_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "15",
- "name" : "CATEG_LVL2_ID",
- "datatype" : "decimal"
- }, {
- "id" : "16",
- "name" : "CATEG_LVL3_ID",
- "datatype" : "decimal"
- }, {
- "id" : "17",
- "name" : "CATEG_LVL4_ID",
- "datatype" : "decimal"
- }, {
- "id" : "18",
- "name" : "CATEG_LVL5_ID",
- "datatype" : "decimal"
- }, {
- "id" : "19",
- "name" : "CATEG_LVL6_ID",
- "datatype" : "decimal"
- }, {
- "id" : "20",
- "name" : "CATEG_LVL7_ID",
- "datatype" : "decimal"
- }, {
- "id" : "21",
- "name" : "CATEG_LVL2_NAME",
- "datatype" : "string"
- }, {
- "id" : "22",
- "name" : "CATEG_LVL3_NAME",
- "datatype" : "string"
- }, {
- "id" : "23",
- "name" : "CATEG_LVL4_NAME",
- "datatype" : "string"
- }, {
- "id" : "24",
- "name" : "CATEG_LVL5_NAME",
- "datatype" : "string"
- }, {
- "id" : "25",
- "name" : "CATEG_LVL6_NAME",
- "datatype" : "string"
- }, {
- "id" : "26",
- "name" : "CATEG_LVL7_NAME",
- "datatype" : "string"
- }, {
- "id" : "27",
- "name" : "CATEG_FLAGS",
- "datatype" : "decimal"
- }, {
- "id" : "28",
- "name" : "ADULT_CATEG_YN",
- "datatype" : "string"
- }, {
- "id" : "29",
- "name" : "DOMAIN_ID",
- "datatype" : "decimal"
- }, {
- "id" : "30",
- "name" : "USER_DEFINED_FIELD5",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "VCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "32",
- "name" : "GCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "33",
- "name" : "MOVE_TO",
- "datatype" : "decimal"
- }, {
- "id" : "34",
- "name" : "SAP_CATEGORY_ID",
- "datatype" : "decimal"
- }, {
- "id" : "35",
- "name" : "SRC_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "36",
- "name" : "BSNS_VRTCL_NAME",
- "datatype" : "string"
- } ],
- "database" : "DEFAULT",
- "last_modified" : 0
+{
+
+ "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
+ "name" : "TEST_CATEGORY_GROUPINGS",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "LEAF_CATEG_ID",
+ "datatype" : "bigint"
+ }, {
+ "id" : "2",
+ "name" : "LEAF_CATEG_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "3",
+ "name" : "SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "4",
+ "name" : "CATEG_BUSN_MGR",
+ "datatype" : "string"
+ }, {
+ "id" : "5",
+ "name" : "CATEG_BUSN_UNIT",
+ "datatype" : "string"
+ }, {
+ "id" : "6",
+ "name" : "REGN_CATEG",
+ "datatype" : "string"
+ }, {
+ "id" : "7",
+ "name" : "USER_DEFINED_FIELD1",
+ "datatype" : "string"
+ }, {
+ "id" : "8",
+ "name" : "USER_DEFINED_FIELD3",
+ "datatype" : "string"
+ }, {
+ "id" : "9",
+ "name" : "GROUPINGS_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "10",
+ "name" : "UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "11",
+ "name" : "GROUPINGS_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "12",
+ "name" : "UPD_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "13",
+ "name" : "META_CATEG_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "14",
+ "name" : "META_CATEG_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "15",
+ "name" : "CATEG_LVL2_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "16",
+ "name" : "CATEG_LVL3_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "17",
+ "name" : "CATEG_LVL4_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "18",
+ "name" : "CATEG_LVL5_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "19",
+ "name" : "CATEG_LVL6_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "20",
+ "name" : "CATEG_LVL7_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "21",
+ "name" : "CATEG_LVL2_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "22",
+ "name" : "CATEG_LVL3_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "23",
+ "name" : "CATEG_LVL4_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "24",
+ "name" : "CATEG_LVL5_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "25",
+ "name" : "CATEG_LVL6_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "26",
+ "name" : "CATEG_LVL7_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "27",
+ "name" : "CATEG_FLAGS",
+ "datatype" : "decimal"
+ }, {
+ "id" : "28",
+ "name" : "ADULT_CATEG_YN",
+ "datatype" : "string"
+ }, {
+ "id" : "29",
+ "name" : "DOMAIN_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "30",
+ "name" : "USER_DEFINED_FIELD5",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "VCS_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "32",
+ "name" : "GCS_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "33",
+ "name" : "MOVE_TO",
+ "datatype" : "decimal"
+ }, {
+ "id" : "34",
+ "name" : "SAP_CATEGORY_ID",
+ "datatype" : "decimal"
+ }, {
+ "id" : "35",
+ "name" : "SRC_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "36",
+ "name" : "BSNS_VRTCL_NAME",
+ "datatype" : "string"
+ } ],
+ "database" : "DEFAULT",
+ "last_modified" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_COUNTRY.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_COUNTRY.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_COUNTRY.json
index 95b34c2..acf5470 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_COUNTRY.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_COUNTRY.json
@@ -1,24 +1,24 @@
-{
- "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882",
-
- "name" : "TEST_KYLIN_COUNTRY",
- "columns" : [ {
- "id" : "1",
- "name" : "COUNTRY",
- "datatype" : "string"
- }, {
- "id" : "2",
- "name" : "LATITUDE",
- "datatype" : "double"
- }, {
- "id" : "3",
- "name" : "LONGITUDE",
- "datatype" : "double"
- }, {
- "id" : "4",
- "name" : "NAME",
- "datatype" : "string"
- } ],
- "database" : "DEFAULT",
- "last_modified" : 0
+{
+ "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882",
+
+ "name" : "TEST_KYLIN_COUNTRY",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "COUNTRY",
+ "datatype" : "string"
+ }, {
+ "id" : "2",
+ "name" : "LATITUDE",
+ "datatype" : "double"
+ }, {
+ "id" : "3",
+ "name" : "LONGITUDE",
+ "datatype" : "double"
+ }, {
+ "id" : "4",
+ "name" : "NAME",
+ "datatype" : "string"
+ } ],
+ "database" : "DEFAULT",
+ "last_modified" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
index ea9c701..5fb3edf 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json
@@ -1,66 +1,67 @@
-{
- "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
- "name" : "TEST_KYLIN_FACT",
- "data_gen" : "1",
- "columns" : [ {
- "id" : "1",
- "name" : "TRANS_ID",
- "datatype" : "bigint",
- "data_gen" : "ID"
- }, {
- "id" : "2",
- "name" : "CAL_DT",
- "datatype" : "date",
- "data_gen" : "FK,order"
- }, {
- "id" : "3",
- "name" : "LSTG_FORMAT_NAME",
- "datatype" : "string",
- "data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others"
- }, {
- "id" : "4",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint",
- "data_gen" : "FK,null,nullstr=0"
- }, {
- "id" : "5",
- "name" : "LSTG_SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "6",
- "name" : "SLR_SEGMENT_CD",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "PRICE",
- "datatype" : "decimal",
- "data_gen" : "RAND|.##|-100|1000"
- }, {
- "id" : "8",
- "name" : "ITEM_COUNT",
- "datatype" : "int",
- "data_gen" : "RAND"
- }, {
- "id" : "9",
- "name" : "SELLER_ID",
- "datatype" : "bigint",
- "data_gen" : "RAND||10000000|10001000"
- }, {
- "id" : "10",
- "name" : "USER_ID",
- "datatype" : "varchar(32)",
- "data_gen" : "RAND,order"
- }, {
- "id" : "11",
- "name" : "BUYER_COUNTRY",
- "datatype" : "string",
- "data_gen" : "CN|DE|FR|JP|UK|US"
- }, {
- "id" : "12",
- "name" : "SELLER_COUNTRY",
- "datatype" : "string",
- "data_gen" : "CN|DE|FR|JP|UK|US"
- } ],
- "database" : "DEFAULT",
- "last_modified" : 0
-}
+{
+ "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
+ "name" : "TEST_KYLIN_FACT",
+ "data_gen" : "1",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "TRANS_ID",
+ "datatype" : "bigint",
+ "data_gen" : "ID"
+ }, {
+ "id" : "2",
+ "name" : "CAL_DT",
+ "datatype" : "date",
+ "data_gen" : "FK,order"
+ }, {
+ "id" : "3",
+ "name" : "LSTG_FORMAT_NAME",
+ "datatype" : "string",
+ "data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others"
+ }, {
+ "id" : "4",
+ "name" : "LEAF_CATEG_ID",
+ "datatype" : "bigint",
+ "data_gen" : "FK,null,nullstr=0"
+ }, {
+ "id" : "5",
+ "name" : "LSTG_SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "6",
+ "name" : "SLR_SEGMENT_CD",
+ "datatype" : "smallint",
+ "data_gen" : "FK,pk=EDW.TEST_SELLER_TYPE_DIM_TABLE.SELLER_TYPE_CD"
+ }, {
+ "id" : "7",
+ "name" : "PRICE",
+ "datatype" : "decimal",
+ "data_gen" : "RAND|.##|-100|1000"
+ }, {
+ "id" : "8",
+ "name" : "ITEM_COUNT",
+ "datatype" : "int",
+ "data_gen" : "RAND"
+ }, {
+ "id" : "9",
+ "name" : "SELLER_ID",
+ "datatype" : "bigint",
+ "data_gen" : "RAND||10000000|10001000"
+ }, {
+ "id" : "10",
+ "name" : "USER_ID",
+ "datatype" : "varchar(32)",
+ "data_gen" : "RAND,order"
+ }, {
+ "id" : "11",
+ "name" : "BUYER_COUNTRY",
+ "datatype" : "string",
+ "data_gen" : "CN|DE|FR|JP|UK|US"
+ }, {
+ "id" : "12",
+ "name" : "SELLER_COUNTRY",
+ "datatype" : "string",
+ "data_gen" : "CN|DE|FR|JP|UK|US"
+ } ],
+ "database" : "DEFAULT",
+ "last_modified" : 0
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
index dde24cf..02299d9 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json
@@ -1,408 +1,408 @@
-{
-
- "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- "name" : "TEST_CAL_DT",
- "columns" : [ {
- "id" : "1",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "2",
- "name" : "YEAR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "QTR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "4",
- "name" : "MONTH_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "5",
- "name" : "WEEK_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "6",
- "name" : "AGE_FOR_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "AGE_FOR_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "8",
- "name" : "AGE_FOR_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "9",
- "name" : "AGE_FOR_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "10",
- "name" : "AGE_FOR_DT_ID",
- "datatype" : "smallint"
- }, {
- "id" : "11",
- "name" : "AGE_FOR_RTL_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "12",
- "name" : "AGE_FOR_RTL_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "13",
- "name" : "AGE_FOR_RTL_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "14",
- "name" : "AGE_FOR_RTL_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "15",
- "name" : "AGE_FOR_CS_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "16",
- "name" : "DAY_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "17",
- "name" : "DAY_OF_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "18",
- "name" : "DAY_OF_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "19",
- "name" : "DAY_OF_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "20",
- "name" : "DAY_OF_WEEK_ID",
- "datatype" : "int"
- }, {
- "id" : "21",
- "name" : "WEEK_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "22",
- "name" : "WEEK_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "23",
- "name" : "MONTH_OF_QTR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "24",
- "name" : "MONTH_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "25",
- "name" : "MONTH_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "26",
- "name" : "QTR_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "27",
- "name" : "QTR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "28",
- "name" : "YEAR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "29",
- "name" : "YEAR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "30",
- "name" : "QTR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "MONTH_END_DT",
- "datatype" : "string"
- }, {
- "id" : "32",
- "name" : "WEEK_END_DT",
- "datatype" : "string"
- }, {
- "id" : "33",
- "name" : "CAL_DT_NAME",
- "datatype" : "string"
- }, {
- "id" : "34",
- "name" : "CAL_DT_DESC",
- "datatype" : "string"
- }, {
- "id" : "35",
- "name" : "CAL_DT_SHORT_NAME",
- "datatype" : "string"
- }, {
- "id" : "36",
- "name" : "YTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "37",
- "name" : "QTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "38",
- "name" : "MTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "39",
- "name" : "WTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "40",
- "name" : "SEASON_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "41",
- "name" : "DAY_IN_YEAR_COUNT",
- "datatype" : "smallint"
- }, {
- "id" : "42",
- "name" : "DAY_IN_QTR_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "43",
- "name" : "DAY_IN_MONTH_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "44",
- "name" : "DAY_IN_WEEK_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "45",
- "name" : "RTL_YEAR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "46",
- "name" : "RTL_QTR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "47",
- "name" : "RTL_MONTH_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "48",
- "name" : "RTL_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "49",
- "name" : "CS_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "50",
- "name" : "CAL_DATE",
- "datatype" : "string"
- }, {
- "id" : "51",
- "name" : "DAY_OF_WEEK",
- "datatype" : "string"
- }, {
- "id" : "52",
- "name" : "MONTH_ID",
- "datatype" : "string"
- }, {
- "id" : "53",
- "name" : "PRD_DESC",
- "datatype" : "string"
- }, {
- "id" : "54",
- "name" : "PRD_FLAG",
- "datatype" : "string"
- }, {
- "id" : "55",
- "name" : "PRD_ID",
- "datatype" : "string"
- }, {
- "id" : "56",
- "name" : "PRD_IND",
- "datatype" : "string"
- }, {
- "id" : "57",
- "name" : "QTR_DESC",
- "datatype" : "string"
- }, {
- "id" : "58",
- "name" : "QTR_ID",
- "datatype" : "string"
- }, {
- "id" : "59",
- "name" : "QTR_IND",
- "datatype" : "string"
- }, {
- "id" : "60",
- "name" : "RETAIL_WEEK",
- "datatype" : "string"
- }, {
- "id" : "61",
- "name" : "RETAIL_YEAR",
- "datatype" : "string"
- }, {
- "id" : "62",
- "name" : "RETAIL_START_DATE",
- "datatype" : "string"
- }, {
- "id" : "63",
- "name" : "RETAIL_WK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "64",
- "name" : "WEEK_IND",
- "datatype" : "string"
- }, {
- "id" : "65",
- "name" : "WEEK_NUM_DESC",
- "datatype" : "string"
- }, {
- "id" : "66",
- "name" : "WEEK_BEG_DATE",
- "datatype" : "string"
- }, {
- "id" : "67",
- "name" : "WEEK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "68",
- "name" : "WEEK_IN_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "69",
- "name" : "WEEK_ID",
- "datatype" : "string"
- }, {
- "id" : "70",
- "name" : "WEEK_BEG_END_DESC_MDY",
- "datatype" : "string"
- }, {
- "id" : "71",
- "name" : "WEEK_BEG_END_DESC_MD",
- "datatype" : "string"
- }, {
- "id" : "72",
- "name" : "YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "73",
- "name" : "YEAR_IND",
- "datatype" : "string"
- }, {
- "id" : "74",
- "name" : "CAL_DT_MNS_1YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "75",
- "name" : "CAL_DT_MNS_2YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "76",
- "name" : "CAL_DT_MNS_1QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "77",
- "name" : "CAL_DT_MNS_2QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "78",
- "name" : "CAL_DT_MNS_1MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "79",
- "name" : "CAL_DT_MNS_2MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "80",
- "name" : "CAL_DT_MNS_1WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "81",
- "name" : "CAL_DT_MNS_2WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "82",
- "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "83",
- "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "84",
- "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "85",
- "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "86",
- "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "87",
- "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "88",
- "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "89",
- "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "90",
- "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "91",
- "name" : "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "92",
- "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "93",
- "name" : "SEASON_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "94",
- "name" : "YTM_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "95",
- "name" : "YTQ_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "96",
- "name" : "YTW_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "97",
- "name" : "CAL_DT_CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "98",
- "name" : "CAL_DT_CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "99",
- "name" : "CAL_DT_UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "100",
- "name" : "CAL_DT_UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "edw",
- "last_modified" : 0
+{
+
+ "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
+ "name" : "TEST_CAL_DT",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "CAL_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "2",
+ "name" : "YEAR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "3",
+ "name" : "QTR_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "4",
+ "name" : "MONTH_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "5",
+ "name" : "WEEK_BEG_DT",
+ "datatype" : "date"
+ }, {
+ "id" : "6",
+ "name" : "AGE_FOR_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "7",
+ "name" : "AGE_FOR_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "8",
+ "name" : "AGE_FOR_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "9",
+ "name" : "AGE_FOR_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "10",
+ "name" : "AGE_FOR_DT_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "11",
+ "name" : "AGE_FOR_RTL_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "12",
+ "name" : "AGE_FOR_RTL_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "13",
+ "name" : "AGE_FOR_RTL_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "14",
+ "name" : "AGE_FOR_RTL_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "15",
+ "name" : "AGE_FOR_CS_WEEK_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "16",
+ "name" : "DAY_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "17",
+ "name" : "DAY_OF_YEAR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "18",
+ "name" : "DAY_OF_QTR_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "19",
+ "name" : "DAY_OF_MONTH_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "20",
+ "name" : "DAY_OF_WEEK_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "21",
+ "name" : "WEEK_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "22",
+ "name" : "WEEK_OF_CAL_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "23",
+ "name" : "MONTH_OF_QTR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "24",
+ "name" : "MONTH_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "25",
+ "name" : "MONTH_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "26",
+ "name" : "QTR_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "27",
+ "name" : "QTR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "28",
+ "name" : "YEAR_OF_CAL_ID",
+ "datatype" : "smallint"
+ }, {
+ "id" : "29",
+ "name" : "YEAR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "30",
+ "name" : "QTR_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "31",
+ "name" : "MONTH_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "32",
+ "name" : "WEEK_END_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "33",
+ "name" : "CAL_DT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "34",
+ "name" : "CAL_DT_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "35",
+ "name" : "CAL_DT_SHORT_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "36",
+ "name" : "YTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "37",
+ "name" : "QTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "38",
+ "name" : "MTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "39",
+ "name" : "WTD_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "40",
+ "name" : "SEASON_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "41",
+ "name" : "DAY_IN_YEAR_COUNT",
+ "datatype" : "smallint"
+ }, {
+ "id" : "42",
+ "name" : "DAY_IN_QTR_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "43",
+ "name" : "DAY_IN_MONTH_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "44",
+ "name" : "DAY_IN_WEEK_COUNT",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "45",
+ "name" : "RTL_YEAR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "46",
+ "name" : "RTL_QTR_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "47",
+ "name" : "RTL_MONTH_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "48",
+ "name" : "RTL_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "49",
+ "name" : "CS_WEEK_BEG_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "50",
+ "name" : "CAL_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "51",
+ "name" : "DAY_OF_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "52",
+ "name" : "MONTH_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "53",
+ "name" : "PRD_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "54",
+ "name" : "PRD_FLAG",
+ "datatype" : "string"
+ }, {
+ "id" : "55",
+ "name" : "PRD_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "56",
+ "name" : "PRD_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "57",
+ "name" : "QTR_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "58",
+ "name" : "QTR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "59",
+ "name" : "QTR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "60",
+ "name" : "RETAIL_WEEK",
+ "datatype" : "string"
+ }, {
+ "id" : "61",
+ "name" : "RETAIL_YEAR",
+ "datatype" : "string"
+ }, {
+ "id" : "62",
+ "name" : "RETAIL_START_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "63",
+ "name" : "RETAIL_WK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "64",
+ "name" : "WEEK_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "65",
+ "name" : "WEEK_NUM_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "66",
+ "name" : "WEEK_BEG_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "67",
+ "name" : "WEEK_END_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "68",
+ "name" : "WEEK_IN_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "69",
+ "name" : "WEEK_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "70",
+ "name" : "WEEK_BEG_END_DESC_MDY",
+ "datatype" : "string"
+ }, {
+ "id" : "71",
+ "name" : "WEEK_BEG_END_DESC_MD",
+ "datatype" : "string"
+ }, {
+ "id" : "72",
+ "name" : "YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "73",
+ "name" : "YEAR_IND",
+ "datatype" : "string"
+ }, {
+ "id" : "74",
+ "name" : "CAL_DT_MNS_1YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "75",
+ "name" : "CAL_DT_MNS_2YEAR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "76",
+ "name" : "CAL_DT_MNS_1QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "77",
+ "name" : "CAL_DT_MNS_2QTR_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "78",
+ "name" : "CAL_DT_MNS_1MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "79",
+ "name" : "CAL_DT_MNS_2MONTH_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "80",
+ "name" : "CAL_DT_MNS_1WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "81",
+ "name" : "CAL_DT_MNS_2WEEK_DT",
+ "datatype" : "string"
+ }, {
+ "id" : "82",
+ "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "83",
+ "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "84",
+ "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "85",
+ "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "86",
+ "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "87",
+ "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "88",
+ "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "89",
+ "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "90",
+ "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
+ "datatype" : "string"
+ }, {
+ "id" : "91",
+ "name" : "RTL_QTR_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "92",
+ "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "93",
+ "name" : "SEASON_OF_YEAR_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "94",
+ "name" : "YTM_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "95",
+ "name" : "YTQ_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "96",
+ "name" : "YTW_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "97",
+ "name" : "CAL_DT_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "98",
+ "name" : "CAL_DT_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "99",
+ "name" : "CAL_DT_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "100",
+ "name" : "CAL_DT_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "edw",
+ "last_modified" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
index eec48ea..2bfbb70 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json
@@ -1,44 +1,44 @@
-{
-
- "uuid" : "9ecc90c4-55df-436f-8602-2fbd4bca72e1",
- "name" : "TEST_SELLER_TYPE_DIM",
- "columns" : [ {
- "id" : "1",
- "name" : "SELLER_TYPE_CD",
- "datatype" : "smallint"
- }, {
- "id" : "2",
- "name" : "SELLER_TYPE_DESC",
- "datatype" : "string"
- }, {
- "id" : "3",
- "name" : "GLBL_RPRT_SLR_SGMNT_CD",
- "datatype" : "tinyint"
- }, {
- "id" : "4",
- "name" : "SELLER_GROUP_CD",
- "datatype" : "tinyint"
- }, {
- "id" : "5",
- "name" : "SELLER_GROUP_DESC",
- "datatype" : "string"
- }, {
- "id" : "6",
- "name" : "DIM_CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "7",
- "name" : "DIM_CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "8",
- "name" : "DIM_UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "9",
- "name" : "DIM_UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "edw",
- "last_modified" : 0
+{
+ "uuid" : "9ecc90c4-55df-436f-8602-2fbd4bca72e1",
+ "name" : "TEST_SELLER_TYPE_DIM",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "SELLER_TYPE_CD",
+ "datatype" : "smallint"
+ }, {
+ "id" : "2",
+ "name" : "SELLER_TYPE_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "3",
+ "name" : "GLBL_RPRT_SLR_SGMNT_CD",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "4",
+ "name" : "SELLER_GROUP_CD",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "5",
+ "name" : "SELLER_GROUP_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "6",
+ "name" : "DIM_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "7",
+ "name" : "DIM_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "8",
+ "name" : "DIM_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "9",
+ "name" : "DIM_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "edw",
+ "table_type" : "VIRTUAL_VIEW",
+ "last_modified" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json
new file mode 100644
index 0000000..3c5749b
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json
@@ -0,0 +1,43 @@
+{
+ "uuid" : "addc20c4-55df-436f-8602-2fbd4bca2910",
+ "name" : "TEST_SELLER_TYPE_DIM_TABLE",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "SELLER_TYPE_CD",
+ "datatype" : "smallint"
+ }, {
+ "id" : "2",
+ "name" : "SELLER_TYPE_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "3",
+ "name" : "GLBL_RPRT_SLR_SGMNT_CD",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "4",
+ "name" : "SELLER_GROUP_CD",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "5",
+ "name" : "SELLER_GROUP_DESC",
+ "datatype" : "string"
+ }, {
+ "id" : "6",
+ "name" : "DIM_CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "7",
+ "name" : "DIM_CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "8",
+ "name" : "DIM_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "9",
+ "name" : "DIM_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "edw",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
index 06a0b2e..cc801af 100644
--- a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
+++ b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json
@@ -1,48 +1,48 @@
-{
-
- "uuid" : "338a3325-a947-46d1-9ece-e079b3b8d4a6",
- "name" : "TEST_SITES",
- "columns" : [ {
- "id" : "1",
- "name" : "SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "2",
- "name" : "SITE_NAME",
- "datatype" : "string"
- }, {
- "id" : "3",
- "name" : "SITE_DOMAIN_CODE",
- "datatype" : "string"
- }, {
- "id" : "4",
- "name" : "DFAULT_LSTG_CURNCY",
- "datatype" : "int"
- }, {
- "id" : "5",
- "name" : "EOA_EMAIL_CSTMZBL_SITE_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "6",
- "name" : "SITE_CNTRY_ID",
- "datatype" : "int"
- }, {
- "id" : "7",
- "name" : "CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "8",
- "name" : "SITES_UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "9",
- "name" : "CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "10",
- "name" : "SITES_UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "edw",
- "last_modified" : 0
+{
+
+ "uuid" : "338a3325-a947-46d1-9ece-e079b3b8d4a6",
+ "name" : "TEST_SITES",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "SITE_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "2",
+ "name" : "SITE_NAME",
+ "datatype" : "string"
+ }, {
+ "id" : "3",
+ "name" : "SITE_DOMAIN_CODE",
+ "datatype" : "string"
+ }, {
+ "id" : "4",
+ "name" : "DFAULT_LSTG_CURNCY",
+ "datatype" : "int"
+ }, {
+ "id" : "5",
+ "name" : "EOA_EMAIL_CSTMZBL_SITE_YN_ID",
+ "datatype" : "tinyint"
+ }, {
+ "id" : "6",
+ "name" : "SITE_CNTRY_ID",
+ "datatype" : "int"
+ }, {
+ "id" : "7",
+ "name" : "CRE_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "8",
+ "name" : "SITES_UPD_DATE",
+ "datatype" : "string"
+ }, {
+ "id" : "9",
+ "name" : "CRE_USER",
+ "datatype" : "string"
+ }, {
+ "id" : "10",
+ "name" : "SITES_UPD_USER",
+ "datatype" : "string"
+ } ],
+ "database" : "edw",
+ "last_modified" : 0
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f563df4/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
deleted file mode 100644
index 9298c7f..0000000
--- a/examples/test_case_data/localmeta/table/EDW.V_TEST_CAL_DT.json
+++ /dev/null
@@ -1,409 +0,0 @@
-{
-
- "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
- "name" : "V_TEST_CAL_DT",
- "columns" : [ {
- "id" : "1",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "2",
- "name" : "YEAR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "QTR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "4",
- "name" : "MONTH_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "5",
- "name" : "WEEK_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "6",
- "name" : "AGE_FOR_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "AGE_FOR_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "8",
- "name" : "AGE_FOR_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "9",
- "name" : "AGE_FOR_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "10",
- "name" : "AGE_FOR_DT_ID",
- "datatype" : "smallint"
- }, {
- "id" : "11",
- "name" : "AGE_FOR_RTL_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "12",
- "name" : "AGE_FOR_RTL_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "13",
- "name" : "AGE_FOR_RTL_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "14",
- "name" : "AGE_FOR_RTL_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "15",
- "name" : "AGE_FOR_CS_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "16",
- "name" : "DAY_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "17",
- "name" : "DAY_OF_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "18",
- "name" : "DAY_OF_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "19",
- "name" : "DAY_OF_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "20",
- "name" : "DAY_OF_WEEK_ID",
- "datatype" : "int"
- }, {
- "id" : "21",
- "name" : "WEEK_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "22",
- "name" : "WEEK_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "23",
- "name" : "MONTH_OF_QTR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "24",
- "name" : "MONTH_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "25",
- "name" : "MONTH_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "26",
- "name" : "QTR_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "27",
- "name" : "QTR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "28",
- "name" : "YEAR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "29",
- "name" : "YEAR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "30",
- "name" : "QTR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "MONTH_END_DT",
- "datatype" : "string"
- }, {
- "id" : "32",
- "name" : "WEEK_END_DT",
- "datatype" : "string"
- }, {
- "id" : "33",
- "name" : "CAL_DT_NAME",
- "datatype" : "string"
- }, {
- "id" : "34",
- "name" : "CAL_DT_DESC",
- "datatype" : "string"
- }, {
- "id" : "35",
- "name" : "CAL_DT_SHORT_NAME",
- "datatype" : "string"
- }, {
- "id" : "36",
- "name" : "YTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "37",
- "name" : "QTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "38",
- "name" : "MTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "39",
- "name" : "WTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "40",
- "name" : "SEASON_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "41",
- "name" : "DAY_IN_YEAR_COUNT",
- "datatype" : "smallint"
- }, {
- "id" : "42",
- "name" : "DAY_IN_QTR_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "43",
- "name" : "DAY_IN_MONTH_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "44",
- "name" : "DAY_IN_WEEK_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "45",
- "name" : "RTL_YEAR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "46",
- "name" : "RTL_QTR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "47",
- "name" : "RTL_MONTH_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "48",
- "name" : "RTL_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "49",
- "name" : "CS_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "50",
- "name" : "CAL_DATE",
- "datatype" : "string"
- }, {
- "id" : "51",
- "name" : "DAY_OF_WEEK",
- "datatype" : "string"
- }, {
- "id" : "52",
- "name" : "MONTH_ID",
- "datatype" : "string"
- }, {
- "id" : "53",
- "name" : "PRD_DESC",
- "datatype" : "string"
- }, {
- "id" : "54",
- "name" : "PRD_FLAG",
- "datatype" : "string"
- }, {
- "id" : "55",
- "name" : "PRD_ID",
- "datatype" : "string"
- }, {
- "id" : "56",
- "name" : "PRD_IND",
- "datatype" : "string"
- }, {
- "id" : "57",
- "name" : "QTR_DESC",
- "datatype" : "string"
- }, {
- "id" : "58",
- "name" : "QTR_ID",
- "datatype" : "string"
- }, {
- "id" : "59",
- "name" : "QTR_IND",
- "datatype" : "string"
- }, {
- "id" : "60",
- "name" : "RETAIL_WEEK",
- "datatype" : "string"
- }, {
- "id" : "61",
- "name" : "RETAIL_YEAR",
- "datatype" : "string"
- }, {
- "id" : "62",
- "name" : "RETAIL_START_DATE",
- "datatype" : "string"
- }, {
- "id" : "63",
- "name" : "RETAIL_WK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "64",
- "name" : "WEEK_IND",
- "datatype" : "string"
- }, {
- "id" : "65",
- "name" : "WEEK_NUM_DESC",
- "datatype" : "string"
- }, {
- "id" : "66",
- "name" : "WEEK_BEG_DATE",
- "datatype" : "string"
- }, {
- "id" : "67",
- "name" : "WEEK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "68",
- "name" : "WEEK_IN_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "69",
- "name" : "WEEK_ID",
- "datatype" : "string"
- }, {
- "id" : "70",
- "name" : "WEEK_BEG_END_DESC_MDY",
- "datatype" : "string"
- }, {
- "id" : "71",
- "name" : "WEEK_BEG_END_DESC_MD",
- "datatype" : "string"
- }, {
- "id" : "72",
- "name" : "YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "73",
- "name" : "YEAR_IND",
- "datatype" : "string"
- }, {
- "id" : "74",
- "name" : "CAL_DT_MNS_1YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "75",
- "name" : "CAL_DT_MNS_2YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "76",
- "name" : "CAL_DT_MNS_1QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "77",
- "name" : "CAL_DT_MNS_2QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "78",
- "name" : "CAL_DT_MNS_1MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "79",
- "name" : "CAL_DT_MNS_2MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "80",
- "name" : "CAL_DT_MNS_1WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "81",
- "name" : "CAL_DT_MNS_2WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "82",
- "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "83",
- "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "84",
- "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "85",
- "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "86",
- "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "87",
- "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "88",
- "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "89",
- "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "90",
- "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "91",
- "name" : "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "92",
- "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "93",
- "name" : "SEASON_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "94",
- "name" : "YTM_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "95",
- "name" : "YTQ_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "96",
- "name" : "YTW_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "97",
- "name" : "V_CAL_DT_CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "98",
- "name" : "V_CAL_DT_CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "99",
- "name" : "V_CAL_DT_UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "100",
- "name" : "V_CAL_DT_UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "edw",
- "table_type" : "VIRTUAL_VIEW",
- "last_modified" : 0
-}
\ No newline at end of file
[44/50] [abbrv] kylin git commit: KYLIN 1875 minor,
update model ang cube designer
Posted by li...@apache.org.
KYLIN 1875 minor,update model ang cube designer
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/878107e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/878107e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/878107e3
Branch: refs/heads/master-cdh5.7
Commit: 878107e35bf4ac3a7963f22ac00a805eea9c323f
Parents: 6f563df
Author: chenzhx <34...@qq.com>
Authored: Mon Dec 19 14:16:47 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Mon Dec 19 16:29:42 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 31 +--------
webapp/app/js/controllers/cubeDimensions.js | 38 ++++-------
webapp/app/js/controllers/cubeEdit.js | 6 +-
webapp/app/js/controllers/cubeMeasures.js | 12 ++--
webapp/app/js/controllers/cubeSchema.js | 1 -
webapp/app/js/controllers/modelDataModel.js | 6 --
webapp/app/js/controllers/modelEdit.js | 2 +-
webapp/app/js/filters/filter.js | 17 ++++-
.../cubeDesigner/advanced_settings.html | 17 +----
.../app/partials/cubeDesigner/dimensions.html | 69 ++++++++++----------
webapp/app/partials/cubeDesigner/measures.html | 19 +++---
.../app/partials/modelDesigner/data_model.html | 34 ++++------
.../partials/modelDesigner/model_measures.html | 2 +-
13 files changed, 100 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 20af908..760133a 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -21,7 +21,7 @@
KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfig,MetaModel,cubesManager,CubeDescModel,SweetAlert) {
$scope.cubesManager = cubesManager;
$scope.getTypeVersion=function(typename){
- var searchResult=/\[v(\d+)\]/.exec(typename);
+ var searchResult=/\s*\(v(\d+)\)/.exec(typename);
if(searchResult&&searchResult.length){
return searchResult.length&&searchResult[1]||1;
}else{
@@ -30,7 +30,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
}
$scope.removeVersion=function(typename){
if(typename){
- return typename.replace(/\[v\d+\]/g,"");
+ return typename.replace(/\s*\(v\d+\)/g,"");
}
return "";
}
@@ -49,7 +49,7 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
_encoding=baseKey;
var rowkeyObj = {
column:item.column,
- encoding:_encoding+(item.encoding_version?"[v"+item.encoding_version+"]":"[v1]"),
+ encoding:_encoding+(item.encoding_version?" (v"+item.encoding_version+")":" (v1)"),
valueLength:_valueLength,
isShardBy:item.isShardBy,
encoding_version:item.encoding_version||1
@@ -109,31 +109,6 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
}
}
- $scope.removeRowkey = function(arr,index,item){
- if (index > -1) {
- arr.splice(index, 1);
- }
- $scope.cubeMetaFrame.rowkey.rowkey_columns.splice(index,1);
- }
-
-
- $scope.addNewRowkeyColumn = function () {
- var rowkeyObj = {
- column:"",
- encoding:"dict",
- valueLength:0,
- isShardBy:"false"
- }
-
- $scope.convertedRowkeys.push(rowkeyObj);
- $scope.cubeMetaFrame.rowkey.rowkey_columns.push({
- column:'',
- encoding:'dict',
- isShardBy:'false'
- });
-
- };
-
$scope.addNewHierarchy = function(grp){
grp.select_rule.hierarchy_dims.push([]);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/cubeDimensions.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeDimensions.js b/webapp/app/js/controllers/cubeDimensions.js
index e2787a5..80d54bb 100644
--- a/webapp/app/js/controllers/cubeDimensions.js
+++ b/webapp/app/js/controllers/cubeDimensions.js
@@ -206,9 +206,7 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
});
modalInstance.result.then(function () {
- if (!$scope.dimState.editing) {
- $scope.doneAddDim();
- } else {
+ if ($scope.dimState.editing) {
$scope.doneEditDim();
}
@@ -268,31 +266,31 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
};
- $scope.addDim = function (dimType) {
- $scope.newDimension = Dimension('', [], dimType);
-
- $scope.openDimModal(dimType);
- };
-
$scope.editDim = function (dim) {
$scope.dimState.editingIndex = dimList.indexOf(dim);
$scope.dimState.editing = true;
// Make a copy of model will be editing.
$scope.newDimension = angular.copy(dim);
+ if($scope.newDimension.derived&&$scope.newDimension.derived.length>0){
+ $scope.newDimension.normal="false";
+ }else{
+ $scope.newDimension.normal="true";
+ }
$scope.openDimModal($scope.getDimType(dim));
};
- $scope.doneAddDim = function () {
- // Push new dimension which bound user input data.
- dimList.push(angular.copy($scope.newDimension));
-
- $scope.resetParams();
- };
-
$scope.doneEditDim = function () {
// Copy edited model to destination model.
+ if($scope.newDimension.derived&&($scope.newDimension.normal=="true")){
+ $scope.newDimension.column=$scope.newDimension.derived[0];
+ $scope.newDimension.derived=null;
+ }
+ if(!$scope.newDimension.derived&&($scope.newDimension.normal=="false")){
+ $scope.newDimension.derived=[$scope.newDimension.column];
+ $scope.newDimension.column=null;
+ }
angular.copy($scope.newDimension, dimList[$scope.dimState.editingIndex]);
$scope.resetParams();
@@ -490,14 +488,6 @@ KylinApp.controller('CubeDimensionsCtrl', function ($scope, $modal,MetaModel,cub
}
}
- $scope.addNewDimension = function(newDimension){
- if(newDimension.derived==null){
- newDimension.derived=[];
- }
- newDimension.derived.push('');
- }
-
-
// Just reset the selected status of columns.
$scope.resetGenDims = function () {
var selectedCols = $scope.getSelectedCols();
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index a2b38f9..90cdbc0 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -60,7 +60,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
for(var s=1;s<=typeVersion;s++){
$scope.store.supportedEncoding.push({
"name":name+" (v"+s+","+(s==typeVersion&&typeVersion>1?"suggest)":")"),
- "value":value+"[v"+s+"]",
+ "value":value+" (v"+s+")",
"version":typeVersion,
"baseValue":value,
"suggest":s==typeVersion
@@ -70,7 +70,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
}else {
$scope.store.supportedEncoding.push({
"name": name,
- "value": value+"[v1]",
+ "value": value+" (v1)",
"encoding_version":1,
"version":typeVersion,
"baseValue":value,
@@ -103,7 +103,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
for(var s=0;s<$scope.cubeMetaFrame.rowkey.rowkey_columns.length;s++){
if(filterName==$scope.cubeMetaFrame.rowkey.rowkey_columns[s].column){
var version=$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding_version;
- filterEncoding=VdmUtil.getFilterObjectListByOrFilterVal(encodings,'value',$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding.replace(/:\d+/,"")+(version?"[v"+version+"]":"[v1]"),'suggest',true)
+ filterEncoding=VdmUtil.getFilterObjectListByOrFilterVal(encodings,'value',$scope.cubeMetaFrame.rowkey.rowkey_columns[s].encoding.replace(/:\d+/,"")+(version?" (v"+version+")":" (v1)"),'suggest',true)
}
}
}else{
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 67a107b..15c330a 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -32,12 +32,12 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
var needLengthKeyList=['fixed_length','fixed_length_hex','int','integer'];
$scope.removeVersion=function(typename){
if(typename){
- return typename.replace(/\[v\d+\]/g,"");
+ return typename.replace(/\s*\(v\d+\)/g,"");
}
return "";
}
$scope.getTypeVersion=function(typename){
- var searchResult=/\[v(\d+)\]/.exec(typename);
+ var searchResult=/\s*\(v(\d+)\)/.exec(typename);
if(searchResult&&searchResult.length){
return searchResult.length&&searchResult[1]||1;
}else{
@@ -62,7 +62,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
if($scope.isEdit) {
if (name && $scope.newMeasure.function.configuration&&$scope.newMeasure.function.configuration['topn.encoding.' + name]) {
var version = $scope.newMeasure.function.configuration['topn.encoding_version.' + name] || 1;
- filterEncoding = VdmUtil.getFilterObjectListByOrFilterVal(encodings, 'value', $scope.newMeasure.function.configuration['topn.encoding.' + name].replace(/:\d+/, "") + (version ? "[v" + version + "]" : "[v1]"), 'suggest', true);
+ filterEncoding = VdmUtil.getFilterObjectListByOrFilterVal(encodings, 'value', $scope.newMeasure.function.configuration['topn.encoding.' + name].replace(/:\d+/, "") + (version ? " (v" + version + ")" : " (v1)"), 'suggest', true);
}else{
filterEncoding=VdmUtil.getFilterObjectListByOrFilterVal(encodings,'suggest', true);
}
@@ -89,7 +89,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
if($scope.newMeasure.function.configuration==null){
var GroupBy = {
name:$scope.newMeasure.function.parameter.next_parameter.value,
- encoding:"dict",
+ encoding:"dict (v1)",
valueLength:0,
}
$scope.convertedColumns.push(GroupBy);
@@ -110,7 +110,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
_encoding=baseKey;
$scope.GroupBy = {
name:_name,
- encoding:_encoding+(version?"[v"+version+"]":"[v1]"),
+ encoding:_encoding+(version?" (v"+version+")":" (v1)"),
valueLength:_valueLength,
encoding_version:version||1
}
@@ -271,7 +271,7 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
$scope.addNewGroupByColumn = function () {
$scope.nextGroupBy = {
name:null,
- encoding:"dict",
+ encoding:"dict (v1)",
valueLength:0,
}
$scope.convertedColumns.push($scope.nextGroupBy);
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/cubeSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeSchema.js b/webapp/app/js/controllers/cubeSchema.js
index d7dfc09..357b6af 100755
--- a/webapp/app/js/controllers/cubeSchema.js
+++ b/webapp/app/js/controllers/cubeSchema.js
@@ -109,7 +109,6 @@ KylinApp.controller('CubeSchemaCtrl', function ($scope, QueryService, UserServic
};
$scope.goToStep = function(stepIndex){
- console.log($scope.cubeMode);
if($scope.cubeMode == "addNewCube"){
if(stepIndex+1>=$scope.curStep.step){
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/modelDataModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelDataModel.js b/webapp/app/js/controllers/modelDataModel.js
index 3834e06..95c42c4 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -264,9 +264,6 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
break;
}
}
- if($scope.aliasName.indexOf($scope.newLookup.alias)!=-1&&$scope.lookupState.editing == false){
- errors.push("Table Alias ["+$scope.newLookup.alias+"] already exist!");
- }
if($scope.aliasName.indexOf($scope.newLookup.alias)!=-1&&$scope.aliasName[$scope.lookupState.editingIndex+1] != $scope.newLookup.alias){
errors.push("Table Alias ["+$scope.newLookup.alias+"] already exist!");
}
@@ -281,7 +278,4 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
return true;
}
};
- $scope.filterNotRoot = function (item) {
- return item.name!==modelsManager.selectedModel.fact_table;
- };
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index c504b52..351e779 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -209,7 +209,7 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
$scope.state.modelSchema = request.modelSchema;
SweetAlert.swal('', 'Created the model successfully.', 'success');
$location.path("/models");
- location.reload();
+ // location.reload();
} else {
$scope.saveModelRollBack();
var message =request.message;
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/js/filters/filter.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index f9f7165..aff4e3a 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -190,4 +190,19 @@ KylinApp
return _day +" (Days)";
}
}
- });
+ }).filter('inDimNotInMea', function ($filter) {
+ return function (inputArr, table, arr) {
+ var out=[];
+ angular.forEach(arr,function(item){
+ if(item.table==table){
+ angular.forEach(inputArr,function(inputItem){
+ if(item.columns.indexOf(inputItem.name)==-1){
+ out.push(inputItem);
+ }
+ });
+ }
+ });
+ return out;
+ }
+ })
+ ;
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index 438eb1f..f9e422c 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -202,11 +202,10 @@
<thead>
<tr>
<th>ID</th>
- <th class="col-xs-4">Column</th>
+ <th class="col-xs-5">Column</th>
<th>Encoding</th>
<th>Length</th>
<th>Shard By</th>
- <th ng-if="state.mode=='edit'"></th>
</tr>
</thead>
@@ -243,11 +242,11 @@
<!--Column Length -->
<input type="text" class="form-control" placeholder="Column Length.." ng-if="state.mode=='edit'"
tooltip="rowkey column length.." tooltip-trigger="focus"
- ng-disabled="rowkey_column.encoding=='dict'||rowkey_column.encoding=='date'||rowkey_column.encoding=='time'||rowkey_column.encoding=='boolean'||instance.status=='READY'"
+ ng-disabled="rowkey_column.encoding.indexOf('dict')>=0||rowkey_column.encoding.indexOf('date')>=0||rowkey_column.encoding.indexOf('time')>=0||instance.status=='READY'"
ng-change="refreshRowKey(convertedRowkeys,$index,rowkey_column);"
ng-model="rowkey_column.valueLength" class="form-control">
- <small class="help-block red" ng-show="state.mode=='edit' && rowkey_column.encoding=='int' && (rowkey_column.valueLength>8 || rowkey_column.valueLength<1)">int encoding column length should between 1 and 8</small>
+ <small class="help-block red" ng-show="state.mode=='edit' && rowkey_column.encoding.indexOf('integer')>=0 && (rowkey_column.valueLength>8 || rowkey_column.valueLength<1)">integer encoding column length should between 1 and 8</small>
<span ng-if="state.mode=='view'">{{rowkey_column.valueLength}}</span>
</td>
@@ -263,21 +262,11 @@
<small class="help-block red" ng-show="state.mode=='edit' && rule.shardColumnAvailable==false && rowkey_column.isShardBy == true">at most one 'shard by' column is allowed.</small>
<span ng-if="state.mode=='view'">{{rowkey_column.isShardBy}}</span>
</td>
-
- <td ng-if="state.mode=='edit'">
- <button class="btn btn-xs btn-info" ng-disabled="instance.status=='READY'"
- ng-click="removeRowkey(convertedRowkeys, $index,rowkey_column)"><i
- class="fa fa-minus"></i>
- </button>
- </td>
</tr>
</tbody>
</table>
</div>
</div>
- <button class="btn btn-sm btn-info" style="margin-left:42px" ng-disabled="instance.status=='READY'"
- ng-click="addNewRowkeyColumn()" ng-show="state.mode=='edit'">New Rowkey Column<i class="fa fa-plus"></i>
- </button>
</div>
<div class="form-group large-popover" style="overflow:auto">
<h3 style="margin-left:42px">Advanced Dictionaries <i kylinpopover placement="right" title="Advanced Dictionaries" template="AdvancedDictionariesTip.html" class="fa fa-info-circle"></i></h3>
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/partials/cubeDesigner/dimensions.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html b/webapp/app/partials/cubeDesigner/dimensions.html
index 7fedeb6..d8a8fce 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -24,7 +24,7 @@
<div class="col-xs-6" ng-if="state.mode=='edit'">
<button type="button" class="btn btn-default" ng-disabled="!metaModel.model.fact_table.length||instance.status=='READY'"
ng-click="openAutoGenModal()" >
- <i class="fa fa-building-o"></i> Auto Generator
+ <i class="fa fa-building-o"></i> Add Dimensions
</button>
</div>
<div ng-class="(state.mode=='edit')?'col-xs-6':'col-xs-12'">
@@ -57,7 +57,7 @@
</td>
<!--Table Name -->
<td>
- <span tooltip="{{availableFactTables.indexOf(dimension.table)==-1? 'Lookup Table':'Fact Table'}}">{{dimension.table}}</span>
+ <span>{{dimension.table}}</span>
</td>
<!--Type-->
<td>
@@ -109,7 +109,7 @@
<!-- Edit Dimension Form -->
<script type="text/ng-template" id="addEditDimension.html">
<div class="modal-header">
- <h4 class="box-title lighter">{{dimState.editing ? 'Edit' : 'Add'}} Dimension
+ <h4 class="box-title lighter">Edit Dimension
<span class="label label-primary" ng-repeat="t in dimType">{{t}}</span>
</h4>
</div>
@@ -138,17 +138,8 @@
<div class="form-group">
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Table Name</b></label>
- <div class="col-xs-12 col-sm-6" ng-if="dimType.indexOf('derived') >= 0">
- <select class="form-control" required="true" chosen data-placeholder="Select a lookup table"
- ng-model="newDimension.table" ng-options="table for table in availableLookupTables">
- <option value=""></option>
- </select>
- </div>
- <div class="col-xs-12 col-sm-6" ng-if="dimType.indexOf('derived') == -1">
- <select class="form-control" required="true" chosen data-placeholder="Select a table"
- ng-model="newDimension.table" ng-options="table for table in availableFactTables">
- <option value=""></option>
- </select>
+ <div class="col-xs-12 col-sm-6" >
+ <span>{{newDimension.table}}</span>
</div>
<div class="col-xs-12 col-sm-3">
<div class="text-info" ng-if="dimType.indexOf('derived') >= 0">
@@ -161,12 +152,11 @@
<!--Normal Dimension: choose one column-->
<div class="form-group" ng-if="dimType.indexOf('normal') >= 0">
<div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Column Name</b></label>
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default">
+ <b>Column Name</b>
+ </label>
<div class="col-xs-12 col-sm-6">
- <select class="form-control" required="true" chosen data-placeholder="Select a column"
- ng-model="newDimension.column" ng-options="column.name as column.name for column in getDimColumnsByAlias(newDimension.table)">
- <option value=""></option>
- </select>
+ <span>{{newDimension.column}}</span>
</div>
</div>
</div>
@@ -174,27 +164,34 @@
<div ng-if="dimType.indexOf('derived') >= 0">
<div class="form-group">
<div class="row">
- <div class="col-sm-6 col-sm-offset-3">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default">
+ <b>Column Name</b>
+ </label>
+ <div class="col-sm-6 ">
<div ng-repeat="derived in newDimension.derived track by $index">
- <div>
- <select chosen style="width: 80%;" data-placeholder="Derived Columns.."
- ng-model="newDimension.derived[$index]"
- ng-options="columns.name as columns.name for columns in getDimColumnsByAlias(newDimension.table)" >
- <option value=""></option>
- </select>
- <button class="pull-right btn btn-xs btn-danger" style="cursor: pointer " tooltip="Delete"
- ng-click="newDimension.derived.splice($index, 1);">
- <i class="fa fa-trash-o"></i>
- </button>
- <div class="space-4"></div>
- </div>
+ <span >
+ {{newDimension.derived[$index]}}
+ </span>
</div>
- <button class="btn btn-xs btn-info" ng-click="addNewDimension(newDimension);">
- <i class="fa fa-plus"></i> New Derived</button>
</div>
</div>
</div>
</div>
+ <div ng-if="availableLookupTables.indexOf(newDimension.table)!=-1">
+ <div class="row">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default">
+ <b>Type</b>
+ </label>
+ <div class="col-xs-12 col-sm-6">
+ <label>
+ <input type="radio" ng-model="newDimension.normal" value="true" > Normal
+ </label>
+ <label>
+ <input type="radio" ng-model="newDimension.normal" value="false" > Derived
+ </label>
+ </div>
+ </div>
+ </div>
</div>
</div>
</ng-form>
@@ -221,7 +218,7 @@
<ul class="list-unstyled columns-region">
<!--FactTable-->
<div ng-repeat="table in availableFactTables track by $index" class="panel-default " >
- <h4>{{table}}[FactTable]</h4>
+ <h4>{{table}} [FactTable]</h4>
<table class="table table-striped table-hover ng-scope">
<tr >
<td class="col-xs-2"><label><input type="checkbox" ng-model="selectedColumns[table].all" ng-change="autoChangeAll(table)">Select All</label></td>
@@ -247,7 +244,7 @@
</div>
<!--LookUp Table-->
<div ng-repeat="table in availableLookupTables track by $index" class="panel-default" >
- <h4>{{table}}[LookupTable]</h4>
+ <h4>{{table}} [LookupTable]</h4>
<table class="table table-striped table-hover ng-scope">
<tr class="row" >
<td class="col-xs-2"><label><input type="checkbox" ng-model="selectedColumns[table].all" ng-change="autoChangeAll(table)">Select All</label></td>
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/partials/cubeDesigner/measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 492220c..9eb04c3 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -239,10 +239,10 @@
class="table table-hover table-bordered list">
<thead>
<tr>
- <th>ID</th>
- <th class="col-xs-5" >Column</th>
- <th>Encoding</th>
- <th>Length</th>
+ <th style="width:40px;">ID</th>
+ <th style="width:300px;">Column</th>
+ <th style="width:140px;">Encoding</th>
+ <th >Length</th>
<th ng-if="state.mode=='edit'"></th>
</tr>
</thead>
@@ -258,7 +258,7 @@
</td>
<!--Column Name -->
<td>
- <select class="form-control col-xs-12" chosen ng-if="nextPara.type !== 'constant'" required
+ <select class="form-control" chosen ng-if="nextPara.type !== 'constant'" required
ng-model="groupby_column.name"
ng-options="column as column for column in getAllModelDimColumns()" >
<option value="">--Select A Column--</option>
@@ -266,25 +266,22 @@
</td>
<!--Column Encoding -->
<td>
- <select ng-if="state.mode=='edit'" style="width:180px;"
+ <select ng-if="state.mode=='edit'"
chosen ng-model="groupby_column.encoding"
ng-change="refreshGroupBy(convertedColumns,$index,groupby_column)"
ng-options="dt.value as dt.name for dt in getEncodings(groupby_column.name)">
<option value=""></option>
</select>
- <span ng-if="state.mode=='view'">{{groupby_column.encoding}}</span>
-
</td>
<td>
<!--Column Length -->
<input type="text" class="form-control" placeholder="Column Length.." ng-if="state.mode=='edit'"
tooltip="group by column length.." tooltip-trigger="focus"
+ ng-disabled="groupby_column.encoding.indexOf('dict')>=0||groupby_column.encoding.indexOf('date')>=0||groupby_column.encoding.indexOf('time')>=0"
ng-change="refreshGroupBy(convertedColumns,$index,groupby_column);"
- ng-disabled="groupby_column.encoding=='dict'||groupby_column.encoding=='date'||groupby_column.encoding=='time'"
ng-model="groupby_column.valueLength" class="form-control">
- <small class="help-block red" ng-show="state.mode=='edit' && groupby_column.encoding=='int' && (groupby_column.valueLength>8 || groupby_column.valueLength<1)">int encoding column length should between 1 and 8</small>
- <span ng-if="state.mode=='view'">{{groupby_column.valueLength}}</span>
+ <small class="help-block red" ng-show="state.mode=='edit' && groupby_column.encoding.indexOf('integer')>=0 && (groupby_column.valueLength>8 || groupby_column.valueLength<1)">integer encoding column length should between 1 and 8</small>
</td>
<td ng-if="state.mode=='edit'">
<button class="btn btn-xs btn-info"
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index 97d3c51..6e66e91 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -126,7 +126,7 @@
<div class="row">
<div class="col-xs-9">
<div>
- <select chosen ng-model="newLookup.joinTable" style="width: 45%"
+ <select chosen ng-model="newLookup.joinTable" class="col-sm-4"
ng-options="table as table for table in aliasName"
name="table_name" ng-disabled="lookupState.editing"
ng-required="true"
@@ -134,20 +134,21 @@
class="chosen-select">
<option value=""> -- Select Join Table -- </option>
</select>
- <small class="help-block" ng-show="lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$submitted)">Table name required</small>
- <b> </b>
- <select chosen ng-model="newLookup.table" style="width: 45%"
- ng-options="table.name as VdmUtil.removeNameSpace(table.name) for table in tableModel.selectProjectTables|filter:filterNotRoot"
+ <!--Join Type-->
+ <select class="form-control" chosen ng-model="newLookup.join.type" style="width:120px;"
+ ng-options="joinType.value as joinType.name for joinType in cubeConfig.joinTypes">
+ <option value=""></option>
+ </select>
+ <label class="control-label font-color-default"><b>Join</b></label>
+ <select chosen ng-model="newLookup.table" class="col-sm-4"
+ ng-options="table.name as VdmUtil.removeNameSpace(table.name) for table in tableModel.selectProjectTables"
name="table_name" ng-disabled="lookupState.editing"
ng-required="true" ng-change="changeJoinTable()"
data-placeholder="Join Table Name"
class="chosen-select">
<option value=""> -- Select Join Table -- </option>
</select>
- <small class="help-block" ng-show="lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$submitted)">Table name required</small>
-
</div>
- <div class="space-4"></div>
</div>
</div>
</div>
@@ -161,18 +162,7 @@
</div>
</div>
</div>
- <!--Join Type and Columns-->
- <div class="form-group">
- <div class="row">
- <label class="col-sm-3 control-label font-color-default"><b>Join Type</b></label>
- <div class="col-sm-6">
- <select class="form-control" chosen ng-model="newLookup.join.type"
- ng-options="joinType.value as joinType.name for joinType in cubeConfig.joinTypes">
- <option value=""></option>
- </select>
- </div>
- </div>
- </div>
+ <!--Table Type-->
<div class="form-group">
<div class="row">
<label class="col-sm-3 control-label font-color-default"><b>Table Type</b></label>
@@ -233,8 +223,8 @@
<div class="row">
<div class="col-xs-12">
<ol class="text-info">
- <li>Pick up lookup table at first</li>
- <li>Specify join relationship between fact table and chosen lookup table.</li>
+ <li>Pick up a table joins another table that already exist.</li>
+ <li>Specify join relationship between two tables.</li>
<li>Join Type have to be same as will be used in query</li>
</ol>
</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/878107e3/webapp/app/partials/modelDesigner/model_measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/model_measures.html b/webapp/app/partials/modelDesigner/model_measures.html
index f34e3fb..271c04f 100644
--- a/webapp/app/partials/modelDesigner/model_measures.html
+++ b/webapp/app/partials/modelDesigner/model_measures.html
@@ -67,7 +67,7 @@
ng-model="selectedFactTables[table]" multiple>
<ui-select-match placeholder="Select Column...">{{$item.name}}</ui-select-match>
<ui-select-choices
- repeat="table+'.'+measure.name as measure in getColumnsByAlias(table) | filter:$select.search">
+ repeat="table+'.'+measure.name as measure in getColumnsByAlias(table) | filter:$select.search|inDimNotInMea:table:modelsManager.selectedModel.dimensions">
{{measure.name}}
</ui-select-choices>
</ui-select>
[04/50] [abbrv] kylin git commit: KYLIN-2271 minor update
Posted by li...@apache.org.
KYLIN-2271 minor update
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4408579d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4408579d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4408579d
Branch: refs/heads/master-cdh5.7
Commit: 4408579d4b3e9f33364d738b569e5f81d6fcb5c9
Parents: 8265c3b
Author: shaofengshi <sh...@apache.org>
Authored: Mon Dec 12 11:37:25 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 12 11:37:25 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/controller/CubeController.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4408579d/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 4cf38e9..6e3668b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -324,12 +324,12 @@ public class CubeController extends BasicController {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
if (cube == null) {
- throw new IllegalArgumentException("Cannot find cube '" + cubeName + "'");
+ throw new InternalErrorException("Cannot find cube '" + cubeName + "'");
}
if (cube.getSegments() != null && cube.getBuildingSegments().size() > 0) {
int num = cube.getBuildingSegments().size();
- throw new IllegalStateException("Cannot purge cube '" + cubeName + "' as there is " + num + " building " + (num > 1 ? "segment(s)." : "segment."));
+ throw new InternalErrorException("Cannot purge cube '" + cubeName + "' as there is " + num + " building " + (num > 1 ? "segment(s)." : "segment. Discard the related job first."));
}
return cubeService.purgeCube(cube);
[41/50] [abbrv] kylin git commit: KYLIN-2292 workaround for
CALCITE-1540
Posted by li...@apache.org.
KYLIN-2292 workaround for CALCITE-1540
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ae4333c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ae4333c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ae4333c
Branch: refs/heads/master-cdh5.7
Commit: 4ae4333c82243c21d253008c9f5146f1e18f6e84
Parents: 6f9bd4a
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Dec 16 17:21:37 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Dec 19 14:16:08 2016 +0800
----------------------------------------------------------------------
.../adapter/enumerable/EnumerableWindow.java | 978 +++++++++++++++++++
.../calcite/adapter/enumerable/PhysType.java | 209 ++++
.../adapter/enumerable/PhysTypeImpl.java | 654 +++++++++++++
.../test/resources/query/sql_window/query11.sql | 23 +
.../test/resources/query/sql_window/query12.sql | 26 +
5 files changed, 1890 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae4333c/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
new file mode 100644
index 0000000..203ce02
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableWindow.java
@@ -0,0 +1,978 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.enumerable;
+
+import org.apache.calcite.adapter.enumerable.impl.WinAggAddContextImpl;
+import org.apache.calcite.adapter.enumerable.impl.WinAggResetContextImpl;
+import org.apache.calcite.adapter.enumerable.impl.WinAggResultContextImpl;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.linq4j.tree.BinaryExpression;
+import org.apache.calcite.linq4j.tree.BlockBuilder;
+import org.apache.calcite.linq4j.tree.BlockStatement;
+import org.apache.calcite.linq4j.tree.DeclarationStatement;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.linq4j.tree.ParameterExpression;
+import org.apache.calcite.linq4j.tree.Primitive;
+import org.apache.calcite.linq4j.tree.Statement;
+import org.apache.calcite.linq4j.tree.Types;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.CalcitePrepareImpl;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Window;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexWindowBound;
+import org.apache.calcite.runtime.SortedMultiMap;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.util.BuiltInMethod;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+
+import java.lang.reflect.Modifier;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+/*
+ * OVERRIDE POINT: patching CALCITE-1540 on calcite 1.8.0
+ */
+
+/** Implementation of {@link org.apache.calcite.rel.core.Window} in
+ * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. */
+public class EnumerableWindow extends Window implements EnumerableRel {
+ /** Creates an EnumerableWindowRel. */
+ EnumerableWindow(RelOptCluster cluster, RelTraitSet traits, RelNode child,
+ List<RexLiteral> constants, RelDataType rowType, List<Group> groups) {
+ super(cluster, traits, child, constants, rowType, groups);
+ }
+
+ @Override public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ return new EnumerableWindow(getCluster(), traitSet, sole(inputs),
+ constants, rowType, groups);
+ }
+
+ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
+ return super.computeSelfCost(planner, mq)
+ .multiplyBy(EnumerableConvention.COST_MULTIPLIER);
+ }
+
+ /** Implementation of {@link RexToLixTranslator.InputGetter}
+ * suitable for generating implementations of windowed aggregate
+ * functions. */
+ private static class WindowRelInputGetter
+ implements RexToLixTranslator.InputGetter {
+ private final Expression row;
+ private final PhysType rowPhysType;
+ private final int actualInputFieldCount;
+ private final List<Expression> constants;
+
+ private WindowRelInputGetter(Expression row,
+ PhysType rowPhysType, int actualInputFieldCount,
+ List<Expression> constants) {
+ this.row = row;
+ this.rowPhysType = rowPhysType;
+ this.actualInputFieldCount = actualInputFieldCount;
+ this.constants = constants;
+ }
+
+ public Expression field(BlockBuilder list, int index, Type storageType) {
+ if (index < actualInputFieldCount) {
+ Expression current = list.append("current", row);
+ return rowPhysType.fieldReference(current, index, storageType);
+ }
+ return constants.get(index - actualInputFieldCount);
+ }
+ }
+
+ private void sampleOfTheGeneratedWindowedAggregate() {
+ // Here's overview of the generated code
+ // For each list of rows that have the same partitioning key, evaluate
+ // all of the windowed aggregate functions.
+
+ // builder
+ Iterator<Integer[]> iterator = null;
+
+ // builder3
+ Integer[] rows = iterator.next();
+
+ int prevStart = -1;
+ int prevEnd = -1;
+
+ for (int i = 0; i < rows.length; i++) {
+ // builder4
+ Integer row = rows[i];
+
+ int start = 0;
+ int end = 100;
+ if (start != prevStart || end != prevEnd) {
+ // builder5
+ int actualStart = 0;
+ if (start != prevStart || end < prevEnd) {
+ // builder6
+ // recompute
+ actualStart = start;
+ // implementReset
+ } else { // must be start == prevStart && end > prevEnd
+ actualStart = prevEnd + 1;
+ }
+ prevStart = start;
+ prevEnd = end;
+
+ if (start != -1) {
+ for (int j = actualStart; j <= end; j++) {
+ // builder7
+ // implementAdd
+ }
+ }
+ // implementResult
+ // list.add(new Xxx(row.deptno, row.empid, sum, count));
+ }
+ }
+ // multiMap.clear(); // allows gc
+ // source = Linq4j.asEnumerable(list);
+ }
+
+ public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
+ final JavaTypeFactory typeFactory = implementor.getTypeFactory();
+ final EnumerableRel child = (EnumerableRel) getInput();
+ final BlockBuilder builder = new BlockBuilder();
+ final Result result = implementor.visitChild(this, 0, child, pref);
+ Expression source_ = builder.append("source", result.block);
+
+ final List<Expression> translatedConstants =
+ new ArrayList<Expression>(constants.size());
+ for (RexLiteral constant : constants) {
+ translatedConstants.add(
+ RexToLixTranslator.translateLiteral(constant, constant.getType(),
+ typeFactory, RexImpTable.NullAs.NULL));
+ }
+
+ PhysType inputPhysType = result.physType;
+
+ ParameterExpression prevStart =
+ Expressions.parameter(int.class, builder.newName("prevStart"));
+ ParameterExpression prevEnd =
+ Expressions.parameter(int.class, builder.newName("prevEnd"));
+
+ builder.add(Expressions.declare(0, prevStart, null));
+ builder.add(Expressions.declare(0, prevEnd, null));
+
+ for (int windowIdx = 0; windowIdx < groups.size(); windowIdx++) {
+ Group group = groups.get(windowIdx);
+ // Comparator:
+ // final Comparator<JdbcTest.Employee> comparator =
+ // new Comparator<JdbcTest.Employee>() {
+ // public int compare(JdbcTest.Employee o1,
+ // JdbcTest.Employee o2) {
+ // return Integer.compare(o1.empid, o2.empid);
+ // }
+ // };
+ final Expression comparator_ =
+ builder.append(
+ "comparator",
+ inputPhysType.generateComparator(
+ group.collation()));
+
+ Pair<Expression, Expression> partitionIterator =
+ getPartitionIterator(builder, source_, inputPhysType, group,
+ comparator_);
+ final Expression collectionExpr = partitionIterator.left;
+ final Expression iterator_ = partitionIterator.right;
+
+ List<AggImpState> aggs = new ArrayList<AggImpState>();
+ List<AggregateCall> aggregateCalls = group.getAggregateCalls(this);
+ for (int aggIdx = 0; aggIdx < aggregateCalls.size(); aggIdx++) {
+ AggregateCall call = aggregateCalls.get(aggIdx);
+ aggs.add(new AggImpState(aggIdx, call, true));
+ }
+
+ // The output from this stage is the input plus the aggregate functions.
+ final RelDataTypeFactory.FieldInfoBuilder typeBuilder =
+ typeFactory.builder();
+ typeBuilder.addAll(inputPhysType.getRowType().getFieldList());
+ for (AggImpState agg : aggs) {
+ typeBuilder.add(agg.call.name, agg.call.type);
+ }
+ RelDataType outputRowType = typeBuilder.build();
+ final PhysType outputPhysType =
+ PhysTypeImpl.of(
+ typeFactory, outputRowType, pref.prefer(result.format));
+
+ final Expression list_ =
+ builder.append(
+ "list",
+ Expressions.new_(
+ ArrayList.class,
+ Expressions.call(
+ collectionExpr, BuiltInMethod.COLLECTION_SIZE.method)),
+ false);
+
+ Pair<Expression, Expression> collationKey =
+ getRowCollationKey(builder, inputPhysType, group, windowIdx);
+ Expression keySelector = collationKey.left;
+ Expression keyComparator = collationKey.right;
+ final BlockBuilder builder3 = new BlockBuilder();
+ final Expression rows_ =
+ builder3.append(
+ "rows",
+ Expressions.convert_(
+ Expressions.call(
+ iterator_, BuiltInMethod.ITERATOR_NEXT.method),
+ Object[].class),
+ false);
+
+ builder3.add(
+ Expressions.statement(
+ Expressions.assign(prevStart, Expressions.constant(-1))));
+ builder3.add(
+ Expressions.statement(
+ Expressions.assign(prevEnd,
+ Expressions.constant(Integer.MAX_VALUE))));
+
+ final BlockBuilder builder4 = new BlockBuilder();
+
+ final ParameterExpression i_ =
+ Expressions.parameter(int.class, builder4.newName("i"));
+
+ final Expression row_ =
+ builder4.append(
+ "row",
+ RexToLixTranslator.convert(
+ Expressions.arrayIndex(rows_, i_),
+ inputPhysType.getJavaRowType()));
+
+ final RexToLixTranslator.InputGetter inputGetter =
+ new WindowRelInputGetter(row_, inputPhysType,
+ result.physType.getRowType().getFieldCount(),
+ translatedConstants);
+
+ final RexToLixTranslator translator =
+ RexToLixTranslator.forAggregation(typeFactory, builder4,
+ inputGetter);
+
+ final List<Expression> outputRow = new ArrayList<Expression>();
+ int fieldCountWithAggResults =
+ inputPhysType.getRowType().getFieldCount();
+ for (int i = 0; i < fieldCountWithAggResults; i++) {
+ outputRow.add(
+ inputPhysType.fieldReference(
+ row_, i,
+ outputPhysType.getJavaFieldType(i)));
+ }
+
+ declareAndResetState(typeFactory, builder, result, windowIdx, aggs,
+ outputPhysType, outputRow);
+
+ // There are assumptions that minX==0. If ever change this, look for
+ // frameRowCount, bounds checking, etc
+ final Expression minX = Expressions.constant(0);
+ final Expression partitionRowCount =
+ builder3.append("partRows", Expressions.field(rows_, "length"));
+ final Expression maxX = builder3.append("maxX",
+ Expressions.subtract(
+ partitionRowCount, Expressions.constant(1)));
+
+ final Expression startUnchecked = builder4.append("start",
+ translateBound(translator, i_, row_, minX, maxX, rows_,
+ group, true,
+ inputPhysType, comparator_, keySelector, keyComparator));
+ final Expression endUnchecked = builder4.append("end",
+ translateBound(translator, i_, row_, minX, maxX, rows_,
+ group, false,
+ inputPhysType, comparator_, keySelector, keyComparator));
+
+ final Expression startX;
+ final Expression endX;
+ final Expression hasRows;
+ if (group.isAlwaysNonEmpty()) {
+ startX = startUnchecked;
+ endX = endUnchecked;
+ hasRows = Expressions.constant(true);
+ } else {
+ Expression startTmp =
+ group.lowerBound.isUnbounded() || startUnchecked == i_
+ ? startUnchecked
+ : builder4.append("startTmp",
+ Expressions.call(null, BuiltInMethod.MATH_MAX.method,
+ startUnchecked, minX));
+ Expression endTmp =
+ group.upperBound.isUnbounded() || endUnchecked == i_
+ ? endUnchecked
+ : builder4.append("endTmp",
+ Expressions.call(null, BuiltInMethod.MATH_MIN.method,
+ endUnchecked, maxX));
+
+ ParameterExpression startPe = Expressions.parameter(0, int.class,
+ builder4.newName("startChecked"));
+ ParameterExpression endPe = Expressions.parameter(0, int.class,
+ builder4.newName("endChecked"));
+ builder4.add(Expressions.declare(Modifier.FINAL, startPe, null));
+ builder4.add(Expressions.declare(Modifier.FINAL, endPe, null));
+
+ hasRows = builder4.append("hasRows",
+ Expressions.lessThanOrEqual(startTmp, endTmp));
+ builder4.add(
+ Expressions.ifThenElse(hasRows,
+ Expressions.block(
+ Expressions.statement(
+ Expressions.assign(startPe, startTmp)),
+ Expressions.statement(
+ Expressions.assign(endPe, endTmp))),
+ Expressions.block(
+ Expressions.statement(
+ Expressions.assign(startPe, Expressions.constant(-1))),
+ Expressions.statement(
+ Expressions.assign(endPe, Expressions.constant(-1))))));
+ startX = startPe;
+ endX = endPe;
+ }
+
+ final BlockBuilder builder5 = new BlockBuilder(true, builder4);
+
+ BinaryExpression rowCountWhenNonEmpty = Expressions.add(
+ startX == minX ? endX : Expressions.subtract(endX, startX),
+ Expressions.constant(1));
+
+ final Expression frameRowCount;
+
+ if (hasRows.equals(Expressions.constant(true))) {
+ frameRowCount =
+ builder4.append("totalRows", rowCountWhenNonEmpty);
+ } else {
+ frameRowCount =
+ builder4.append("totalRows",
+ Expressions.condition(hasRows, rowCountWhenNonEmpty,
+ Expressions.constant(0)));
+ }
+
+ ParameterExpression actualStart = Expressions.parameter(
+ 0, int.class, builder5.newName("actualStart"));
+
+ final BlockBuilder builder6 = new BlockBuilder(true, builder5);
+ builder6.add(
+ Expressions.statement(Expressions.assign(actualStart, startX)));
+
+ for (final AggImpState agg : aggs) {
+ agg.implementor.implementReset(agg.context,
+ new WinAggResetContextImpl(builder6, agg.state, i_, startX, endX,
+ hasRows, partitionRowCount, frameRowCount));
+ }
+
+ Expression lowerBoundCanChange =
+ group.lowerBound.isUnbounded() && group.lowerBound.isPreceding()
+ ? Expressions.constant(false)
+ : Expressions.notEqual(startX, prevStart);
+ Expression needRecomputeWindow = Expressions.orElse(
+ lowerBoundCanChange,
+ Expressions.lessThan(endX, prevEnd));
+
+ BlockStatement resetWindowState = builder6.toBlock();
+ if (resetWindowState.statements.size() == 1) {
+ builder5.add(
+ Expressions.declare(0, actualStart,
+ Expressions.condition(needRecomputeWindow, startX,
+ Expressions.add(prevEnd, Expressions.constant(1)))));
+ } else {
+ builder5.add(
+ Expressions.declare(0, actualStart, null));
+ builder5.add(
+ Expressions.ifThenElse(needRecomputeWindow,
+ resetWindowState,
+ Expressions.statement(
+ Expressions.assign(actualStart,
+ Expressions.add(prevEnd, Expressions.constant(1))))));
+ }
+
+ if (lowerBoundCanChange instanceof BinaryExpression) {
+ builder5.add(
+ Expressions.statement(Expressions.assign(prevStart, startX)));
+ }
+ builder5.add(
+ Expressions.statement(Expressions.assign(prevEnd, endX)));
+
+ final BlockBuilder builder7 = new BlockBuilder(true, builder5);
+ final DeclarationStatement jDecl =
+ Expressions.declare(0, "j", actualStart);
+
+ final PhysType inputPhysTypeFinal = inputPhysType;
+ final Function<BlockBuilder, WinAggFrameResultContext>
+ resultContextBuilder =
+ getBlockBuilderWinAggFrameResultContextFunction(typeFactory, result,
+ translatedConstants, comparator_, rows_, i_, startX, endX,
+ minX, maxX,
+ hasRows, frameRowCount, partitionRowCount,
+ jDecl, inputPhysTypeFinal);
+
+ final Function<AggImpState, List<RexNode>> rexArguments =
+ new Function<AggImpState, List<RexNode>>() {
+ public List<RexNode> apply(AggImpState agg) {
+ List<Integer> argList = agg.call.getArgList();
+ List<RelDataType> inputTypes =
+ EnumUtils.fieldRowTypes(
+ result.physType.getRowType(),
+ constants,
+ argList);
+ List<RexNode> args = new ArrayList<RexNode>(
+ inputTypes.size());
+ for (int i = 0; i < argList.size(); i++) {
+ Integer idx = argList.get(i);
+ args.add(new RexInputRef(idx, inputTypes.get(i)));
+ }
+ return args;
+ }
+ };
+
+ implementAdd(aggs, builder7, resultContextBuilder, rexArguments, jDecl);
+
+ BlockStatement forBlock = builder7.toBlock();
+ if (!forBlock.statements.isEmpty()) {
+ // For instance, row_number does not use for loop to compute the value
+ Statement forAggLoop = Expressions.for_(
+ Arrays.asList(jDecl),
+ Expressions.lessThanOrEqual(jDecl.parameter, endX),
+ Expressions.preIncrementAssign(jDecl.parameter),
+ forBlock);
+ if (!hasRows.equals(Expressions.constant(true))) {
+ forAggLoop = Expressions.ifThen(hasRows, forAggLoop);
+ }
+ builder5.add(forAggLoop);
+ }
+
+ if (implementResult(aggs, builder5, resultContextBuilder, rexArguments,
+ true)) {
+ builder4.add(
+ Expressions.ifThen(
+ Expressions.orElse(lowerBoundCanChange,
+ Expressions.notEqual(endX, prevEnd)),
+ builder5.toBlock()));
+ }
+
+ implementResult(aggs, builder4, resultContextBuilder, rexArguments,
+ false);
+
+ builder4.add(
+ Expressions.statement(
+ Expressions.call(
+ list_,
+ BuiltInMethod.COLLECTION_ADD.method,
+ outputPhysType.record(outputRow))));
+
+ builder3.add(
+ Expressions.for_(
+ Expressions.declare(0, i_, Expressions.constant(0)),
+ Expressions.lessThan(
+ i_,
+ Expressions.field(rows_, "length")),
+ Expressions.preIncrementAssign(i_),
+ builder4.toBlock()));
+
+ builder.add(
+ Expressions.while_(
+ Expressions.call(
+ iterator_,
+ BuiltInMethod.ITERATOR_HAS_NEXT.method),
+ builder3.toBlock()));
+ builder.add(
+ Expressions.statement(
+ Expressions.call(
+ collectionExpr,
+ BuiltInMethod.MAP_CLEAR.method)));
+
+ // We're not assigning to "source". For each group, create a new
+ // final variable called "source" or "sourceN".
+ source_ =
+ builder.append(
+ "source",
+ Expressions.call(
+ BuiltInMethod.AS_ENUMERABLE.method, list_));
+
+ inputPhysType = outputPhysType;
+ }
+
+ // return Linq4j.asEnumerable(list);
+ builder.add(
+ Expressions.return_(null, source_));
+ return implementor.result(inputPhysType, builder.toBlock());
+ }
+
+ private Function<BlockBuilder, WinAggFrameResultContext>
+ getBlockBuilderWinAggFrameResultContextFunction(
+ final JavaTypeFactory typeFactory, final Result result,
+ final List<Expression> translatedConstants,
+ final Expression comparator_,
+ final Expression rows_, final ParameterExpression i_,
+ final Expression startX, final Expression endX,
+ final Expression minX, final Expression maxX,
+ final Expression hasRows, final Expression frameRowCount,
+ final Expression partitionRowCount,
+ final DeclarationStatement jDecl,
+ final PhysType inputPhysType) {
+ return new Function<BlockBuilder,
+ WinAggFrameResultContext>() {
+ public WinAggFrameResultContext apply(
+ final BlockBuilder block) {
+ return new WinAggFrameResultContext() {
+ public RexToLixTranslator rowTranslator(Expression rowIndex) {
+ Expression row =
+ getRow(rowIndex);
+ final RexToLixTranslator.InputGetter inputGetter =
+ new WindowRelInputGetter(row, inputPhysType,
+ result.physType.getRowType().getFieldCount(),
+ translatedConstants);
+
+ return RexToLixTranslator.forAggregation(typeFactory,
+ block, inputGetter);
+ }
+
+ public Expression computeIndex(Expression offset,
+ WinAggImplementor.SeekType seekType) {
+ Expression index;
+ if (seekType == WinAggImplementor.SeekType.AGG_INDEX) {
+ index = jDecl.parameter;
+ } else if (seekType == WinAggImplementor.SeekType.SET) {
+ index = i_;
+ } else if (seekType == WinAggImplementor.SeekType.START) {
+ index = startX;
+ } else if (seekType == WinAggImplementor.SeekType.END) {
+ index = endX;
+ } else {
+ throw new IllegalArgumentException("SeekSet " + seekType
+ + " is not supported");
+ }
+ if (!Expressions.constant(0).equals(offset)) {
+ index = block.append("idx", Expressions.add(index, offset));
+ }
+ return index;
+ }
+
+ private Expression checkBounds(Expression rowIndex,
+ Expression minIndex, Expression maxIndex) {
+ if (rowIndex == i_ || rowIndex == startX || rowIndex == endX) {
+ // No additional bounds check required
+ return hasRows;
+ }
+
+ //noinspection UnnecessaryLocalVariable
+ Expression res = block.append("rowInFrame",
+ Expressions.foldAnd(
+ ImmutableList.of(hasRows,
+ Expressions.greaterThanOrEqual(rowIndex, minIndex),
+ Expressions.lessThanOrEqual(rowIndex, maxIndex))));
+
+ return res;
+ }
+
+ public Expression rowInFrame(Expression rowIndex) {
+ return checkBounds(rowIndex, startX, endX);
+ }
+
+ public Expression rowInPartition(Expression rowIndex) {
+ return checkBounds(rowIndex, minX, maxX);
+ }
+
+ public Expression compareRows(Expression a, Expression b) {
+ return Expressions.call(comparator_,
+ BuiltInMethod.COMPARATOR_COMPARE.method,
+ getRow(a), getRow(b));
+ }
+
+ public Expression getRow(Expression rowIndex) {
+ return block.append(
+ "jRow",
+ RexToLixTranslator.convert(
+ Expressions.arrayIndex(rows_, rowIndex),
+ inputPhysType.getJavaRowType()));
+ }
+
+ public Expression index() {
+ return i_;
+ }
+
+ public Expression startIndex() {
+ return startX;
+ }
+
+ public Expression endIndex() {
+ return endX;
+ }
+
+ public Expression hasRows() {
+ return hasRows;
+ }
+
+ public Expression getFrameRowCount() {
+ return frameRowCount;
+ }
+
+ public Expression getPartitionRowCount() {
+ return partitionRowCount;
+ }
+ };
+ }
+ };
+ }
+
+ private Pair<Expression, Expression> getPartitionIterator(
+ BlockBuilder builder,
+ Expression source_,
+ PhysType inputPhysType,
+ Group group,
+ Expression comparator_) {
+ // Populate map of lists, one per partition
+ // final Map<Integer, List<Employee>> multiMap =
+ // new SortedMultiMap<Integer, List<Employee>>();
+ // source.foreach(
+ // new Function1<Employee, Void>() {
+ // public Void apply(Employee v) {
+ // final Integer k = v.deptno;
+ // multiMap.putMulti(k, v);
+ // return null;
+ // }
+ // });
+ // final List<Xxx> list = new ArrayList<Xxx>(multiMap.size());
+ // Iterator<Employee[]> iterator = multiMap.arrays(comparator);
+ //
+ if (group.keys.isEmpty()) {
+ // If partition key is empty, no need to partition.
+ //
+ // final List<Employee> tempList =
+ // source.into(new ArrayList<Employee>());
+ // Iterator<Employee[]> iterator =
+ // SortedMultiMap.singletonArrayIterator(comparator, tempList);
+ // final List<Xxx> list = new ArrayList<Xxx>(tempList.size());
+
+ final Expression tempList_ = builder.append(
+ "tempList",
+ Expressions.convert_(
+ Expressions.call(
+ source_,
+ BuiltInMethod.INTO.method,
+ Expressions.new_(ArrayList.class)),
+ List.class));
+ return Pair.of(tempList_,
+ builder.append(
+ "iterator",
+ Expressions.call(
+ null,
+ BuiltInMethod.SORTED_MULTI_MAP_SINGLETON.method,
+ comparator_,
+ tempList_)));
+ }
+ Expression multiMap_ =
+ builder.append(
+ "multiMap", Expressions.new_(SortedMultiMap.class));
+ final BlockBuilder builder2 = new BlockBuilder();
+ final ParameterExpression v_ =
+ Expressions.parameter(inputPhysType.getJavaRowType(),
+ builder2.newName("v"));
+
+ Pair<Type, List<Expression>> selector = inputPhysType.selector(
+ v_,
+ group.keys.asList(),
+ JavaRowFormat.CUSTOM);
+ final ParameterExpression key_;
+ if(selector.left instanceof Types.RecordType) {
+ Types.RecordType keyJavaType = (Types.RecordType) selector.left;
+ List<Expression> initExpressions = selector.right;
+
+ key_ = Expressions.parameter(keyJavaType, "key");
+ builder2.add(Expressions.declare(0, key_, null));
+ builder2.add(Expressions.statement(Expressions.assign(key_, Expressions.new_(keyJavaType))));
+ List<Types.RecordField> fieldList = keyJavaType.getRecordFields();
+ for (int i = 0; i < initExpressions.size(); i++) {
+ Expression right = initExpressions.get(i);
+ builder2.add(
+ Expressions.statement(
+ Expressions.assign(
+ Expressions.field(key_, fieldList.get(i)), right)));
+ }
+ }
+ else
+ {
+ DeclarationStatement declare = Expressions.declare(0, "key", selector.right.get(0));
+ builder2.add(declare);
+ key_ = declare.parameter;
+ }
+ builder2.add(
+ Expressions.statement(
+ Expressions.call(
+ multiMap_,
+ BuiltInMethod.SORTED_MULTI_MAP_PUT_MULTI.method,
+ key_,
+ v_)));
+ builder2.add(
+ Expressions.return_(
+ null, Expressions.constant(null)));
+
+ builder.add(
+ Expressions.statement(
+ Expressions.call(
+ source_,
+ BuiltInMethod.ENUMERABLE_FOREACH.method,
+ Expressions.lambda(
+ builder2.toBlock(), v_))));
+
+ return Pair.of(multiMap_,
+ builder.append(
+ "iterator",
+ Expressions.call(
+ multiMap_,
+ BuiltInMethod.SORTED_MULTI_MAP_ARRAYS.method,
+ comparator_)));
+ }
+
+ private Pair<Expression, Expression> getRowCollationKey(
+ BlockBuilder builder, PhysType inputPhysType,
+ Group group, int windowIdx) {
+ if (!(group.isRows || (group.upperBound.isUnbounded()
+ && group.lowerBound.isUnbounded()))) {
+ Pair<Expression, Expression> pair =
+ inputPhysType.generateCollationKey(
+ group.collation().getFieldCollations());
+ // optimize=false to prevent inlining of object create into for-loops
+ return Pair.of(
+ builder.append("keySelector" + windowIdx, pair.left, false),
+ builder.append("keyComparator" + windowIdx, pair.right, false));
+ } else {
+ return Pair.of(null, null);
+ }
+ }
+
+ private void declareAndResetState(final JavaTypeFactory typeFactory,
+ BlockBuilder builder, final Result result, int windowIdx,
+ List<AggImpState> aggs, PhysType outputPhysType,
+ List<Expression> outputRow) {
+ for (final AggImpState agg : aggs) {
+ agg.context =
+ new WinAggContext() {
+ public SqlAggFunction aggregation() {
+ return agg.call.getAggregation();
+ }
+
+ public RelDataType returnRelType() {
+ return agg.call.type;
+ }
+
+ public Type returnType() {
+ return EnumUtils.javaClass(typeFactory, returnRelType());
+ }
+
+ public List<? extends Type> parameterTypes() {
+ return EnumUtils.fieldTypes(typeFactory,
+ parameterRelTypes());
+ }
+
+ public List<? extends RelDataType> parameterRelTypes() {
+ return EnumUtils.fieldRowTypes(result.physType.getRowType(),
+ constants, agg.call.getArgList());
+ }
+ };
+ String aggName = "a" + agg.aggIdx;
+ if (CalcitePrepareImpl.DEBUG) {
+ aggName = Util.toJavaId(agg.call.getAggregation().getName(), 0)
+ .substring("ID$0$".length()) + aggName;
+ }
+ List<Type> state = agg.implementor.getStateType(agg.context);
+ final List<Expression> decls =
+ new ArrayList<Expression>(state.size());
+ for (int i = 0; i < state.size(); i++) {
+ Type type = state.get(i);
+ ParameterExpression pe =
+ Expressions.parameter(type,
+ builder.newName(aggName
+ + "s" + i + "w" + windowIdx));
+ builder.add(Expressions.declare(0, pe, null));
+ decls.add(pe);
+ }
+ agg.state = decls;
+ Type aggHolderType = agg.context.returnType();
+ Type aggStorageType =
+ outputPhysType.getJavaFieldType(outputRow.size());
+ if (Primitive.is(aggHolderType) && !Primitive.is(aggStorageType)) {
+ aggHolderType = Primitive.box(aggHolderType);
+ }
+ ParameterExpression aggRes = Expressions.parameter(0,
+ aggHolderType,
+ builder.newName(aggName + "w" + windowIdx));
+
+ builder.add(
+ Expressions.declare(0, aggRes,
+ Expressions.constant(Primitive.is(aggRes.getType())
+ ? Primitive.of(aggRes.getType()).defaultValue
+ : null,
+ aggRes.getType())));
+ agg.result = aggRes;
+ outputRow.add(aggRes);
+ agg.implementor.implementReset(agg.context,
+ new WinAggResetContextImpl(builder, agg.state,
+ null, null, null, null, null, null));
+ }
+ }
+
+ private void implementAdd(List<AggImpState> aggs,
+ final BlockBuilder builder7,
+ final Function<BlockBuilder, WinAggFrameResultContext> frame,
+ final Function<AggImpState, List<RexNode>> rexArguments,
+ final DeclarationStatement jDecl) {
+ for (final AggImpState agg : aggs) {
+ final WinAggAddContext addContext =
+ new WinAggAddContextImpl(builder7, agg.state, frame) {
+ public Expression currentPosition() {
+ return jDecl.parameter;
+ }
+
+ public List<RexNode> rexArguments() {
+ return rexArguments.apply(agg);
+ }
+
+ public RexNode rexFilterArgument() {
+ return null; // REVIEW
+ }
+ };
+ agg.implementor.implementAdd(agg.context, addContext);
+ }
+ }
+
+ private boolean implementResult(List<AggImpState> aggs,
+ final BlockBuilder builder,
+ final Function<BlockBuilder, WinAggFrameResultContext> frame,
+ final Function<AggImpState, List<RexNode>> rexArguments,
+ boolean cachedBlock) {
+ boolean nonEmpty = false;
+ for (final AggImpState agg : aggs) {
+ boolean needCache = true;
+ if (agg.implementor instanceof WinAggImplementor) {
+ WinAggImplementor imp = (WinAggImplementor) agg.implementor;
+ needCache = imp.needCacheWhenFrameIntact();
+ }
+ if (needCache ^ cachedBlock) {
+ // Regular aggregates do not change when the windowing frame keeps
+ // the same. Ths
+ continue;
+ }
+ nonEmpty = true;
+ Expression res = agg.implementor.implementResult(agg.context,
+ new WinAggResultContextImpl(builder, agg.state, frame) {
+ public List<RexNode> rexArguments() {
+ return rexArguments.apply(agg);
+ }
+ });
+ // Several count(a) and count(b) might share the result
+ Expression aggRes = builder.append("a" + agg.aggIdx + "res",
+ RexToLixTranslator.convert(res, agg.result.getType()));
+ builder.add(
+ Expressions.statement(Expressions.assign(agg.result, aggRes)));
+ }
+ return nonEmpty;
+ }
+
+ private Expression translateBound(RexToLixTranslator translator,
+ ParameterExpression i_, Expression row_, Expression min_,
+ Expression max_, Expression rows_, Group group,
+ boolean lower,
+ PhysType physType, Expression rowComparator,
+ Expression keySelector, Expression keyComparator) {
+ RexWindowBound bound = lower ? group.lowerBound : group.upperBound;
+ if (bound.isUnbounded()) {
+ return bound.isPreceding() ? min_ : max_;
+ }
+ if (group.isRows) {
+ if (bound.isCurrentRow()) {
+ return i_;
+ }
+ RexNode node = bound.getOffset();
+ Expression offs = translator.translate(node);
+ // Floating offset does not make sense since we refer to array index.
+ // Nulls do not make sense as well.
+ offs = RexToLixTranslator.convert(offs, int.class);
+
+ Expression b = i_;
+ if (bound.isFollowing()) {
+ b = Expressions.add(b, offs);
+ } else {
+ b = Expressions.subtract(b, offs);
+ }
+ return b;
+ }
+ Expression searchLower = min_;
+ Expression searchUpper = max_;
+ if (bound.isCurrentRow()) {
+ if (lower) {
+ searchUpper = i_;
+ } else {
+ searchLower = i_;
+ }
+ }
+
+ List<RelFieldCollation> fieldCollations =
+ group.collation().getFieldCollations();
+ if (bound.isCurrentRow() && fieldCollations.size() != 1) {
+ return Expressions.call(
+ (lower
+ ? BuiltInMethod.BINARY_SEARCH5_LOWER
+ : BuiltInMethod.BINARY_SEARCH5_UPPER).method,
+ rows_, row_, searchLower, searchUpper, keySelector, keyComparator);
+ }
+ assert fieldCollations.size() == 1
+ : "When using range window specification, ORDER BY should have"
+ + " exactly one expression."
+ + " Actual collation is " + group.collation();
+ // isRange
+ int orderKey =
+ fieldCollations.get(0).getFieldIndex();
+ RelDataType keyType =
+ physType.getRowType().getFieldList().get(orderKey).getType();
+ Type desiredKeyType = translator.typeFactory.getJavaClass(keyType);
+ if (bound.getOffset() == null) {
+ desiredKeyType = Primitive.box(desiredKeyType);
+ }
+ Expression val = translator.translate(
+ new RexInputRef(orderKey, keyType), desiredKeyType);
+ if (!bound.isCurrentRow()) {
+ RexNode node = bound.getOffset();
+ Expression offs = translator.translate(node);
+ // TODO: support date + interval somehow
+ if (bound.isFollowing()) {
+ val = Expressions.add(val, offs);
+ } else {
+ val = Expressions.subtract(val, offs);
+ }
+ }
+ return Expressions.call(
+ (lower
+ ? BuiltInMethod.BINARY_SEARCH6_LOWER
+ : BuiltInMethod.BINARY_SEARCH6_UPPER).method,
+ rows_, val, searchLower, searchUpper, keySelector, keyComparator);
+ }
+}
+
+// End EnumerableWindow.java
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae4333c/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java
new file mode 100644
index 0000000..e37b196
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/PhysType.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.enumerable;
+
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.linq4j.tree.ParameterExpression;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.util.Pair;
+
+import java.lang.reflect.Type;
+import java.util.List;
+
+/*
+ * OVERRIDE POINT: patching CALCITE-1540 on calcite 1.8.0
+ */
+
+/**
+ * Physical type of a row.
+ *
+ * <p>Consists of the SQL row type (returned by {@link #getRowType()}), the Java
+ * type of the row (returned by {@link #getJavaRowType()}), and methods to
+ * generate expressions to access fields, generate records, and so forth.
+ * Together, the records encapsulate how the logical type maps onto the physical
+ * type.</p>
+ */
+public interface PhysType {
+ /** Returns the Java type (often a Class) that represents a row. For
+ * example, in one row format, always returns {@code Object[].class}. */
+ Type getJavaRowType();
+
+ /**
+ * Returns the Java class that is used to store the field with the given
+ * ordinal.
+ *
+ * <p>For instance, when the java row type is {@code Object[]}, the java
+ * field type is {@code Object} even if the field is not nullable.</p> */
+ Type getJavaFieldType(int field);
+
+ /** Returns the physical type of a field. */
+ PhysType field(int ordinal);
+
+ /** Returns the physical type of a given field's component type. */
+ PhysType component(int field);
+
+ /** Returns the SQL row type. */
+ RelDataType getRowType();
+
+ /** Returns the Java class of the field with the given ordinal. */
+ Class fieldClass(int field);
+
+ /** Returns whether a given field allows null values. */
+ boolean fieldNullable(int index);
+
+ /** Generates a reference to a given field in an expression.
+ *
+ * <p>For example given {@code expression=employee} and {@code field=2},
+ * generates</p>
+ *
+ * <pre>{@code employee.deptno}</pre>
+ *
+ * @param expression Expression
+ * @param field Ordinal of field
+ * @return Expression to access the field of the expression
+ */
+ Expression fieldReference(Expression expression, int field);
+
+ /** Generates a reference to a given field in an expression.
+ *
+ * <p>This method optimizes for the target storage type (i.e. avoids
+ * casts).</p>
+ *
+ * <p>For example given {@code expression=employee} and {@code field=2},
+ * generates</p>
+ *
+ * <pre>{@code employee.deptno}</pre>
+ *
+ * @param expression Expression
+ * @param field Ordinal of field
+ * @param storageType optional hint for storage class
+ * @return Expression to access the field of the expression
+ */
+ Expression fieldReference(Expression expression, int field,
+ Type storageType);
+
+ /** Generates an accessor function for a given list of fields. The resulting
+ * object is a {@link List} (implementing {@link Object#hashCode()} and
+ * {@link Object#equals(Object)} per that interface) and also implements
+ * {@link Comparable}.
+ *
+ * <p>For example:</p>
+ *
+ * <pre>{@code
+ * new Function1<Employee, Object[]> {
+ * public Object[] apply(Employee v1) {
+ * return FlatLists.of(v1.<fieldN>, v1.<fieldM>);
+ * }
+ * }
+ * }</pre>
+ */
+ Expression generateAccessor(List<Integer> fields);
+
+ /** Generates a selector for the given fields from an expression, with the
+ * default row format. */
+ Expression generateSelector(
+ ParameterExpression parameter,
+ List<Integer> fields);
+
+ /** Generates a lambda expression that is a selector for the given fields from
+ * an expression. */
+ Expression generateSelector(
+ ParameterExpression parameter,
+ List<Integer> fields,
+ JavaRowFormat targetFormat);
+
+ /** Generates a lambda expression that is a selector for the given fields from
+ * an expression.
+ *
+ * <p>{@code usedFields} must be a subset of {@code fields}.
+ * For each field, there is a corresponding indicator field.
+ * If a field is used, its value is assigned and its indicator is left
+ * {@code false}.
+ * If a field is not used, its value is not assigned and its indicator is
+ * set to {@code true};
+ * This will become a value of 1 when {@code GROUPING(field)} is called. */
+ Expression generateSelector(
+ ParameterExpression parameter,
+ List<Integer> fields,
+ List<Integer> usedFields,
+ JavaRowFormat targetFormat);
+
+ /** Generates a selector for the given fields from an expression. */
+ /** Only used by EnumerableWindow */
+ Pair<Type, List<Expression>> selector(
+ ParameterExpression parameter,
+ List<Integer> fields,
+ JavaRowFormat targetFormat);
+
+ /** Projects a given collection of fields from this input record, into
+ * a particular preferred output format. The output format is optimized
+ * if there are 0 or 1 fields. */
+ PhysType project(
+ List<Integer> integers,
+ JavaRowFormat format);
+
+ /** Projects a given collection of fields from this input record, optionally
+ * with indicator fields, into a particular preferred output format.
+ *
+ * <p>The output format is optimized if there are 0 or 1 fields
+ * and indicators are disabled. */
+ PhysType project(
+ List<Integer> integers,
+ boolean indicator,
+ JavaRowFormat format);
+
+ /** Returns a lambda to create a collation key and a comparator. The
+ * comparator is sometimes null. */
+ Pair<Expression, Expression> generateCollationKey(
+ List<RelFieldCollation> collations);
+
+ /** Returns a comparator. Unlike the comparator returned by
+ * {@link #generateCollationKey(java.util.List)}, this comparator acts on the
+ * whole element. */
+ Expression generateComparator(
+ RelCollation collation);
+
+ /** Returns a expression that yields a comparer, or null if this type
+ * is comparable. */
+ Expression comparer();
+
+ /** Generates an expression that creates a record for a row, initializing
+ * its fields with the given expressions. There must be one expression per
+ * field.
+ *
+ * @param expressions Expression to initialize each field
+ * @return Expression to create a row
+ */
+ Expression record(List<Expression> expressions);
+
+ /** Returns the format. */
+ JavaRowFormat getFormat();
+
+ List<Expression> accessors(Expression parameter, List<Integer> argList);
+
+ /** Returns a copy of this type that allows nulls if {@code nullable} is
+ * true. */
+ PhysType makeNullable(boolean nullable);
+
+ /** Converts an enumerable of this physical type to an enumerable that uses a
+ * given physical type for its rows. */
+ Expression convertTo(Expression expression, PhysType targetPhysType);
+}
+
+// End PhysType.java
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae4333c/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java
new file mode 100644
index 0000000..678b469
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/PhysTypeImpl.java
@@ -0,0 +1,654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.enumerable;
+
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.linq4j.function.Function1;
+import org.apache.calcite.linq4j.tree.BlockBuilder;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.linq4j.tree.MemberDeclaration;
+import org.apache.calcite.linq4j.tree.ParameterExpression;
+import org.apache.calcite.linq4j.tree.Primitive;
+import org.apache.calcite.linq4j.tree.Types;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.runtime.Utilities;
+import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.BuiltInMethod;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.Type;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import static org.apache.calcite.adapter.enumerable.EnumUtils.javaRowClass;
+import static org.apache.calcite.adapter.enumerable.EnumUtils.overridingMethodDecl;
+
+/*
+ * OVERRIDE POINT: patching CALCITE-1540 on calcite 1.8.0
+ */
+
+/** Implementation of {@link PhysType}. */
+public class PhysTypeImpl implements PhysType {
+ private final JavaTypeFactory typeFactory;
+ private final RelDataType rowType;
+ private final Type javaRowClass;
+ private final List<Class> fieldClasses = new ArrayList<>();
+ final JavaRowFormat format;
+
+ /** Creates a PhysTypeImpl. */
+ PhysTypeImpl(
+ JavaTypeFactory typeFactory,
+ RelDataType rowType,
+ Type javaRowClass,
+ JavaRowFormat format) {
+ this.typeFactory = typeFactory;
+ this.rowType = rowType;
+ this.javaRowClass = javaRowClass;
+ this.format = format;
+ for (RelDataTypeField field : rowType.getFieldList()) {
+ fieldClasses.add(javaRowClass(typeFactory, field.getType()));
+ }
+ }
+
+ public static PhysType of(
+ JavaTypeFactory typeFactory,
+ RelDataType rowType,
+ JavaRowFormat format) {
+ return of(typeFactory, rowType, format, true);
+ }
+
+ public static PhysType of(
+ JavaTypeFactory typeFactory,
+ RelDataType rowType,
+ JavaRowFormat format,
+ boolean optimize) {
+ if (optimize) {
+ format = format.optimize(rowType);
+ }
+ final Type javaRowClass = format.javaRowClass(typeFactory, rowType);
+ return new PhysTypeImpl(typeFactory, rowType, javaRowClass, format);
+ }
+
+ static PhysType of(
+ final JavaTypeFactory typeFactory,
+ Type javaRowClass) {
+ final RelDataTypeFactory.FieldInfoBuilder builder = typeFactory.builder();
+ if (javaRowClass instanceof Types.RecordType) {
+ final Types.RecordType recordType = (Types.RecordType) javaRowClass;
+ for (Types.RecordField field : recordType.getRecordFields()) {
+ builder.add(field.getName(), typeFactory.createType(field.getType()));
+ }
+ }
+ RelDataType rowType = builder.build();
+ // Do not optimize if there are 0 or 1 fields.
+ return new PhysTypeImpl(typeFactory, rowType, javaRowClass,
+ JavaRowFormat.CUSTOM);
+ }
+
+ public JavaRowFormat getFormat() {
+ return format;
+ }
+
+ public PhysType project(List<Integer> integers, JavaRowFormat format) {
+ return project(integers, false, format);
+ }
+
+ public PhysType project(List<Integer> integers, boolean indicator,
+ JavaRowFormat format) {
+ final RelDataTypeFactory.FieldInfoBuilder builder = typeFactory.builder();
+ for (int index : integers) {
+ builder.add(rowType.getFieldList().get(index));
+ }
+ if (indicator) {
+ final RelDataType booleanType =
+ typeFactory.createTypeWithNullability(
+ typeFactory.createSqlType(SqlTypeName.BOOLEAN), false);
+ for (int index : integers) {
+ builder.add("i$" + rowType.getFieldList().get(index).getName(),
+ booleanType);
+ }
+ }
+ RelDataType projectedRowType = builder.build();
+ return of(typeFactory, projectedRowType, format.optimize(projectedRowType));
+ }
+
+ public Expression generateSelector(
+ ParameterExpression parameter,
+ List<Integer> fields) {
+ return generateSelector(parameter, fields, format);
+ }
+
+ public Expression generateSelector(
+ ParameterExpression parameter,
+ List<Integer> fields,
+ JavaRowFormat targetFormat) {
+ // Optimize target format
+ switch (fields.size()) {
+ case 0:
+ targetFormat = JavaRowFormat.LIST;
+ break;
+ case 1:
+ targetFormat = JavaRowFormat.SCALAR;
+ break;
+ }
+ final PhysType targetPhysType =
+ project(fields, targetFormat);
+ switch (format) {
+ case SCALAR:
+ return Expressions.call(BuiltInMethod.IDENTITY_SELECTOR.method);
+ default:
+ return Expressions.lambda(Function1.class,
+ targetPhysType.record(fieldReferences(parameter, fields)), parameter);
+ }
+ }
+
+ public Expression generateSelector(final ParameterExpression parameter,
+ final List<Integer> fields, List<Integer> usedFields,
+ JavaRowFormat targetFormat) {
+ final PhysType targetPhysType =
+ project(fields, true, targetFormat);
+ final List<Expression> expressions = Lists.newArrayList();
+ for (Ord<Integer> ord : Ord.zip(fields)) {
+ final Integer field = ord.e;
+ if (usedFields.contains(field)) {
+ expressions.add(fieldReference(parameter, field));
+ } else {
+ final Primitive primitive =
+ Primitive.of(targetPhysType.fieldClass(ord.i));
+ expressions.add(
+ Expressions.constant(
+ primitive != null ? primitive.defaultValue : null));
+ }
+ }
+ for (Integer field : fields) {
+ expressions.add(Expressions.constant(!usedFields.contains(field)));
+ }
+ return Expressions.lambda(Function1.class,
+ targetPhysType.record(expressions), parameter);
+ }
+
+ public Pair<Type, List<Expression>> selector(
+ ParameterExpression parameter,
+ List<Integer> fields,
+ JavaRowFormat targetFormat) {
+ // Optimize target format
+ switch (fields.size()) {
+ case 0:
+ targetFormat = JavaRowFormat.LIST;
+ break;
+ case 1:
+ targetFormat = JavaRowFormat.SCALAR;
+ break;
+ }
+ final PhysType targetPhysType =
+ project(fields, targetFormat);
+ switch (format) {
+ case SCALAR:
+ return Pair.of(parameter.getType(), Collections.<Expression>singletonList(parameter));
+ default:
+ return Pair.of(targetPhysType.getJavaRowType(), fieldReferences(parameter, fields));
+ }
+ }
+
+ public List<Expression> accessors(Expression v1, List<Integer> argList) {
+ final List<Expression> expressions = new ArrayList<>();
+ for (int field : argList) {
+ expressions.add(
+ Types.castIfNecessary(
+ fieldClass(field),
+ fieldReference(v1, field)));
+ }
+ return expressions;
+ }
+
+ public PhysType makeNullable(boolean nullable) {
+ if (!nullable) {
+ return this;
+ }
+ return new PhysTypeImpl(typeFactory,
+ typeFactory.createTypeWithNullability(rowType, true),
+ Primitive.box(javaRowClass), format);
+ }
+
+ public Expression convertTo(Expression exp, PhysType targetPhysType) {
+ final JavaRowFormat targetFormat = targetPhysType.getFormat();
+ if (format == targetFormat) {
+ return exp;
+ }
+ final ParameterExpression o_ =
+ Expressions.parameter(javaRowClass, "o");
+ final int fieldCount = rowType.getFieldCount();
+ return Expressions.call(exp, BuiltInMethod.SELECT.method,
+ generateSelector(o_, Util.range(fieldCount), targetFormat));
+ }
+
+ public Pair<Expression, Expression> generateCollationKey(
+ final List<RelFieldCollation> collations) {
+ final Expression selector;
+ if (collations.size() == 1) {
+ RelFieldCollation collation = collations.get(0);
+ ParameterExpression parameter =
+ Expressions.parameter(javaRowClass, "v");
+ selector =
+ Expressions.lambda(
+ Function1.class,
+ fieldReference(parameter, collation.getFieldIndex()),
+ parameter);
+ return Pair.<Expression, Expression>of(
+ selector,
+ Expressions.call(
+ BuiltInMethod.NULLS_COMPARATOR.method,
+ Expressions.constant(
+ collation.nullDirection
+ == RelFieldCollation.NullDirection.FIRST),
+ Expressions.constant(
+ collation.getDirection()
+ == RelFieldCollation.Direction.DESCENDING)));
+ }
+ selector =
+ Expressions.call(BuiltInMethod.IDENTITY_SELECTOR.method);
+
+ // int c;
+ // c = Utilities.compare(v0, v1);
+ // if (c != 0) return c; // or -c if descending
+ // ...
+ // return 0;
+ BlockBuilder body = new BlockBuilder();
+ final ParameterExpression parameterV0 =
+ Expressions.parameter(javaRowClass, "v0");
+ final ParameterExpression parameterV1 =
+ Expressions.parameter(javaRowClass, "v1");
+ final ParameterExpression parameterC =
+ Expressions.parameter(int.class, "c");
+ final int mod = collations.size() == 1 ? Modifier.FINAL : 0;
+ body.add(Expressions.declare(mod, parameterC, null));
+ for (RelFieldCollation collation : collations) {
+ final int index = collation.getFieldIndex();
+ Expression arg0 = fieldReference(parameterV0, index);
+ Expression arg1 = fieldReference(parameterV1, index);
+ switch (Primitive.flavor(fieldClass(index))) {
+ case OBJECT:
+ arg0 = Types.castIfNecessary(Comparable.class, arg0);
+ arg1 = Types.castIfNecessary(Comparable.class, arg1);
+ }
+ final boolean nullsFirst =
+ collation.nullDirection
+ == RelFieldCollation.NullDirection.FIRST;
+ final boolean descending =
+ collation.getDirection()
+ == RelFieldCollation.Direction.DESCENDING;
+ final Method method = (fieldNullable(index)
+ ? (nullsFirst ^ descending
+ ? BuiltInMethod.COMPARE_NULLS_FIRST
+ : BuiltInMethod.COMPARE_NULLS_LAST)
+ : BuiltInMethod.COMPARE).method;
+ body.add(
+ Expressions.statement(
+ Expressions.assign(
+ parameterC,
+ Expressions.call(method.getDeclaringClass(),
+ method.getName(),
+ arg0,
+ arg1))));
+ body.add(
+ Expressions.ifThen(
+ Expressions.notEqual(
+ parameterC, Expressions.constant(0)),
+ Expressions.return_(
+ null,
+ descending
+ ? Expressions.negate(parameterC)
+ : parameterC)));
+ }
+ body.add(
+ Expressions.return_(null, Expressions.constant(0)));
+
+ final List<MemberDeclaration> memberDeclarations =
+ Expressions.<MemberDeclaration>list(
+ Expressions.methodDecl(
+ Modifier.PUBLIC,
+ int.class,
+ "compare",
+ ImmutableList.of(
+ parameterV0, parameterV1),
+ body.toBlock()));
+
+ if (EnumerableRules.BRIDGE_METHODS) {
+ final ParameterExpression parameterO0 =
+ Expressions.parameter(Object.class, "o0");
+ final ParameterExpression parameterO1 =
+ Expressions.parameter(Object.class, "o1");
+ BlockBuilder bridgeBody = new BlockBuilder();
+ bridgeBody.add(
+ Expressions.return_(
+ null,
+ Expressions.call(
+ Expressions.parameter(
+ Comparable.class, "this"),
+ BuiltInMethod.COMPARATOR_COMPARE.method,
+ Expressions.convert_(
+ parameterO0,
+ javaRowClass),
+ Expressions.convert_(
+ parameterO1,
+ javaRowClass))));
+ memberDeclarations.add(
+ overridingMethodDecl(
+ BuiltInMethod.COMPARATOR_COMPARE.method,
+ ImmutableList.of(parameterO0, parameterO1),
+ bridgeBody.toBlock()));
+ }
+ return Pair.<Expression, Expression>of(
+ selector,
+ Expressions.new_(
+ Comparator.class,
+ Collections.<Expression>emptyList(),
+ memberDeclarations));
+ }
+
+ public Expression generateComparator(RelCollation collation) {
+ // int c;
+ // c = Utilities.compare(v0, v1);
+ // if (c != 0) return c; // or -c if descending
+ // ...
+ // return 0;
+ BlockBuilder body = new BlockBuilder();
+ final Type javaRowClass = Primitive.box(this.javaRowClass);
+ final ParameterExpression parameterV0 =
+ Expressions.parameter(javaRowClass, "v0");
+ final ParameterExpression parameterV1 =
+ Expressions.parameter(javaRowClass, "v1");
+ final ParameterExpression parameterC =
+ Expressions.parameter(int.class, "c");
+ final int mod =
+ collation.getFieldCollations().size() == 1 ? Modifier.FINAL : 0;
+ body.add(Expressions.declare(mod, parameterC, null));
+ for (RelFieldCollation fieldCollation : collation.getFieldCollations()) {
+ final int index = fieldCollation.getFieldIndex();
+ Expression arg0 = fieldReference(parameterV0, index);
+ Expression arg1 = fieldReference(parameterV1, index);
+ switch (Primitive.flavor(fieldClass(index))) {
+ case OBJECT:
+ arg0 = Types.castIfNecessary(Comparable.class, arg0);
+ arg1 = Types.castIfNecessary(Comparable.class, arg1);
+ }
+ final boolean nullsFirst =
+ fieldCollation.nullDirection
+ == RelFieldCollation.NullDirection.FIRST;
+ final boolean descending =
+ fieldCollation.getDirection()
+ == RelFieldCollation.Direction.DESCENDING;
+ body.add(
+ Expressions.statement(
+ Expressions.assign(
+ parameterC,
+ Expressions.call(
+ Utilities.class,
+ fieldNullable(index)
+ ? (nullsFirst != descending
+ ? "compareNullsFirst"
+ : "compareNullsLast")
+ : "compare",
+ arg0,
+ arg1))));
+ body.add(
+ Expressions.ifThen(
+ Expressions.notEqual(
+ parameterC, Expressions.constant(0)),
+ Expressions.return_(
+ null,
+ descending
+ ? Expressions.negate(parameterC)
+ : parameterC)));
+ }
+ body.add(
+ Expressions.return_(null, Expressions.constant(0)));
+
+ final List<MemberDeclaration> memberDeclarations =
+ Expressions.<MemberDeclaration>list(
+ Expressions.methodDecl(
+ Modifier.PUBLIC,
+ int.class,
+ "compare",
+ ImmutableList.of(parameterV0, parameterV1),
+ body.toBlock()));
+
+ if (EnumerableRules.BRIDGE_METHODS) {
+ final ParameterExpression parameterO0 =
+ Expressions.parameter(Object.class, "o0");
+ final ParameterExpression parameterO1 =
+ Expressions.parameter(Object.class, "o1");
+ BlockBuilder bridgeBody = new BlockBuilder();
+ bridgeBody.add(
+ Expressions.return_(
+ null,
+ Expressions.call(
+ Expressions.parameter(
+ Comparable.class, "this"),
+ BuiltInMethod.COMPARATOR_COMPARE.method,
+ Expressions.convert_(
+ parameterO0,
+ javaRowClass),
+ Expressions.convert_(
+ parameterO1,
+ javaRowClass))));
+ memberDeclarations.add(
+ overridingMethodDecl(
+ BuiltInMethod.COMPARATOR_COMPARE.method,
+ ImmutableList.of(parameterO0, parameterO1),
+ bridgeBody.toBlock()));
+ }
+ return Expressions.new_(
+ Comparator.class,
+ Collections.<Expression>emptyList(),
+ memberDeclarations);
+ }
+
+ public RelDataType getRowType() {
+ return rowType;
+ }
+
+ public Expression record(List<Expression> expressions) {
+ return format.record(javaRowClass, expressions);
+ }
+
+ public Type getJavaRowType() {
+ return javaRowClass;
+ }
+
+ public Type getJavaFieldType(int index) {
+ return format.javaFieldClass(typeFactory, rowType, index);
+ }
+
+ public PhysType component(int fieldOrdinal) {
+ final RelDataTypeField field = rowType.getFieldList().get(fieldOrdinal);
+ return PhysTypeImpl.of(typeFactory,
+ toStruct(field.getType().getComponentType()), format, false);
+ }
+
+ public PhysType field(int ordinal) {
+ final RelDataTypeField field = rowType.getFieldList().get(ordinal);
+ final RelDataType type = field.getType();
+ return PhysTypeImpl.of(typeFactory, toStruct(type), format, false);
+ }
+
+ private RelDataType toStruct(RelDataType type) {
+ if (type.isStruct()) {
+ return type;
+ }
+ return typeFactory.builder()
+ .add(SqlUtil.deriveAliasFromOrdinal(0), type)
+ .build();
+ }
+
+ public Expression comparer() {
+ return format.comparer();
+ }
+
+ private List<Expression> fieldReferences(
+ final Expression parameter, final List<Integer> fields) {
+ return new AbstractList<Expression>() {
+ public Expression get(int index) {
+ return fieldReference(parameter, fields.get(index));
+ }
+
+ public int size() {
+ return fields.size();
+ }
+ };
+ }
+
+ public Class fieldClass(int field) {
+ return fieldClasses.get(field);
+ }
+
+ public boolean fieldNullable(int field) {
+ return rowType.getFieldList().get(field).getType().isNullable();
+ }
+
+ public Expression generateAccessor(
+ List<Integer> fields) {
+ ParameterExpression v1 =
+ Expressions.parameter(javaRowClass, "v1");
+ switch (fields.size()) {
+ case 0:
+ return Expressions.lambda(
+ Function1.class,
+ Expressions.field(
+ null,
+ BuiltInMethod.COMPARABLE_EMPTY_LIST.field),
+ v1);
+ case 1:
+ int field0 = fields.get(0);
+
+ // new Function1<Employee, Res> {
+ // public Res apply(Employee v1) {
+ // return v1.<fieldN>;
+ // }
+ // }
+ Class returnType = fieldClasses.get(field0);
+ Expression fieldReference =
+ Types.castIfNecessary(
+ returnType,
+ fieldReference(v1, field0));
+ return Expressions.lambda(
+ Function1.class,
+ fieldReference,
+ v1);
+ default:
+ // new Function1<Employee, List> {
+ // public List apply(Employee v1) {
+ // return Arrays.asList(
+ // new Object[] {v1.<fieldN>, v1.<fieldM>});
+ // }
+ // }
+ Expressions.FluentList<Expression> list = Expressions.list();
+ for (int field : fields) {
+ list.add(fieldReference(v1, field));
+ }
+ switch (list.size()) {
+ case 2:
+ return Expressions.lambda(
+ Function1.class,
+ Expressions.call(
+ List.class,
+ null,
+ BuiltInMethod.LIST2.method,
+ list),
+ v1);
+ case 3:
+ return Expressions.lambda(
+ Function1.class,
+ Expressions.call(
+ List.class,
+ null,
+ BuiltInMethod.LIST3.method,
+ list),
+ v1);
+ case 4:
+ return Expressions.lambda(
+ Function1.class,
+ Expressions.call(
+ List.class,
+ null,
+ BuiltInMethod.LIST4.method,
+ list),
+ v1);
+ case 5:
+ return Expressions.lambda(
+ Function1.class,
+ Expressions.call(
+ List.class,
+ null,
+ BuiltInMethod.LIST5.method,
+ list),
+ v1);
+ case 6:
+ return Expressions.lambda(
+ Function1.class,
+ Expressions.call(
+ List.class,
+ null,
+ BuiltInMethod.LIST6.method,
+ list),
+ v1);
+ default:
+ return Expressions.lambda(
+ Function1.class,
+ Expressions.call(
+ List.class,
+ null,
+ BuiltInMethod.LIST_N.method,
+ Expressions.newArrayInit(
+ Comparable.class,
+ list)),
+ v1);
+ }
+ }
+ }
+
+ public Expression fieldReference(
+ Expression expression, int field) {
+ return fieldReference(expression, field, null);
+ }
+
+ public Expression fieldReference(
+ Expression expression, int field, Type storageType) {
+ if (storageType == null) {
+ storageType = fieldClass(field);
+ }
+ return format.field(expression, field, storageType);
+ }
+}
+
+// End PhysTypeImpl.java
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae4333c/kylin-it/src/test/resources/query/sql_window/query11.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_window/query11.sql b/kylin-it/src/test/resources/query/sql_window/query11.sql
new file mode 100644
index 0000000..3002f4c
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_window/query11.sql
@@ -0,0 +1,23 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+select lstg_format_name,cal_dt,
+sum(sum(price)) over(partition by lstg_format_name,cal_dt),
+max(sum(price)) over(partition by lstg_format_name,cal_dt),
+min(sum(price)) over(partition by lstg_format_name)
+from test_kylin_fact
+group by cal_dt, lstg_format_name
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ae4333c/kylin-it/src/test/resources/query/sql_window/query12.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_window/query12.sql b/kylin-it/src/test/resources/query/sql_window/query12.sql
new file mode 100644
index 0000000..8073312
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_window/query12.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+select * from(
+ select cal_dt, lstg_format_name, sum(price) as GMV,
+ 100*sum(price)/first_value(sum(price)) over (partition by lstg_format_name,SLR_SEGMENT_CD order by cast(cal_dt as timestamp) range interval '1' day PRECEDING) as "last_day",
+ first_value(sum(price)) over (partition by lstg_format_name order by cast(cal_dt as timestamp) range cast(366 as INTERVAL day) preceding)
+ from test_kylin_fact as "last_year"
+ where cal_dt between '2013-01-08' and '2013-01-15' or cal_dt between '2013-01-07' and '2013-01-15' or cal_dt between '2012-01-01' and '2012-01-15'
+ group by cal_dt, lstg_format_name,SLR_SEGMENT_CD
+)t
+where cal_dt between '2013-01-06' and '2013-01-15'
[31/50] [abbrv] kylin git commit: fix sub_query/query02
Posted by li...@apache.org.
fix sub_query/query02
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e14f4e1e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e14f4e1e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e14f4e1e
Branch: refs/heads/master-cdh5.7
Commit: e14f4e1e837ecb02c4f49788567c0ed16d6347da
Parents: d79d937
Author: Yang Li <li...@apache.org>
Authored: Sun Dec 18 08:17:34 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Dec 18 08:17:34 2016 +0800
----------------------------------------------------------------------
kylin-it/src/test/resources/query/sql_subquery/query02.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e14f4e1e/kylin-it/src/test/resources/query/sql_subquery/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
index b0dcf73..342aa32 100644
--- a/kylin-it/src/test/resources/query/sql_subquery/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
@@ -30,6 +30,6 @@ FROM
inner JOIN test_category_groupings
ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
inner JOIN edw.test_sites as test_sites
- ON test_kylin_fact.lstg_site_id = test_sites.site_id inner join edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id
) t
group by week_beg_dt
[22/50] [abbrv] kylin git commit: minor sql fix, CI passed
Posted by li...@apache.org.
minor sql fix, CI passed
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/91cf788c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/91cf788c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/91cf788c
Branch: refs/heads/master-cdh5.7
Commit: 91cf788cebd66ffa8b56e37586fa3207c09f2d2e
Parents: 520b05f
Author: Yang Li <li...@apache.org>
Authored: Tue Dec 6 07:22:03 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 15 18:57:36 2016 +0800
----------------------------------------------------------------------
kylin-it/src/test/resources/query/sql_subquery/query02.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/91cf788c/kylin-it/src/test/resources/query/sql_subquery/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
index 81b4887..e6751b7 100644
--- a/kylin-it/src/test/resources/query/sql_subquery/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
@@ -18,7 +18,7 @@
SELECT
week_beg_dt
- ,sum(price)
+ ,sum(price) as sum_price
FROM
(
select
[30/50] [abbrv] kylin git commit: KYLIN-2291 Collect hive table
property skip-header-line-count
Posted by li...@apache.org.
KYLIN-2291 Collect hive table property skip-header-line-count
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d79d9374
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d79d9374
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d79d9374
Branch: refs/heads/master-cdh5.7
Commit: d79d9374cd8411518ee8eafc7640910826f6838c
Parents: f2377db
Author: Cheng Wang <ch...@kyligence.io>
Authored: Fri Dec 16 17:23:15 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 16 17:45:16 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/hive/BeelineHiveClient.java | 3 +++
.../java/org/apache/kylin/source/hive/CLIHiveClient.java | 2 +-
.../apache/kylin/source/hive/HiveSourceTableLoader.java | 1 +
.../java/org/apache/kylin/source/hive/HiveTableMeta.java | 4 +++-
.../apache/kylin/source/hive/HiveTableMetaBuilder.java | 11 ++++++++++-
5 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 47b551b..468ccb1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -193,6 +193,9 @@ public class BeelineHiveClient implements IHiveClient {
if ("numFiles".equals(resultSet.getString(2).trim())) {
builder.setFileNum(Long.parseLong(resultSet.getString(3).trim()));
}
+ if ("skip.header.line.count".equals(resultSet.getString(2).trim())) {
+ builder.setSkipHeaderLineCount(resultSet.getString(3).trim());
+ }
}
}
if ("InputFormat:".equals(resultSet.getString(1).trim())) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index 5a17f1f..e8a93bd 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -42,7 +42,6 @@ import com.google.common.collect.Lists;
*
*/
public class CLIHiveClient implements IHiveClient {
-
protected HiveConf hiveConf = null;
protected Driver driver = null;
protected HiveMetaStoreClient metaStoreClient = null;
@@ -109,6 +108,7 @@ public class CLIHiveClient implements IHiveClient {
builder.setOwner(table.getOwner());
builder.setLastAccessTime(table.getLastAccessTime());
builder.setTableType(table.getTableType());
+ builder.setSkipHeaderLineCount(table.getParameters().get("skip.header.line.count"));
return builder.createHiveTableMeta();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
index 57292dc..77e1084 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java
@@ -140,6 +140,7 @@ public class HiveSourceTableLoader {
tableExtDesc.addDataSourceProp("total_file_number", String.valueOf(hiveTableMeta.fileNum));
tableExtDesc.addDataSourceProp("hive_inputFormat", hiveTableMeta.sdInputFormat);
tableExtDesc.addDataSourceProp("hive_outputFormat", hiveTableMeta.sdOutputFormat);
+ tableExtDesc.addDataSourceProp("skip_header_line_count", String.valueOf(hiveTableMeta.skipHeaderLineCount));
metaMgr.saveTableExt(tableExtDesc);
metaMgr.saveSourceTable(tableDesc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
index 784a0bb..fa9eb29 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMeta.java
@@ -44,6 +44,7 @@ class HiveTableMeta {
String sdOutputFormat;
String owner;
String tableType;
+ int skipHeaderLineCount;
int lastAccessTime;
long fileSize;
long fileNum;
@@ -51,7 +52,7 @@ class HiveTableMeta {
List<HiveTableColumnMeta> allColumns;
List<HiveTableColumnMeta> partitionColumns;
- public HiveTableMeta(String tableName, String sdLocation, String sdInputFormat, String sdOutputFormat, String owner, String tableType, int lastAccessTime, long fileSize, long fileNum, boolean isNative, List<HiveTableColumnMeta> allColumns, List<HiveTableColumnMeta> partitionColumns) {
+ public HiveTableMeta(String tableName, String sdLocation, String sdInputFormat, String sdOutputFormat, String owner, String tableType, int lastAccessTime, long fileSize, long fileNum, int skipHeaderLineCount, boolean isNative, List<HiveTableColumnMeta> allColumns, List<HiveTableColumnMeta> partitionColumns) {
this.tableName = tableName;
this.sdLocation = sdLocation;
this.sdInputFormat = sdInputFormat;
@@ -64,6 +65,7 @@ class HiveTableMeta {
this.isNative = isNative;
this.allColumns = allColumns;
this.partitionColumns = partitionColumns;
+ this.skipHeaderLineCount = skipHeaderLineCount;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/d79d9374/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
index 7a3e5d6..073ded5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableMetaBuilder.java
@@ -32,6 +32,7 @@ public class HiveTableMetaBuilder {
private int lastAccessTime;
private long fileSize;
private long fileNum;
+ private int skipHeaderLineCount;
private boolean isNative = true;
private List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
private List<HiveTableMeta.HiveTableColumnMeta> partitionColumns = Lists.newArrayList();
@@ -81,6 +82,14 @@ public class HiveTableMetaBuilder {
return this;
}
+ public HiveTableMetaBuilder setSkipHeaderLineCount(String skipHeaderLineCount) {
+ if (null == skipHeaderLineCount)
+ this.skipHeaderLineCount = 0;
+ else
+ this.skipHeaderLineCount = Integer.parseInt(skipHeaderLineCount);
+ return this;
+ }
+
public HiveTableMetaBuilder setIsNative(boolean isNative) {
this.isNative = isNative;
return this;
@@ -97,6 +106,6 @@ public class HiveTableMetaBuilder {
}
public HiveTableMeta createHiveTableMeta() {
- return new HiveTableMeta(tableName, sdLocation, sdInputFormat, sdOutputFormat, owner, tableType, lastAccessTime, fileSize, fileNum, isNative, allColumns, partitionColumns);
+ return new HiveTableMeta(tableName, sdLocation, sdInputFormat, sdOutputFormat, owner, tableType, lastAccessTime, fileSize, fileNum, skipHeaderLineCount, isNative, allColumns, partitionColumns);
}
}
\ No newline at end of file
[21/50] [abbrv] kylin git commit: KYLIN 1875 update model designer
Posted by li...@apache.org.
KYLIN 1875 update model designer
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5b8c7f2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5b8c7f2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5b8c7f2
Branch: refs/heads/master-cdh5.7
Commit: d5b8c7f27820196400f43cdcd88b94206b581484
Parents: 91cf788
Author: chenzhx <34...@qq.com>
Authored: Fri Dec 9 11:44:35 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 15 18:57:36 2016 +0800
----------------------------------------------------------------------
webapp/app/index.html | 3 +
webapp/app/js/controllers/cubeModel.js | 191 +-------------
.../js/controllers/modelConditionsSettings.js | 84 ++++++
webapp/app/js/controllers/modelDataModel.js | 254 +++++++++++++++++++
webapp/app/js/controllers/modelDimensions.js | 6 +-
webapp/app/js/controllers/modelEdit.js | 100 +++-----
webapp/app/js/controllers/modelMeasures.js | 29 ++-
webapp/app/js/services/tree.js | 48 +++-
webapp/app/js/utils/utils.js | 24 +-
.../modelDesigner/conditions_settings.html | 57 ++---
.../app/partials/modelDesigner/data_model.html | 89 ++++---
.../modelDesigner/model_dimensions.html | 4 +-
.../partials/modelDesigner/model_measures.html | 59 +++--
13 files changed, 585 insertions(+), 363 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/index.html
----------------------------------------------------------------------
diff --git a/webapp/app/index.html b/webapp/app/index.html
index ad881d5..13b54c8 100644
--- a/webapp/app/index.html
+++ b/webapp/app/index.html
@@ -187,6 +187,9 @@
<!--New GUI-->
<script src="js/controllers/modelSchema.js"></script>
<script src="js/controllers/modelDimensions.js"></script>
+<script src="js/controllers/modelDataModel.js"></script>
+<script src="js/controllers/modelMeasures.js"></script>
+<script src="js/controllers/modelConditionsSettings.js"></script>
<script src="js/controllers/modelRefresh.js"></script>
<script src="js/controllers/modelEdit.js"></script>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/js/controllers/cubeModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeModel.js b/webapp/app/js/controllers/cubeModel.js
index b20a829..2c7de2c 100644
--- a/webapp/app/js/controllers/cubeModel.js
+++ b/webapp/app/js/controllers/cubeModel.js
@@ -18,8 +18,7 @@
'use strict';
-KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConfig,MetaModel,SweetAlert,ModelGraphService,$log,TableModel,ModelService,loadingRequest,modelsManager) {
-
+KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConfig,MetaModel,SweetAlert,ModelGraphService,$log,TableModel,ModelService,loadingRequest,modelsManager,VdmUtil) {
$scope.modelsManager = modelsManager;
$scope.buildGraph = function (model) {
@@ -45,193 +44,5 @@ KylinApp.controller('CubeModelCtrl', function ($location,$scope, $modal,cubeConf
};
$scope.cubeConfig = cubeConfig;
- var DataModel = function () {
- return {
- name: '',
- fact_table: '',
- lookups: []
- };
- };
-
- var Lookup = function () {
- return {
- table: '',
- join: {
- type: '',
- primary_key: [],
- foreign_key: [],
- isCompatible:[],
- pk_type:[],
- fk_type:[]
- }
- };
- };
-
- // Initialize params.
- $scope.lookupState = {
- editing: false,
- editingIndex: -1,
- filter: ''
- };
-
- $scope.newLookup = Lookup();
-
- var lookupList = modelsManager.selectedModel.lookups;
-
- $scope.openLookupModal = function () {
- var modalInstance = $modal.open({
- templateUrl: 'dataModelLookupTable.html',
- controller: cubeModelLookupModalCtrl,
- backdrop: 'static',
- scope: $scope
- });
- modalInstance.result.then(function () {
- if (!$scope.lookupState.editing) {
- $scope.doneAddLookup();
- } else {
- $scope.doneEditLookup();
- }
-
- }, function () {
- $scope.cancelLookup();
- });
- };
-
- // Controller for cube model lookup modal.
- var cubeModelLookupModalCtrl = function ($scope, $modalInstance) {
- $scope.ok = function () {
- $modalInstance.close();
- };
-
- $scope.cancel = function () {
- $modalInstance.dismiss('cancel');
- };
- };
-
- $scope.editLookup = function (lookup) {
- $scope.lookupState.editingIndex = lookupList.indexOf(lookup);
- $scope.lookupState.editing = true;
-
- // Make a copy of model will be editing.
- $scope.newLookup = angular.copy(lookup);
-
- $scope.openLookupModal();
- };
-
- $scope.doneAddLookup = function () {
- // Push newLookup which bound user input data.
- lookupList.push(angular.copy($scope.newLookup));
-
- $scope.resetParams();
- };
-
- $scope.doneEditLookup = function () {
- // Copy edited model to destination model.
- angular.copy($scope.newLookup, lookupList[$scope.lookupState.editingIndex]);
-
- $scope.resetParams();
- };
-
- $scope.cancelLookup = function () {
- $scope.resetParams();
- };
-
- $scope.removeLookup = function (lookup) {
- var dimExist = _.some(modelsManager.selectedModel.dimensions,function(item,index){
- return item.table===lookup.table;
- });
- if(dimExist) {
- SweetAlert.swal({
- title: '',
- text: "Once it's removed, all relative dimensions will be removed. Are you sure to remove the lookup table?",
- type: '',
- showCancelButton: true,
- confirmButtonColor: '#DD6B55',
- confirmButtonText: "Yes",
- closeOnConfirm: true
- }, function (isConfirm) {
- if (isConfirm) {
- for (var i = modelsManager.selectedModel.dimensions.length - 1; i >= 0; i--) {
- if (modelsManager.selectedModel.dimensions[i].table === lookup.table) {
- modelsManager.selectedModel.dimensions.splice(i, 1);
- }
- }
- lookupList.splice(lookupList.indexOf(lookup), 1);
- }
- });
- }else{
- lookupList.splice(lookupList.indexOf(lookup), 1);
- }
- };
-
-
- $scope.changeKey = function(index){
- var fact_table = modelsManager.selectedModel.fact_table;
- var lookup_table = $scope.newLookup.table;
- var pk_column = $scope.newLookup.join.primary_key[index];
- var fk_column = $scope.newLookup.join.foreign_key[index];
- if(pk_column!=='null'&&fk_column!=='null'){
- $scope.newLookup.join.pk_type[index] = TableModel.getColumnType(pk_column,lookup_table);
- $scope.newLookup.join.fk_type[index] = TableModel.getColumnType(fk_column,fact_table);
- if($scope.newLookup.join.pk_type[index]!==$scope.newLookup.join.fk_type[index]){
- $scope.newLookup.join.isCompatible[index]=false;
- }else{
- $scope.newLookup.join.isCompatible[index]=true;
- }
-
- }
- }
-
- $scope.addNewJoin = function(){
- $scope.newLookup.join.primary_key.push("null");
- $scope.newLookup.join.foreign_key.push("null");
- $scope.newLookup.join.fk_type.push("null");
- $scope.newLookup.join.pk_type.push("null");
- $scope.newLookup.join.isCompatible.push(true);
- };
-
- $scope.removeJoin = function($index){
- $scope.newLookup.join.primary_key.splice($index,1);
- $scope.newLookup.join.foreign_key.splice($index,1);
- $scope.newLookup.join.fk_type.splice($index,1);
- $scope.newLookup.join.pk_type.splice($index,1);
- $scope.newLookup.join.isCompatible.splice($index,1);
- };
-
- $scope.resetParams = function () {
- $scope.lookupState.editing = false;
- $scope.lookupState.editingIndex = -1;
- $scope.newLookup = Lookup();
- };
-
- $scope.checkLookupForm = function(){
- var errors = [];
- // null validate
- for(var i = 0;i<$scope.newLookup.join.primary_key.length;i++){
- if($scope.newLookup.join.primary_key[i]==='null'){
- errors.push("Primary Key can't be null.");
- break;
- }
- }
- for(var i = 0;i<$scope.newLookup.join.foreign_key.length;i++){
- if($scope.newLookup.join.foreign_key[i]==='null'){
- errors.push("Foreign Key can't be null.");
- break;
- }
- }
-
- var errorInfo = "";
- angular.forEach(errors,function(item){
- errorInfo+="\n"+item;
- });
- if(errors.length){
-// SweetAlert.swal('Warning!', errorInfo, '');
- SweetAlert.swal('', errorInfo, 'warning');
- return false;
- }else{
- return true;
- }
-
- };
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/js/controllers/modelConditionsSettings.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelConditionsSettings.js b/webapp/app/js/controllers/modelConditionsSettings.js
new file mode 100644
index 0000000..b3e4998
--- /dev/null
+++ b/webapp/app/js/controllers/modelConditionsSettings.js
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Created by jiazhong on 2015/3/13.
+ */
+
+'use strict';
+
+KylinApp.controller('ModelConditionsSettingsCtrl', function ($scope, $modal,MetaModel,modelsManager,VdmUtil) {
+ $scope.modelsManager = modelsManager;
+ $scope.availableFactTables = [];
+ $scope.selectedTables={fact:VdmUtil.getNameSpaceAliasName($scope.modelsManager.selectedModel.partition_desc.partition_date_column)}
+ $scope.availableFactTables.push(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table));
+ var joinTable = $scope.modelsManager.selectedModel.lookups;
+ for (var j = 0; j < joinTable.length; j++) {
+ if(joinTable[j].kind=='FACT'){
+ $scope.availableFactTables.push(joinTable[j].alias);
+ }
+ }
+ $scope.isFormatEdit = {editable:false};
+ var judgeFormatEditable = function(dateColumn){
+ if(dateColumn == null){
+ $scope.isFormatEdit.editable = false;
+ return;
+ }
+ var column = _.filter($scope.getColumnsByAlias(VdmUtil.getNameSpaceAliasName(dateColumn)),function(_column){
+ var columnName=VdmUtil.getNameSpaceAliasName(dateColumn)+"."+_column.name;
+ if(dateColumn == columnName){
+ return _column;
+ }
+ });
+
+ var data_type = column[0].datatype;
+ if(data_type ==="bigint" ||data_type ==="int" ||data_type ==="integer"){
+ $scope.isFormatEdit.editable = false;
+ $scope.modelsManager.selectedModel.partition_desc.partition_date_format='yyyyMMdd';
+ $scope.partitionColumn.hasSeparateTimeColumn=false;
+ $scope.modelsManager.selectedModel.partition_desc.partition_time_column=null;
+ $scope.modelsManager.selectedModel.partition_desc.partition_time_format=null;
+
+ return;
+ }
+
+ $scope.isFormatEdit.editable = true;
+ return;
+
+ };
+ $scope.partitionChange = function (dateColumn) {
+ judgeFormatEditable(dateColumn);
+ };
+ $scope.partitionColumn ={
+ "hasSeparateTimeColumn" : false
+ }
+
+ if (($scope.state.mode=='edit')&&($scope.isEdit = !!$scope.route.params)) {
+ if($scope.modelsManager.selectedModel.partition_desc.partition_time_column){
+ $scope.partitionColumn.hasSeparateTimeColumn = true;
+ }
+ judgeFormatEditable($scope.modelsManager.selectedModel.partition_desc.partition_date_column);
+ }
+
+
+ $scope.toggleHasSeparateColumn = function(){
+ if($scope.partitionColumn.hasSeparateTimeColumn == false){
+ $scope.modelsManager.selectedModel.partition_desc.partition_time_column = null;
+ }
+ }
+});
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/js/controllers/modelDataModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelDataModel.js b/webapp/app/js/controllers/modelDataModel.js
new file mode 100644
index 0000000..d17af91
--- /dev/null
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+'use strict';
+
+KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cubeConfig,MetaModel,SweetAlert,ModelGraphService,$log,TableModel,ModelService,loadingRequest,modelsManager,VdmUtil) {
+ $scope.modelsManager = modelsManager;
+ $scope.init = function (){
+ $scope.rootFactTable=$scope.modelsManager.selectedModel.fact_table;
+ $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)]=$scope.modelsManager.selectedModel.fact_table;
+ $scope.tableAliasMap[$scope.modelsManager.selectedModel.fact_table]=VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table);
+ $scope.aliasName.push(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table));
+ angular.forEach($scope.modelsManager.selectedModel.lookups,function(joinTable){
+ $scope.aliasTableMap[joinTable.alias]=joinTable.table;
+ $scope.tableAliasMap[joinTable.table]=joinTable.alias;
+ $scope.aliasName.push(joinTable.alias);
+ });
+ }
+ if($scope.state.mode=='edit'){
+ $scope.init();
+ }
+
+ $scope.cubeConfig = cubeConfig;
+ var DataModel = function () {
+ return {
+ name: '',
+ fact_table: '',
+ lookups: []
+ };
+ };
+
+ var Lookup = function () {
+ return {
+ table: '',
+ alias: '',
+ joinTable:'',
+ kind:'LOOKUP',
+ join: {
+ type: '',
+ primary_key: [],
+ foreign_key: [],
+ isCompatible:[],
+ pk_type:[],
+ fk_type:[]
+ }
+ };
+ };
+
+ // Initialize params.
+ $scope.lookupState = {
+ editing: false,
+ editingIndex: -1,
+ filter: ''
+ };
+
+ $scope.newLookup = Lookup();
+
+ var lookupList = modelsManager.selectedModel.lookups;
+
+ $scope.openLookupModal = function () {
+ var modalInstance = $modal.open({
+ templateUrl: 'dataModelLookupTable.html',
+ controller: cubeModelLookupModalCtrl,
+ backdrop: 'static',
+ scope: $scope
+ });
+ modalInstance.result.then(function () {
+ if (!$scope.lookupState.editing) {
+ $scope.doneAddLookup();
+ } else {
+ $scope.doneEditLookup();
+ }
+
+ }, function () {
+ $scope.cancelLookup();
+ });
+ };
+
+ // Controller for cube model lookup modal.
+ var cubeModelLookupModalCtrl = function ($scope, $modalInstance) {
+ $scope.ok = function () {
+ $modalInstance.close();
+ };
+
+ $scope.cancel = function () {
+ $modalInstance.dismiss('cancel');
+ };
+ };
+
+ $scope.editLookup = function (lookup) {
+ $scope.lookupState.editingIndex = lookupList.indexOf(lookup);
+ $scope.lookupState.editing = true;
+
+ // Make a copy of model will be editing.
+ $scope.newLookup = angular.copy(lookup);
+ $scope.newLookup.joinTable=VdmUtil.getNameSpaceTopName($scope.newLookup.join.foreign_key[0]);
+ $scope.openLookupModal();
+ };
+
+ $scope.doneAddLookup = function () {
+ // Push newLookup which bound user input data.
+ $scope.aliasTableMap[$scope.newLookup.alias]=$scope.newLookup.table;
+ $scope.tableAliasMap[$scope.newLookup.table]=$scope.newLookup.alias;
+ $scope.aliasName.push($scope.newLookup.alias);
+ lookupList.push(angular.copy($scope.newLookup));
+
+ $scope.resetParams();
+ };
+
+ $scope.doneEditLookup = function () {
+ // Copy edited model to destination model.
+ $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)]=$scope.modelsManager.selectedModel.fact_table;
+ $scope.aliasName=[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)];
+ angular.copy($scope.newLookup, lookupList[$scope.lookupState.editingIndex]);
+ angular.forEach(lookupList,function(joinTable){
+ $scope.aliasName.push(joinTable.alias);
+ $scope.aliasTableMap[joinTable.alias]=joinTable.table;
+ // $scope.tableAliasMap[joinTable.alias]=joinTable.table;
+ });
+
+ $scope.resetParams();
+ };
+ $scope.changeFactTable = function () {
+ delete $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)];
+ $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.rootFactTable)]=$scope.rootFactTable;
+ delete $scope.tableAliasMap[$scope.modelsManager.selectedModel.fact_table];
+ $scope.tableAliasMap[$scope.rootFactTable]=VdmUtil.removeNameSpace($scope.rootFactTable);
+ $scope.aliasName.splice($scope.aliasName.indexOf(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)),1);
+ $scope.aliasName.push(VdmUtil.removeNameSpace($scope.rootFactTable));
+ $scope.modelsManager.selectedModel.fact_table=$scope.rootFactTable;
+ }
+ $scope.changeJoinTable = function () {
+ $scope.newLookup.alias=$scope.newLookup.table;
+ }
+ $scope.cancelLookup = function () {
+ $scope.resetParams();
+ };
+
+ $scope.removeLookup = function (lookup) {
+ var dimExist = _.some(modelsManager.selectedModel.dimensions,function(item,index){
+ return item.alias===lookup.alias;
+ });
+ if(dimExist) {
+ SweetAlert.swal({
+ title: '',
+ text: "Once it's removed, all relative dimensions will be removed. Are you sure to remove the lookup table?",
+ type: '',
+ showCancelButton: true,
+ confirmButtonColor: '#DD6B55',
+ confirmButtonText: "Yes",
+ closeOnConfirm: true
+ }, function (isConfirm) {
+ if (isConfirm) {
+ for (var i = modelsManager.selectedModel.dimensions.length - 1; i >= 0; i--) {
+ if (modelsManager.selectedModel.dimensions[i].alias === lookup.alias) {
+ modelsManager.selectedModel.dimensions.splice(i, 1);
+ }
+ }
+ lookupList.splice(lookupList.indexOf(lookup), 1);
+ }
+ });
+ }else{
+ lookupList.splice(lookupList.indexOf(lookup), 1);
+ }
+ };
+ $scope.changeAlias = function (){
+ }
+
+ $scope.changeKey = function(index){
+ var join_table = $scope.newLookup.jointable;
+ var lookup_table = $scope.newLookup.table;
+ var pk_column = $scope.newLookup.join.primary_key[index];
+ var fk_column = $scope.newLookup.join.foreign_key[index];
+ if(pk_column!=='null'&&fk_column!=='null'){
+ $scope.newLookup.join.pk_type[index] = TableModel.getColumnType(pk_column,lookup_table);
+ $scope.newLookup.join.fk_type[index] = TableModel.getColumnType(fk_column,join_table);
+ if($scope.newLookup.join.pk_type[index]!==$scope.newLookup.join.fk_type[index]){
+ $scope.newLookup.join.isCompatible[index]=false;
+ }else{
+ $scope.newLookup.join.isCompatible[index]=true;
+ }
+
+ }
+ }
+
+ $scope.addNewJoin = function(){
+ $scope.newLookup.join.primary_key.push("null");
+ $scope.newLookup.join.foreign_key.push("null");
+ $scope.newLookup.join.fk_type.push("null");
+ $scope.newLookup.join.pk_type.push("null");
+ $scope.newLookup.join.isCompatible.push(true);
+ };
+
+ $scope.removeJoin = function($index){
+ $scope.newLookup.join.primary_key.splice($index,1);
+ $scope.newLookup.join.foreign_key.splice($index,1);
+ $scope.newLookup.join.fk_type.splice($index,1);
+ $scope.newLookup.join.pk_type.splice($index,1);
+ $scope.newLookup.join.isCompatible.splice($index,1);
+ };
+
+ $scope.resetParams = function () {
+ $scope.lookupState.editing = false;
+ $scope.lookupState.editingIndex = -1;
+ $scope.newLookup = Lookup();
+ };
+
+ $scope.checkLookupForm = function(){
+ var errors = [];
+ // null validate
+ for(var i = 0;i<$scope.newLookup.join.primary_key.length;i++){
+ if($scope.newLookup.join.primary_key[i]==='null'){
+ errors.push("Primary Key can't be null.");
+ break;
+ }
+ }
+ for(var i = 0;i<$scope.newLookup.join.foreign_key.length;i++){
+ if($scope.newLookup.join.foreign_key[i]==='null'){
+ errors.push("Foreign Key can't be null.");
+ break;
+ }
+ }
+
+ var errorInfo = "";
+ angular.forEach(errors,function(item){
+ errorInfo+="\n"+item;
+ });
+ if(errors.length){
+// SweetAlert.swal('Warning!', errorInfo, '');
+ SweetAlert.swal('', errorInfo, 'warning');
+ return false;
+ }else{
+ return true;
+ }
+
+ };
+
+
+});
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/js/controllers/modelDimensions.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelDimensions.js b/webapp/app/js/controllers/modelDimensions.js
index 1b0e889..3f8a55b 100644
--- a/webapp/app/js/controllers/modelDimensions.js
+++ b/webapp/app/js/controllers/modelDimensions.js
@@ -18,7 +18,7 @@
'use strict';
-KylinApp.controller('ModelDimensionsCtrl', function ($scope, $modal,MetaModel,modelsManager) {
+KylinApp.controller('ModelDimensionsCtrl', function ($scope, $modal,MetaModel,modelsManager,VdmUtil) {
$scope.modelsManager = modelsManager;
// Available columns list derived from cube data model.
@@ -34,10 +34,10 @@ KylinApp.controller('ModelDimensionsCtrl', function ($scope, $modal,MetaModel,mo
// Dump available columns plus column table name, whether is from lookup table.
$scope.initColumns = function () {
- $scope.availableTables.push(modelsManager.selectedModel.fact_table);
+ $scope.availableTables.push(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table));
var lookups = modelsManager.selectedModel.lookups;
for (var j = 0; j < lookups.length; j++) {
- $scope.availableTables.push(lookups[j].table);
+ $scope.availableTables.push(lookups[j].alias);
}
for(var i = 0;i<$scope.availableTables.length;i++){
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index 2f1b35e..4d1e12c 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -22,6 +22,10 @@
KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $location, $templateCache, $interpolate, MessageService, TableService, CubeDescService, ModelService, loadingRequest, SweetAlert,$log,cubeConfig,CubeDescModel,ModelDescService,MetaModel,TableModel,ProjectService,ProjectModel,modelsManager,VdmUtil) {
//add or edit ?
var absUrl = $location.absUrl();
+ $scope.tableAliasMap={};
+ $scope.aliasTableMap={};
+ $scope.aliasName=[];
+ $scope.route={params:$routeParams.modelName};
$scope.modelMode = absUrl.indexOf("/models/add")!=-1?'addNewModel':absUrl.indexOf("/models/edit")!=-1?'editExistModel':'default';
if($scope.modelMode=="addNewModel"&&ProjectModel.selectedProject==null){
@@ -30,11 +34,10 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
}
$scope.modelsManager = modelsManager;
-
$scope.cubeConfig = cubeConfig;
- $scope.getPartitonColumns = function(tableName){
- var columns = _.filter($scope.getColumnsByTable(tableName),function(column){
+ $scope.getPartitonColumns = function(alias){
+ var columns = _.filter($scope.getColumnsByAlias(alias),function(column){
return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar")||column.datatype==="bigint"||column.datatype==="int"||column.datatype==="integer";
});
return columns;
@@ -56,9 +59,17 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
});
return temp;
};
-
- $scope.getColumnType = function (_column,table){
- var columns = $scope.getColumnsByTable(table);
+ $scope.getColumnsByAlias = function (aliasName) {
+ var temp = [];
+ angular.forEach(TableModel.selectProjectTables, function (table) {
+ if (table.name == $scope.aliasTableMap[aliasName]) {
+ temp = table.columns;
+ }
+ });
+ return temp;
+ };
+ $scope.getColumnType = function (_column,alias){
+ var columns = $scope.getColumnsByAlias(alias);
var type;
angular.forEach(columns,function(column){
if(_column === column.name){
@@ -69,38 +80,6 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
return type;
};
- $scope.isFormatEditable = false;
- var judgeFormatEditable = function(dateColumn){
-
- if(dateColumn == null){
- $scope.isFormatEditable = false;
- return;
- }
- var column = _.filter($scope.getColumnsByTable($scope.modelsManager.selectedModel.fact_table),function(_column){
- var columnName=$scope.modelsManager.selectedModel.fact_table+"."+_column.name;
- if(dateColumn == columnName)
- return _column;
- });
-
- var data_type = column[0].datatype;
- if(data_type ==="bigint" ||data_type ==="int" ||data_type ==="integer"){
- $scope.isFormatEditable = false;
- $scope.modelsManager.selectedModel.partition_desc.partition_date_format='yyyyMMdd';
- $scope.partitionColumn.hasSeparateTimeColumn=false;
- $scope.modelsManager.selectedModel.partition_desc.partition_time_column=null;
- $scope.modelsManager.selectedModel.partition_desc.partition_time_format=null;
-
- return;
- }
-
- $scope.isFormatEditable = true;
- return;
-
- };
- $scope.partitionChange = function (dateColumn) {
- judgeFormatEditable(dateColumn);
- };
-
// ~ Define data
$scope.state = {
"modelSchema": "",
@@ -109,46 +88,27 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
project:$scope.projectModel.selectedProject
};
- $scope.partitionColumn ={
- "hasSeparateTimeColumn" : false
- }
- // ~ init
if ($scope.isEdit = !!$routeParams.modelName) {
-
- var modelName = $routeParams.modelName;
- ModelDescService.query({model_name: modelName}, function (model) {
- if (model) {
- modelsManager.selectedModel = model;
-
- if($scope.modelsManager.selectedModel.partition_desc.partition_time_column){
- $scope.partitionColumn.hasSeparateTimeColumn = true;
- }
- modelsManager.selectedModel.project = ProjectModel.getProjectByCubeModel(modelName);
-
- if(!ProjectModel.getSelectedProject()){
- ProjectModel.setSelectedProject(modelsManager.selectedModel.project);
- }
-
- TableModel.aceSrcTbLoaded().then(function(){
- judgeFormatEditable($scope.modelsManager.selectedModel.partition_desc.partition_date_column);
- });
- }
- });
+ var modelName = $routeParams.modelName;
+ ModelDescService.query({model_name: modelName}, function (model) {
+ if (model) {
+ modelsManager.selectedModel = model;
+ if($scope.modelsManager.selectedModel.partition_desc.partition_time_column){
+ $scope.partitionColumn.hasSeparateTimeColumn = true;
+ }
+ modelsManager.selectedModel.project = ProjectModel.getProjectByCubeModel(modelName);
+ if(!ProjectModel.getSelectedProject()){
+ ProjectModel.setSelectedProject(modelsManager.selectedModel.project);
+ }
+ }
+ });
//init project
-
} else {
MetaModel.initModel();
modelsManager.selectedModel = MetaModel.getMetaModel();
modelsManager.selectedModel.project = ProjectModel.getSelectedProject();
}
-
- $scope.toggleHasSeparateColumn = function(){
- if($scope.partitionColumn.hasSeparateTimeColumn == false){
- $scope.modelsManager.selectedModel.partition_desc.partition_time_column = null;
- }
- }
-
$scope.prepareModel = function () {
// generate column family
$scope.state.project = modelsManager.selectedModel.project;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/js/controllers/modelMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelMeasures.js b/webapp/app/js/controllers/modelMeasures.js
index 01cd35a..342f341 100644
--- a/webapp/app/js/controllers/modelMeasures.js
+++ b/webapp/app/js/controllers/modelMeasures.js
@@ -22,6 +22,31 @@
'use strict';
-KylinApp.controller('ModelMeasuresCtrl', function ($scope) {
-
+KylinApp.controller('ModelMeasuresCtrl', function ($scope, $modal,MetaModel,modelsManager,VdmUtil) {
+ $scope.modelsManager = modelsManager;
+ $scope.availableFactTables = [];
+ $scope.selectedFactTables = {};
+ $scope.availableFactTables.push(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table));
+ var joinTable = $scope.modelsManager.selectedModel.lookups;
+ for (var j = 0; j < joinTable.length; j++) {
+ if(joinTable[j].kind=='FACT'){
+ $scope.availableFactTables.push(joinTable[j].alias);
+ }
+ }
+ angular.forEach($scope.modelsManager.selectedModel.metrics,function(metric){
+ $scope.selectedFactTables[VdmUtil.getNameSpaceAliasName(metric)]=$scope.selectedFactTables[VdmUtil.getNameSpaceAliasName(metric)]||[];
+ $scope.selectedFactTables[VdmUtil.getNameSpaceAliasName(metric)].push(metric);
+ });
+ $scope.changeColumns = function (table){
+ angular.forEach($scope.selectedFactTables[table],function(column){
+ if($scope.modelsManager.selectedModel.metrics.indexOf(column)==-1){
+ $scope.modelsManager.selectedModel.metrics.push(column);
+ }
+ });
+ angular.forEach($scope.modelsManager.selectedModel.metrics,function(metric){
+ if($scope.selectedFactTables[VdmUtil.getNameSpaceAliasName(metric)].indexOf(metric)==-1){
+ $scope.modelsManager.selectedModel.metrics.splice($scope.modelsManager.selectedModel.metrics.indexOf(metric),1);
+ }
+ });
+ }
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/js/services/tree.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/tree.js b/webapp/app/js/services/tree.js
index acc4626..3b444c6 100755
--- a/webapp/app/js/services/tree.js
+++ b/webapp/app/js/services/tree.js
@@ -16,8 +16,9 @@
* limitations under the License.
*/
-KylinApp.service('ModelGraphService', function () {
-
+KylinApp.service('ModelGraphService', function (VdmUtil) {
+ var tablesNodeList=[];
+ var aliasList=[];
var margin = {top: 20, right: 100, bottom: 20, left: 100},
width = 1100 - margin.right - margin.left,
height = 600;
@@ -37,31 +38,52 @@ KylinApp.service('ModelGraphService', function () {
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
var graphData = {
- "type": "fact",
- "name": model.fact_table,
+ "type": "FACT",
+ "name": VdmUtil.removeNameSpace(model.fact_table),
"children": []
};
-
+ tablesNodeList.push(graphData);
+ aliasList.push(graphData.name);
model.graph = (!!model.graph) ? model.graph : {};
- angular.forEach(model.lookups, function (lookup, index) {
- if (lookup.join && lookup.join.primary_key.length > 0) {
+ angular.forEach(model.lookups, function (lookup, index) {
+ if (lookup.join && aliasList.indexOf(lookup.alias)==-1) {
+ var dimensionNode={
+ "type": lookup.kind,
+ "name": lookup.alias,
+ "join": lookup.join,
+ "children": [],
+ "_children": []
+ }
+ aliasList.push(dimensionNode.name);
+ tablesNodeList.push(dimensionNode);
+ }
+
+ });
+ angular.forEach(model.lookups, function (lookup) {
+ if (lookup.join && lookup.join.primary_key.length > 0) {
+ VdmUtil.getNameSpaceAliasName(lookup.join.primary_key[0]);
+ VdmUtil.getNameSpaceAliasName(lookup.join.foreign_key[0]);
+ }
+ });
+
- var dimensionNode;
+/* Loop through the graphData.children array to find out: If the LKP table is already existed *//*
- /* Loop through the graphData.children array to find out: If the LKP table is already existed */
for(var j = 0; j < graphData.children.length; j++ ) {
- if(graphData.children[j].name == lookup.table){
+ if(graphData.children[j].name == lookup.alias){
dimensionNode = graphData.children[j];
break;
}
}
- /* If not existed, create dimensionNode and push it */
+ */
+/* If not existed, create dimensionNode and push it *//*
+
if(j == graphData.children.length) {
dimensionNode = {
"type": "dimension",
- "name": lookup.table,
+ "name": lookup.alias,
"join": lookup.join,
"children": [],
"_children": []
@@ -73,7 +95,7 @@ KylinApp.service('ModelGraphService', function () {
}
}
- });
+*/
angular.forEach(model.dimensions, function (dimension, index) {
// for dimension on lookup table
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/js/utils/utils.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/utils/utils.js b/webapp/app/js/utils/utils.js
index 0875e52..1da840e 100644
--- a/webapp/app/js/utils/utils.js
+++ b/webapp/app/js/utils/utils.js
@@ -140,7 +140,27 @@ KylinApp.factory('VdmUtil', function ($modal, $timeout, $location, $anchorScroll
return obj;
}
return angular.toJson(filterData(newObj),true);
- }
-
+ },
+ removeNameSpace:function(str){
+ if(str){
+ return str.replace(/([^.\s]+\.)+/,'');
+ }else{
+ return '';
+ }
+ },
+ getNameSpaceTopName:function(str){
+ if(str){
+ return str.replace(/(\.[^.]+)/,'');
+ }else{
+ return '';
+ }
+ },
+ getNameSpaceAliasName:function(str){
+ if(str){
+ return str.replace(/\.[^.]+$/,'');
+ }else{
+ return '';
+ }
+ },
}
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/partials/modelDesigner/conditions_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index 6e7a883..6820b60 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -15,52 +15,44 @@
* See the License for the specific language governing permissions and
* limitations under the License.
-->
-
-<div class="row">
+<div ng-controller="ModelConditionsSettingsCtrl">
+ <div class="row">
<div class="col-xs-12">
<ng-form name="forms.model_setting_form" novalidate>
<h3>Partition</h3>
<div class="box-body">
- <!--Cube Partition Type-->
- <div class="form-group" ng-hide="true">
- <div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Type</b></label>
- <div class="col-xs-12 col-sm-6">
- <select class="form-control"
- ng-if="state.mode=='edit'"
- chosen ng-model="modelsManager.selectedModel.partition_desc.partition_type"
- ng-options="ddt as ddt for ddt in cubeConfig.cubePartitionTypes">
- <option value=""></option>
- </select>
- <span ng-if="state.mode=='view'">{{modelsManager.selectedModel.partition_desc.partition_type}}</span>
- </div>
- </div>
- </div>
-
<!--Partition Column-->
<div class="form-group">
<div class="row middle-popover">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Date Column</b> <i kylinpopover placement="right" title="Partition Date Column" template="partitionTip.html" class="fa fa-info-circle"></i></label>
- <div class="col-xs-12 col-sm-6">
-
- <select style="width: 100%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
- ng-model="modelsManager.selectedModel.partition_desc.partition_date_column"
- ng-if="state.mode=='edit'" ng-change="partitionChange(modelsManager.selectedModel.partition_desc.partition_date_column)"
+ <div class="col-xs-12 col-sm-6" ng-if="state.mode=='edit'">
+ <select style="width: 45%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
+ ng-model="selectedTables.fact"
data-placement=""
- ng-options="modelsManager.selectedModel.fact_table+'.'+columns.name as modelsManager.selectedModel.fact_table+'.'+columns.name for columns in getPartitonColumns(modelsManager.selectedModel.fact_table)" >
- <option value="">--Select Partition Column--</option>
+ ng-options="alias as alias for alias in availableFactTables" >
+ <option value="">--Select Partition Table--</option>
</select>
- <!--<small class="text-info" ng-show="state.mode=='edit'">(Column Type should be DATE or TIMESTAMP Type)</small>-->
- <span ng-if="state.mode=='view'">
- {{!!(modelsManager.selectedModel.partition_desc.partition_date_column)?modelsManager.selectedModel.partition_desc.partition_date_column: ''}}</span>
+ <select style="width: 45%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
+ ng-model="modelsManager.selectedModel.partition_desc.partition_date_column"
+ ng-change="partitionChange(modelsManager.selectedModel.partition_desc.partition_date_column)"
+ data-placement=""
+ ng-options="selectedTables.fact+'.'+columns.name as columns.name for columns in getPartitonColumns(selectedTables.fact)" >
+ <option value="">--Select Partition Column--</option>
+ </select>
+ <!--<small class="text-info" ng-show="state.mode=='edit'">(Column Type should be DATE or TIMESTAMP Type)</small>-->
+ </div>
+ <div class="col-xs-12 col-sm-6" ng-if="state.mode=='view'">
+ <span >
+ {{!!(modelsManager.selectedModel.partition_desc.partition_date_column)?modelsManager.selectedModel.partition_desc.partition_date_column: ''}}
+ </span>
</div>
</div>
</div>
<!--Date Format-->
- <div class="form-group" ng-if="(state.mode=='edit') || (state.mode=='view')">
+ <div class="form-group" >
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Date Format</b></label>
<div class="col-xs-12 col-sm-6">
@@ -68,7 +60,7 @@
ng-required="modelsManager.selectedModel.partition_desc.partition_date_format"
ng-model="modelsManager.selectedModel.partition_desc.partition_date_format"
ng-if="state.mode=='edit'"
- ng-disabled="isFormatEditable!==true"
+ ng-disabled="$parent.isFormatEdit.editable!==true"
data-placement=""
ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt">
<option value="">--Select Date Format--</option>
@@ -79,7 +71,7 @@
</div>
<!--Date Format-->
- <div class="form-group middle-popover" ng-if="isFormatEditable==true">
+ <div class="form-group middle-popover" ng-if="isFormatEdit.editable==true">
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Has a separate "time of the day" column ?</b> <i kylinpopover placement="right" title="Separate Time Column" template="separateTimeColumnTip.html" class="fa fa-info-circle"></i></label>
<div class="col-xs-12 col-sm-6">
@@ -160,8 +152,8 @@
</div>
</ng-form>
</div>
+ </div>
</div>
-
<script type="text/ng-template" id="partitionTip.html">
<ol>
<li>Partition date column not required,leave as default if cube always need full build</Li>
@@ -183,3 +175,4 @@
<script type="text/ng-template" id="separateTimeColumnTip.html">
<p>For cases where fact table saves date and hour at two columns (KYLIN-1427)</p>
</script>
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index ace977f..82be964 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -16,7 +16,7 @@
* limitations under the License.
-->
-<div ng-controller="CubeModelCtrl">
+<div ng-controller="ModelDataModelCtrl">
<ng-form name="forms.data_model_form">
<!-- Fact Table Name -->
@@ -26,9 +26,9 @@
<b>Fact Table</b>
</label>
<div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.data_model_form.table_name.$invalid && (forms.data_model_form.table_name.$dirty||forms.data_model_form.$submitted)}">
- <select chosen ng-model="modelsManager.selectedModel.fact_table" ng-if="state.mode=='edit'"
+ <select chosen ng-model="$parent.rootFactTable" ng-if="state.mode=='edit'"
ng-options="table.name as table.name for table in tableModel.selectProjectTables"
- style="width:100%;"
+ style="width:100%;" ng-change="changeFactTable()"
name="table_name"
ng-required="true"
data-placeholder="Fact Table Name"
@@ -37,7 +37,7 @@
</select>
<small class="help-block" ng-show="forms.data_model_form.innerform.typeahead.$error.required && (forms.data_model_form.innerform.typeahead.$dirty||forms.data_model_form.$submitted)">The fact table is required</small>
- <span ng-if="state.mode=='view'">{{modelsManager.selectedModel.fact_table}}</span>
+ <span ng-if="state.mode=='view'">{{$parent.rootFactTable}}</span>
</div>
</div>
</div>
@@ -48,11 +48,11 @@
<div class="col-xs-6" ng-if="state.mode=='edit'">
<button type="button" class="btn btn-primary" ng-disabled="!modelsManager.selectedModel.fact_table.length"
ng-click="openLookupModal()">
- <i class="fa fa-plus"></i> Add Lookup Table
+ <i class="fa fa-plus"></i> Add Join Table
</button>
</div>
<div class="col-xs-6" ng-if="state.mode!='edit'">
- <b>{{modelsManager.selectedModel.lookups.length ? 'Lookup Tables' : 'No Lookup Tables'}}</b>
+ <b>{{modelsManager.selectedModel.lookups.length ? 'Join Tables' : 'No Join Tables'}}</b>
</div>
<div class="col-xs-6">
<span class="pull-right input-icon input-icon-right nav-search" style="margin-left: 22px;" ng-if="modelsManager.selectedModel.lookups.length">
@@ -66,19 +66,24 @@
<tr>
<th>ID</th>
<th>Table Name</th>
+ <th>Table Kind</th>
<th>Join Type</th>
<th>Join Condition</th>
<th ng-if="state.mode=='edit'">Actions</th>
</tr>
</thead>
<tbody>
- <tr ng-repeat="lookup in modelsManager.selectedModel.lookups | filter:lookupState.filter track by $index">
+ <tr ng-repeat="lookup in modelsManager.selectedModel.lookups | filter:lookupState.filter">
<td>
<b>{{$index + 1}}</b>
</td>
<!-- Table Name -->
<td>
- <span tooltip="Lookup Table Name">{{lookup.table}}</span>
+ <span>{{lookup.alias}}</span>
+ </td>
+ <!-- Table Kind -->
+ <td>
+ <span>{{lookup.kind}}</span>
</td>
<!-- Join Type -->
<td>
@@ -88,7 +93,7 @@
<td>
<ul class="list-unstyled">
<li ng-repeat="pk in lookup.join.primary_key track by $index">
- <code>{{modelsManager.selectedModel.fact_table + '.' + lookup.join.foreign_key[$index]}} = {{lookup.table + '.' + pk}}</code>
+ <code>{{lookup.join.foreign_key[$index]}} = {{pk}}</code>
</li>
</ul>
</td>
@@ -108,7 +113,7 @@
</div>
</ng-form>
- <!-- Add Lookup Table Form -->
+ <!-- Add Join Table Form -->
<script type="text/ng-template" id="dataModelLookupTable.html">
<div class="modal-header">
<h4 class="box-title lighter">{{lookupState.editing ? 'Edit' : 'Add'}} Lookup</h4>
@@ -117,28 +122,45 @@
<div class="row">
<div class="col-xs-8">
<ng-form name="lookup_form">
- <!--Table Name-->
- <div class="form-group">
+ <div class="form-group">
<div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Lookup Table Name</b></label>
- <div ng-class="{'has-error':lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$submitted)}">
- <div class="col-xs-12 col-sm-6">
- <select chosen ng-model="newLookup.table"
- ng-options="table.name as table.name for table in tableModel.selectProjectTables"
- style="width:100%;"
- name="table_name"
- ng-required="true"
- data-placeholder="Lookup Table Name"
- class="chosen-select">
- <option value=""> -- Select Lookup Table -- </option>
- </select>
+ <div class="col-xs-9">
+ <div>
+ <select chosen ng-model="newLookup.joinTable" style="width: 45%"
+ ng-options="table as table for table in aliasName"
+ name="table_name"
+ ng-required="true"
+ data-placeholder="Join Table Name"
+ class="chosen-select">
+ <option value=""> -- Select Join Table -- </option>
+ </select>
+ <small class="help-block" ng-show="lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$submitted)">Table name required</small>
+ <b> </b>
+ <select chosen ng-model="newLookup.table" style="width: 45%"
+ ng-options="table.name as table.name for table in tableModel.selectProjectTables"
+ name="table_name"
+ ng-required="true" ng-change="changeJoinTable()"
+ data-placeholder="Join Table Name"
+ class="chosen-select">
+ <option value=""> -- Select Join Table -- </option>
+ </select>
+ <small class="help-block" ng-show="lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$submitted)">Table name required</small>
- <small class="help-block" ng-show="lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$submitted)">Table name required</small>
- </div>
+ </div>
+ <div class="space-4"></div>
+ <small class="help-block red" ng-show="newLookup.join.isCompatible[$index]==false"><i class="fa fa-exclamation-triangle"></i> <b>Column Type incompatible {{newLookup.join.foreign_key[$index]}}[{{newLookup.join.fk_type[$index]}}], {{newLookup.join.primary_key[$index]}} [{{newLookup.join.pk_type[$index]}}]</b></small>
+ </div>
+ </div>
+ </div>
+ <div class="form-group">
+ <div class="row">
+ <label class="col-sm-3 control-label font-color-default"><b>Alias</b></label>
+ <div class="col-sm-6">
+ <input type="text" class="form-control " placeholder="Input Table Alias" ng-required="true"
+ ng-model="newLookup.alias" ng-change="changeAlias()" ng-disabled="">
</div>
</div>
</div>
-
<!--Join Type and Columns-->
<div class="form-group">
<div class="row">
@@ -153,18 +175,27 @@
</div>
<div class="form-group">
<div class="row">
+ <label class="col-sm-3 control-label font-color-default"><b>Table Type</b></label>
+ <div class="col-sm-6">
+ <label> <input type="radio" ng-model="newLookup.kind" value="FACT" > Fact Table </label>
+ <label> <input type="radio" ng-model="newLookup.kind" value="LOOKUP"> Lookup Table </label>
+ </div>
+ </div>
+ </div>
+ <div class="form-group">
+ <div class="row">
<div class="col-xs-9">
<div ng-repeat="joinIndex in [] | range: newLookup.join.primary_key.length">
<div>
<select style="width: 45%" chosen data-placeholder="Fact Table Column"
ng-model="newLookup.join.foreign_key[$index]" ng-change="changeKey($index)"
- ng-options="columns.name as columns.name for columns in getColumnsByTable(modelsManager.selectedModel.fact_table)" >
+ ng-options="newLookup.joinTable+'.'+columns.name as columns.name for columns in getColumnsByAlias(newLookup.joinTable)" >
<option value=""></option>
</select>
<b>=</b>
<select style="width: 45%" chosen data-placeholder="Lookup Table Column"
ng-model="newLookup.join.primary_key[$index]" ng-change="changeKey($index)"
- ng-options="columns.name as columns.name for columns in getColumnsByTable(newLookup.table)" >
+ ng-options="newLookup.alias+'.'+columns.name as columns.name for columns in getColumnsByTable(newLookup.table)" >
<option value=""></option>
</select>
<button class="pull-right btn btn-xs btn-danger" style="cursor: pointer" tooltip="Delete"
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/partials/modelDesigner/model_dimensions.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/model_dimensions.html b/webapp/app/partials/modelDesigner/model_dimensions.html
index f767b9d..6249521 100644
--- a/webapp/app/partials/modelDesigner/model_dimensions.html
+++ b/webapp/app/partials/modelDesigner/model_dimensions.html
@@ -70,13 +70,13 @@
</td>
<td class="col-xs-9">
<ui-select
- ng-if="state.mode=='edit'" style="width: 100%"
+ style="width: 100%"
autofocus="true"
close-on-select="false"
ng-model="modelsManager.selectedModel.dimensions[$index].columns" multiple>
<ui-select-match placeholder="Select Column...">{{$item.name}}</ui-select-match>
<ui-select-choices
- repeat="column.name as column in getColumnsByTable(dimension.table) | filter:$select.search">
+ repeat="column.name as column in getColumnsByAlias(dimension.table) | filter:$select.search">
{{column.name}}
</ui-select-choices>
</ui-select>
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5b8c7f2/webapp/app/partials/modelDesigner/model_measures.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/model_measures.html b/webapp/app/partials/modelDesigner/model_measures.html
index 20c73c3..f34e3fb 100644
--- a/webapp/app/partials/modelDesigner/model_measures.html
+++ b/webapp/app/partials/modelDesigner/model_measures.html
@@ -17,10 +17,10 @@
-->
<!-- Measures Summary -->
-<ng-form name="forms.model_measure_form" novalidate>
-<div class="dataTables_wrapper form-inline no-footer">
-
- <table ng-if="state.mode=='view'&&modelsManager.selectedModel.metrics.length > 0" class="table table-striped table-hover">
+<div ng-controller="ModelMeasuresCtrl">
+ <ng-form name="forms.model_measure_form" novalidate>
+ <div class="dataTables_wrapper form-inline no-footer">
+ <table ng-if="state.mode=='view'&&modelsManager.selectedModel.metrics.length > 0" class="table table-striped table-hover">
<thead>
<tr>
<th>ID</th>
@@ -39,23 +39,42 @@
</td>
</tr>
</tbody>
- </table>
- <div ng-if="state.mode=='edit'" class="form-group" style="width: 100%">
+ </table>
+ <div ng-if="state.mode=='edit'" class="form-group" style="width: 100%">
<h4 style="margin-left:42px">Select measure columns</h4>
- <ui-select
- close-on-select="false"
- ng-if="state.mode=='edit'"
- style="width: 100%"
- autofocus="true"
- ng-model="modelsManager.selectedModel.metrics" multiple>
- <ui-select-match placeholder="Select Column...">{{$item.name}}</ui-select-match>
- <ui-select-choices
- repeat="measure.name as measure in getColumnsByTable(modelsManager.selectedModel.fact_table) | filter:$select.search">
- {{measure.name}}
- </ui-select-choices>
- </ui-select>
+ <table style="margin-left:42px; width:92%"
+ class="table table-hover list">
+ <tr class="row">
+ <th class="col-xs-1">ID</th>
+ <th class="col-xs-2">Table Name</th>
+ <th class="col-xs-9">Columns</th>
+ </tr>
+ <tr class="row" ng-repeat="table in availableFactTables ">
+ <td class="col-xs-1">
+ <!-- ID -->
+ <b>{{($index + 1)}}</b>
+ </td>
+ <td class="col-xs-2">
+ {{table}}
+ </td>
+ <td class="col-xs-9">
+ <ui-select
+ close-on-select="false"
+ ng-if="state.mode=='edit'"
+ style="width: 100%" ng-change="changeColumns(table)"
+ autofocus="true"
+ ng-model="selectedFactTables[table]" multiple>
+ <ui-select-match placeholder="Select Column...">{{$item.name}}</ui-select-match>
+ <ui-select-choices
+ repeat="table+'.'+measure.name as measure in getColumnsByAlias(table) | filter:$select.search">
+ {{measure.name}}
+ </ui-select-choices>
+ </ui-select>
+ </td>
+ </tr>
+ </table>
+ </div>
</div>
+ </ng-form>
</div>
-
-</ng-form>
[25/50] [abbrv] kylin git commit: KYLIN 1875 update tooltip
Posted by li...@apache.org.
KYLIN 1875 update tooltip
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7cc152e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7cc152e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7cc152e
Branch: refs/heads/master-cdh5.7
Commit: e7cc152e69ce98cba7fd3be5251d334314f25828
Parents: a923047
Author: chenzhx <34...@qq.com>
Authored: Tue Dec 13 15:57:34 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 15 18:57:36 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/cubeDesigner/dimensions.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7cc152e/webapp/app/partials/cubeDesigner/dimensions.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/dimensions.html b/webapp/app/partials/cubeDesigner/dimensions.html
index 8c1cdab..7fedeb6 100644
--- a/webapp/app/partials/cubeDesigner/dimensions.html
+++ b/webapp/app/partials/cubeDesigner/dimensions.html
@@ -57,7 +57,7 @@
</td>
<!--Table Name -->
<td>
- <span>{{dimension.table}}</span>
+ <span tooltip="{{availableFactTables.indexOf(dimension.table)==-1? 'Lookup Table':'Fact Table'}}">{{dimension.table}}</span>
</td>
<!--Type-->
<td>
[34/50] [abbrv] kylin git commit: KYLIN-2283 A new data gen tool for
CI
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
new file mode 100644
index 0000000..82455ab
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.datagen;
+
+import java.io.IOException;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DataGenTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testCIConfigured() throws IOException {
+ DataModelDesc model = getModel("test_kylin_inner_join_model_desc");
+ ModelDataGenerator gen = new ModelDataGenerator(model, 100);
+ gen.outprint = true;
+
+ gen.generate();
+ }
+
+ @Test
+ public void testSSBNoConfig() throws IOException {
+ DataModelDesc model = getModel("ssb");
+ ModelDataGenerator gen = new ModelDataGenerator(model, 100);
+ gen.outprint = true;
+
+ gen.generate();
+ }
+
+ private DataModelDesc getModel(String name) {
+ MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelDesc model = mgr.getDataModelDesc(name);
+ return model;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/examples/test_case_data/localmeta/data/data_gen_config.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/data_gen_config.json b/examples/test_case_data/localmeta/data/data_gen_config.json
deleted file mode 100644
index 15b3fd0..0000000
--- a/examples/test_case_data/localmeta/data/data_gen_config.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
- "differentiateBoundary": "2013-07-01",
- "columnConfigs": [
- {
- "columnName": "lstg_format_name",
- "valueSet": [
- "FP-GTC",
- "FP-non GTC",
- "ABIN",
- "Auction",
- "Others"
- ],
- "exclusive": true,
- "differentiateByDateBoundary": true
- },
- {
- "columnName": "BUYER_COUNTRY",
- "valueSet": [
- "CN",
- "DE",
- "FR",
- "JP",
- "UK",
- "US"
- ],
- "exclusive": true
- },
- {
- "columnName": "SELLER_COUNTRY",
- "valueSet": [
- "CN",
- "DE",
- "FR",
- "JP",
- "UK",
- "US"
- ],
- "exclusive": true
- },
- {
- "columnName": "SELLER_ID",
- "valueSet": [
- "10000000",
- "10001000"
- ],
- "asRange": true
- },
- {
- "columnName": "ITEM_COUNT",
- "valueSet": [
- "0",
- "2000000"
- ],
- "asRange": true
- },
- {
- "columnName": "PRICE",
- "valueSet": [
- "0",
- "1000"
- ],
- "asRange": true
- }
- ]
-}
[17/50] [abbrv] kylin git commit: minor,
add DataModelDesc.setFilterCondition()
Posted by li...@apache.org.
minor, add DataModelDesc.setFilterCondition()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8d0b504b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8d0b504b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8d0b504b
Branch: refs/heads/master-cdh5.7
Commit: 8d0b504bfc69738b4b0ccc1611f6765b76c47884
Parents: 654144f
Author: Li Yang <li...@apache.org>
Authored: Thu Dec 15 14:48:42 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 15 14:48:42 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/metadata/model/DataModelDesc.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8d0b504b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 0be240e..0b494e3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -209,6 +209,11 @@ public class DataModelDesc extends RootPersistentEntity {
return filterCondition;
}
+ // for internal only
+ public void setFilterCondition(String filterCondition) {
+ this.filterCondition = filterCondition;
+ }
+
public PartitionDesc getPartitionDesc() {
return partitionDesc;
}
[35/50] [abbrv] kylin git commit: KYLIN-2283 A new data gen tool for
CI
Posted by li...@apache.org.
KYLIN-2283 A new data gen tool for CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d1175d2c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d1175d2c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d1175d2c
Branch: refs/heads/master-cdh5.7
Commit: d1175d2c414853f4d223601a2db096f7f963ce72
Parents: e14f4e1
Author: Li Yang <li...@apache.org>
Authored: Thu Dec 15 18:51:00 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Dec 18 17:23:18 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DataGenTest.java | 56 --
.../java/org/apache/kylin/job/DeployUtil.java | 27 +-
.../apache/kylin/job/dataGen/ColumnConfig.java | 80 ---
.../kylin/job/dataGen/FactTableGenerator.java | 696 -------------------
.../org/apache/kylin/job/dataGen/GenConfig.java | 92 ---
.../apache/kylin/metadata/model/ColumnDesc.java | 8 +
.../apache/kylin/metadata/model/TableDesc.java | 9 +
.../kylin/source/datagen/ColumnGenConfig.java | 110 +++
.../kylin/source/datagen/ColumnGenerator.java | 361 ++++++++++
.../source/datagen/ModelDataGenerator.java | 282 ++++++++
.../kylin/source/datagen/TableGenConfig.java | 48 ++
.../org/apache/kylin/source/datagen/Util.java | 75 ++
.../topn/TopNCounterSerializerTest.java | 76 --
.../hllc/NewHyperLogLogBenchmarkTest.java | 2 +
.../measure/topn/TopNCounterSerializerTest.java | 76 ++
.../kylin/source/datagen/DataGenTest.java | 67 ++
.../localmeta/data/data_gen_config.json | 65 --
.../localmeta/data/flatten_data_for_ii.csv | 402 -----------
.../table/DEFAULT.TEST_KYLIN_FACT.json | 32 +-
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 4 +-
.../resources/query/sql_timeout/query02.sql | 2 +-
21 files changed, 1070 insertions(+), 1500 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/assembly/src/test/java/org/apache/kylin/job/DataGenTest.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DataGenTest.java b/assembly/src/test/java/org/apache/kylin/job/DataGenTest.java
deleted file mode 100644
index af4f9fb..0000000
--- a/assembly/src/test/java/org/apache/kylin/job/DataGenTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job;
-
-import static org.junit.Assert.assertTrue;
-
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.job.dataGen.FactTableGenerator;
-import org.apache.kylin.metadata.MetadataManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- *
- */
-public class DataGenTest extends LocalFileMetadataTestCase {
-
- @Before
- public void before() throws Exception {
- this.createTestMetadata();
- MetadataManager.clearCache();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testBasics() throws Exception {
- String content = FactTableGenerator.generate("test_kylin_cube_with_slr_ready", "10000", "1", null);// default settings
- //System.out.println(content);
- assertTrue(content.contains("FP-non GTC"));
- assertTrue(content.contains("ABIN"));
-
- //DeployUtil.overrideFactTableData(content, "default.test_kylin_fact");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 23b3670..8fc583d 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -33,20 +33,21 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ResourceTool;
+import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.job.dataGen.FactTableGenerator;
import org.apache.kylin.job.streaming.StreamDataLoader;
import org.apache.kylin.job.streaming.StreamingTableDataGenerator;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.source.datagen.ModelDataGenerator;
import org.apache.kylin.source.hive.HiveClientFactory;
-import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.source.hive.IHiveClient;
import org.apache.kylin.source.kafka.TimedJsonStreamParser;
import org.apache.maven.model.Model;
@@ -131,16 +132,15 @@ public class DeployUtil {
public static void prepareTestDataForNormalCubes(String cubeName) throws Exception {
- String factTableName = TABLE_KYLIN_FACT.toUpperCase();
- String content = null;
-
boolean buildCubeUsingProvidedData = Boolean.parseBoolean(System.getProperty("buildCubeUsingProvidedData"));
if (!buildCubeUsingProvidedData) {
System.out.println("build cube with random dataset");
+
// data is generated according to cube descriptor and saved in resource store
- content = FactTableGenerator.generate(cubeName, "10000", "0.6", null);
- assert content != null;
- overrideFactTableData(content, factTableName);
+ MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelDesc model = mgr.getDataModelDesc("test_kylin_inner_join_model_desc");
+ ModelDataGenerator gen = new ModelDataGenerator(model, 10000);
+ gen.generate();
} else {
System.out.println("build normal cubes with provided dataset");
}
@@ -168,17 +168,6 @@ public class DeployUtil {
appendFactTableData(sb.toString(), cubeInstance.getRootFactTable());
}
- public static void overrideFactTableData(String factTableContent, String factTableName) throws IOException {
- // Write to resource store
- ResourceStore store = ResourceStore.getStore(config());
-
- InputStream in = new ByteArrayInputStream(factTableContent.getBytes("UTF-8"));
- String factTablePath = "/data/" + factTableName + ".csv";
- store.deleteResource(factTablePath);
- store.putResource(factTablePath, in, System.currentTimeMillis());
- in.close();
- }
-
public static void appendFactTableData(String factTableContent, String factTableName) throws IOException {
// Write to resource store
ResourceStore store = ResourceStore.getStore(config());
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/assembly/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
deleted file mode 100644
index 5e1c09f..0000000
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/ColumnConfig.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.dataGen;
-
-import java.util.ArrayList;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- */
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class ColumnConfig {
- @JsonProperty("columnName")
- private String columnName;
- @JsonProperty("valueSet")
- private ArrayList<String> valueSet;
- @JsonProperty("exclusive")
- private boolean exclusive;
- @JsonProperty("asRange")
- private boolean asRange;
- @JsonProperty("differentiateByDateBoundary")
- private boolean differentiateByDateBoundary;
-
- public boolean isAsRange() {
- return asRange;
- }
-
- public void setAsRange(boolean asRange) {
- this.asRange = asRange;
- }
-
- public boolean isExclusive() {
- return exclusive;
- }
-
- public void setExclusive(boolean exclusive) {
- this.exclusive = exclusive;
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
-
- public ArrayList<String> getValueSet() {
- return valueSet;
- }
-
- public void setValueSet(ArrayList<String> valueSet) {
- this.valueSet = valueSet;
- }
-
- public boolean isDifferentiateByDateBoundary() {
- return differentiateByDateBoundary;
- }
-
- public void setDifferentiateByDateBoundary(boolean differentiateByDateBoundary) {
- this.differentiateByDateBoundary = differentiateByDateBoundary;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
deleted file mode 100644
index 011035b..0000000
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java
+++ /dev/null
@@ -1,696 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.dataGen;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.google.common.collect.Lists;
-
-/**
- */
-public class FactTableGenerator {
- CubeInstance cube = null;
- CubeDesc desc = null;
- ResourceStore store = null;
- String factTableName = null;
-
- GenConfig genConf = null;
-
- Random r = null;
-
- String cubeName;
- long randomSeed;
- int rowCount;
- int unlinkableRowCount;
- int unlinkableRowCountMax;
- double conflictRatio;
- double linkableRatio;
-
- long differentiateBoundary = -1;
- List<Integer> differentiateColumns = Lists.newArrayList();
-
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
-
- // the names of lookup table columns which is in relation with fact
- // table(appear as fk in fact table)
- TreeMap<String, LinkedList<String>> lookupTableKeys = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- // possible values of lookupTableKeys, extracted from existing lookup tables.
- // The key is in the format of tablename/columnname
- TreeMap<String, ArrayList<String>> feasibleValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- // lookup table name -> sets of all composite keys
- TreeMap<String, HashSet<Array<String>>> lookupTableCompositeKeyValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- private void init(String cubeName, int rowCount, double conflictRaio, double linkableRatio, long randomSeed) {
- this.rowCount = rowCount;
- this.conflictRatio = conflictRaio;
- this.cubeName = cubeName;
- this.randomSeed = randomSeed;
- this.linkableRatio = linkableRatio;
-
- this.unlinkableRowCountMax = (int) (this.rowCount * (1 - linkableRatio));
- this.unlinkableRowCount = 0;
-
- r = new Random(randomSeed);
-
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- cube = CubeManager.getInstance(config).getCube(cubeName);
- desc = cube.getDescriptor();
- factTableName = cube.getRootFactTable();
- store = ResourceStore.getStore(config);
- }
-
- /*
- * users can specify the value preference for each column
- */
- private void loadConfig() {
- try {
- InputStream configStream = store.getResource("/data/data_gen_config.json").inputStream;
- this.genConf = GenConfig.loadConfig(configStream);
-
- if (configStream != null)
- configStream.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- private void loadLookupTableValues(String lookupTableName, LinkedList<String> columnNames, int distinctRowCount) throws Exception {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- // only deal with composite keys
- if (columnNames.size() > 1 && !lookupTableCompositeKeyValues.containsKey(lookupTableName)) {
- lookupTableCompositeKeyValues.put(lookupTableName, new HashSet<Array<String>>());
- }
-
- InputStream tableStream = null;
- BufferedReader tableReader = null;
- try {
- TreeMap<String, Integer> zeroBasedInice = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- for (String columnName : columnNames) {
- ColumnDesc cDesc = MetadataManager.getInstance(config).getTableDesc(lookupTableName).findColumnByName(columnName);
- zeroBasedInice.put(columnName, cDesc.getZeroBasedIndex());
- }
-
- String path = "/data/" + lookupTableName + ".csv";
- tableStream = store.getResource(path).inputStream;
- tableReader = new BufferedReader(new InputStreamReader(tableStream));
- tableReader.mark(0);
- int rowCount = 0;
- int curRowNum = 0;
- String curRow;
-
- while (tableReader.readLine() != null)
- rowCount++;
-
- HashSet<Integer> rows = new HashSet<Integer>();
- distinctRowCount = (distinctRowCount < rowCount) ? distinctRowCount : rowCount;
- while (rows.size() < distinctRowCount) {
- rows.add(r.nextInt(rowCount));
- }
-
- // reopen the stream
- tableReader.close();
- tableStream.close();
- tableStream = null;
- tableReader = null;
-
- tableStream = store.getResource(path).inputStream;
- tableReader = new BufferedReader(new InputStreamReader(tableStream));
-
- while ((curRow = tableReader.readLine()) != null) {
- if (rows.contains(curRowNum)) {
- String[] tokens = curRow.split(",");
-
- String[] comboKeys = null;
- int index = 0;
- if (columnNames.size() > 1)
- comboKeys = new String[columnNames.size()];
-
- for (String columnName : columnNames) {
- int zeroBasedIndex = zeroBasedInice.get(columnName);
- if (!feasibleValues.containsKey(lookupTableName + "/" + columnName))
- feasibleValues.put(lookupTableName + "/" + columnName, new ArrayList<String>());
- feasibleValues.get(lookupTableName + "/" + columnName).add(tokens[zeroBasedIndex]);
-
- if (columnNames.size() > 1) {
- comboKeys[index] = tokens[zeroBasedIndex];
- index++;
- }
- }
-
- if (columnNames.size() > 1) {
- Array<String> wrap = new Array<String>(comboKeys);
- if (lookupTableCompositeKeyValues.get(lookupTableName).contains(wrap)) {
- throw new Exception("The composite key already exist in the lookup table");
- }
- lookupTableCompositeKeyValues.get(lookupTableName).add(wrap);
- }
- }
- curRowNum++;
- }
-
- } catch (IOException e) {
- e.printStackTrace();
- System.exit(1);
- } finally {
- if (tableStream != null)
- tableStream.close();
- if (tableReader != null)
- tableReader.close();
- }
- }
-
- // prepare the candidate values for each joined column
- private void prepare() throws Exception {
- // load config
- loadConfig();
-
- int index = 0;
- for (ColumnDesc cDesc : MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(factTableName).getColumns()) {
- ColumnConfig cConfig = genConf.getColumnConfigByName(cDesc.getName());
-
- if (cConfig != null && cConfig.isDifferentiateByDateBoundary()) {
- if (!cDesc.getType().isStringFamily()) {
- throw new IllegalStateException("differentiateByDateBoundary only applies to text types, actual:" + cDesc.getType());
- }
- if (genConf.getDifferentiateBoundary() == null) {
- throw new IllegalStateException("differentiateBoundary not provided");
- }
- if (differentiateBoundary == -1) {
- differentiateBoundary = format.parse(genConf.getDifferentiateBoundary()).getTime();
- }
- differentiateColumns.add(index);
- }
- index++;
- }
-
- TreeSet<String> factTableColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
-
- for (DimensionDesc dim : desc.getDimensions()) {
- for (TblColRef col : dim.getColumnRefs()) {
- if (col.getTable().equals(factTableName))
- factTableColumns.add(col.getName());
- }
-
- JoinDesc join = dim.getJoin();
- if (join != null) {
- String lookupTable = dim.getTableRef().getTableIdentity();
- for (String column : dropAlias(join.getPrimaryKey())) {
- if (!lookupTableKeys.containsKey(lookupTable)) {
- lookupTableKeys.put(lookupTable, new LinkedList<String>());
- }
-
- if (!lookupTableKeys.get(lookupTable).contains(column))
- lookupTableKeys.get(lookupTable).add(column);
- }
- }
- }
-
- int distinctRowCount = (int) (this.rowCount / this.conflictRatio);
- distinctRowCount = (distinctRowCount == 0) ? 1 : distinctRowCount;
- // lookup tables
- for (String lookupTable : lookupTableKeys.keySet()) {
- this.loadLookupTableValues(lookupTable, lookupTableKeys.get(lookupTable), distinctRowCount);
- }
- }
-
- private List<DimensionDesc> getSortedDimentsionDescs() {
- List<DimensionDesc> dimensions = desc.getDimensions();
- Collections.sort(dimensions, new Comparator<DimensionDesc>() {
- @Override
- public int compare(DimensionDesc o1, DimensionDesc o2) {
- JoinDesc j1 = o2.getJoin();
- JoinDesc j2 = o1.getJoin();
- return Integer.valueOf(j1 != null ? j1.getPrimaryKey().length : 0).compareTo(j2 != null ? j2.getPrimaryKey().length : 0);
- }
- });
- return dimensions;
- }
-
- /**
- * Generate the fact table and return it as text
- *
- * @return
- * @throws Exception
- */
- private String cookData() throws Exception {
- // the columns on the fact table can be classified into three groups:
- // 1. foreign keys
- TreeMap<String, String> factTableCol2LookupCol = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- // 2. metrics or directly used dimensions
- TreeSet<String> usedCols = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
- // 3. others, not referenced anywhere
-
- TreeMap<String, String> lookupCol2factTableCol = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
- // find fact table columns in fks
- List<DimensionDesc> dimensions = getSortedDimentsionDescs();
- for (DimensionDesc dim : dimensions) {
- JoinDesc jDesc = dim.getJoin();
- if (jDesc != null) {
- String[] fks = dropAlias(jDesc.getForeignKey());
- String[] pks = dropAlias(jDesc.getPrimaryKey());
- int num = fks.length;
- for (int i = 0; i < num; ++i) {
- String value = dim.getTableRef().getTableIdentity() + "/" + pks[i];
-
- lookupCol2factTableCol.put(value, fks[i]);
-
- if (factTableCol2LookupCol.containsKey(fks[i])) {
- if (!factTableCol2LookupCol.get(fks[i]).equals(value)) {
- System.out.println("Warning: Disambiguation on the mapping of column " + fks[i] + ", " + factTableCol2LookupCol.get(fks[i]) + "(chosen) or " + value);
- continue;
- }
- }
- factTableCol2LookupCol.put(fks[i], value);
- }
- }
- //else, deal with it in next roung
- }
-
- // find fact table columns in direct dimension
- // DO NOT merge this with the previous loop
- for (DimensionDesc dim : dimensions) {
- JoinDesc jDesc = dim.getJoin();
- if (jDesc == null) {
- // column on fact table used directly as a dimension
- String aColumn = dim.getColumn();
- if (!factTableCol2LookupCol.containsKey(aColumn))
- usedCols.add(aColumn);
- }
- }
-
- // find fact table columns in measures
- for (MeasureDesc mDesc : desc.getMeasures()) {
- List<TblColRef> pcols = mDesc.getFunction().getParameter().getColRefs();
- if (pcols != null) {
- for (TblColRef col : pcols) {
- if (!factTableCol2LookupCol.containsKey(col.getName()))
- usedCols.add(col.getName());
- }
- }
- }
-
- return createTable(this.rowCount, factTableCol2LookupCol, lookupCol2factTableCol, usedCols);
- }
-
- private String[] dropAlias(String[] aliasDotCol) {
- String[] result = new String[aliasDotCol.length];
- for (int i = 0; i < aliasDotCol.length; i++) {
- String str = aliasDotCol[i];
- int cut = str.lastIndexOf('.');
- if (cut >= 0) {
- str = str.substring(cut + 1);
- }
- result[i] = str;
- }
- return result;
- }
-
- private String normToTwoDigits(int v) {
- if (v < 10)
- return "0" + v;
- else
- return Integer.toString(v);
- }
-
- private String randomPick(ArrayList<String> candidates) {
- int index = r.nextInt(candidates.size());
- return candidates.get(index);
- }
-
- private String createRandomCell(ColumnDesc cDesc, ArrayList<String> range) throws Exception {
- DataType type = cDesc.getType();
- if (type.isStringFamily()) {
- throw new Exception("Can't handle range values for string");
-
- } else if (type.isIntegerFamily()) {
- int low = Integer.parseInt(range.get(0));
- int high = Integer.parseInt(range.get(1));
- return Integer.toString(r.nextInt(high - low) + low);
-
- } else if (type.isDouble()) {
- double low = Double.parseDouble(range.get(0));
- double high = Double.parseDouble(range.get(1));
- return String.format("%.4f", r.nextDouble() * (high - low) + low);
-
- } else if (type.isFloat()) {
- float low = Float.parseFloat(range.get(0));
- float high = Float.parseFloat(range.get(1));
- return String.format("%.4f", r.nextFloat() * (high - low) + low);
-
- } else if (type.isDecimal()) {
- double low = Double.parseDouble(range.get(0));
- double high = Double.parseDouble(range.get(1));
- return String.format("%.4f", r.nextDouble() * (high - low) + low);
-
- } else if (type.isDateTimeFamily()) {
- if (!type.isDate()) {
- throw new RuntimeException("Does not support " + type);
- }
-
- Date start = format.parse(range.get(0));
- Date end = format.parse(range.get(1));
- long diff = end.getTime() - start.getTime();
- Date temp = new Date(start.getTime() + (long) (diff * r.nextDouble()));
- Calendar cal = Calendar.getInstance();
- cal.setTime(temp);
- // first day
- cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
-
- return cal.get(Calendar.YEAR) + "-" + normToTwoDigits(cal.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(cal.get(Calendar.DAY_OF_MONTH));
- } else {
- System.out.println("The data type " + type + "is not recognized");
- System.exit(1);
- }
- return null;
- }
-
- private String createRandomCell(ColumnDesc cDesc) {
- DataType type =cDesc.getType();
- String s = type.getName();
- if (s.equals("char") || s.equals("varchar")) {
- StringBuilder sb = new StringBuilder();
- int len = Math.min(type.getPrecision(), 3);
- for (int i = 0; i < len; i++) {
- sb.append((char) ('a' + r.nextInt(10))); // cardinality at most 10x10x10
- }
- return sb.toString();
- } else if (s.equals("bigint") || s.equals("int") || s.equals("tinyint") || s.equals("smallint")) {
- return Integer.toString(r.nextInt(128));
- } else if (s.equals("double")) {
- return String.format("%.4f", r.nextDouble() * 100);
- } else if (s.equals("float")) {
- return String.format("%.4f", r.nextFloat() * 100);
- } else if (s.equals("decimal")) {
- return String.format("%.4f", r.nextDouble() * 100);
- } else if (s.equals("date")) {
- long date20131231 = 61349312153265L;
- long date20010101 = 60939158400000L;
- long diff = date20131231 - date20010101;
- Date temp = new Date(date20010101 + (long) (diff * r.nextDouble()));
- Calendar cal = Calendar.getInstance();
- cal.setTime(temp);
- // first day
- cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
-
- return cal.get(Calendar.YEAR) + "-" + normToTwoDigits(cal.get(Calendar.MONTH) + 1) + "-" + normToTwoDigits(cal.get(Calendar.DAY_OF_MONTH));
- } else {
- System.out.println("The data type " + type + "is not recognized");
- System.exit(1);
- }
- return null;
- }
-
- private String createDefaultsCell(String type) {
- String s = type.toLowerCase();
- if (s.equals("string") || s.equals("char") || s.equals("varchar")) {
- return "abcde";
- } else if (s.equals("bigint") || s.equals("int") || s.equals("tinyint") || s.equals("smallint")) {
- return "0";
- } else if (s.equals("double")) {
- return "0";
- } else if (s.equals("float")) {
- return "0";
- } else if (s.equals("decimal")) {
- return "0";
- } else if (s.equals("date")) {
- return "1970-01-01";
- } else {
- System.out.println("The data type " + type + "is not recognized");
- System.exit(1);
- }
- return null;
- }
-
- private void printColumnMappings(TreeMap<String, String> factTableCol2LookupCol, TreeSet<String> usedCols, TreeSet<String> defaultColumns) {
-
- System.out.println("=======================================================================");
- System.out.format("%-30s %s", "FACT_TABLE_COLUMN", "MAPPING");
- System.out.println();
- System.out.println();
- for (Map.Entry<String, String> entry : factTableCol2LookupCol.entrySet()) {
- System.out.format("%-30s %s", entry.getKey(), entry.getValue());
- System.out.println();
- }
- for (String key : usedCols) {
- System.out.format("%-30s %s", key, "Random Values");
- System.out.println();
- }
- for (String key : defaultColumns) {
- System.out.format("%-30s %s", key, "Default Values");
- System.out.println();
- }
- System.out.println("=======================================================================");
-
- System.out.println("Parameters:");
- System.out.println();
- System.out.println("CubeName: " + cubeName);
- System.out.println("RowCount: " + rowCount);
- System.out.println("ConflictRatio: " + conflictRatio);
- System.out.println("LinkableRatio: " + linkableRatio);
- System.out.println("Seed: " + randomSeed);
- System.out.println();
- System.out.println("The number of actual unlinkable fact rows is: " + this.unlinkableRowCount);
- System.out.println("You can vary the above parameters to generate different datasets.");
- System.out.println();
- }
-
- // Any row in the column must finally appear in the flatten big table.
- // for single-column joins the generated row is guaranteed to have a match
- // in lookup table
- // for composite keys we'll need an extra check
- private boolean matchAllCompositeKeys(TreeMap<String, String> lookupCol2FactTableCol, LinkedList<String> columnValues) {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- for (String lookupTable : lookupTableKeys.keySet()) {
- if (lookupTableKeys.get(lookupTable).size() == 1)
- continue;
-
- String[] comboKey = new String[lookupTableKeys.get(lookupTable).size()];
- int index = 0;
- for (String column : lookupTableKeys.get(lookupTable)) {
- String key = lookupTable + "/" + column;
- String factTableCol = lookupCol2FactTableCol.get(key);
- int cardinal = MetadataManager.getInstance(config).getTableDesc(factTableName).findColumnByName(factTableCol).getZeroBasedIndex();
- comboKey[index] = columnValues.get(cardinal);
-
- index++;
- }
- Array<String> wrap = new Array<String>(comboKey);
- if (!lookupTableCompositeKeyValues.get(lookupTable).contains(wrap)) {
- // System.out.println("Try " + wrap + " Failed, continue...");
- return false;
- }
- }
- return true;
- }
-
- private String createCell(ColumnDesc cDesc) throws Exception {
- ColumnConfig cConfig = null;
-
- if ((cConfig = genConf.getColumnConfigByName(cDesc.getName())) == null) {
- // if the column is not configured, use random values
- return (createRandomCell(cDesc));
-
- } else {
- // the column has a configuration
- if (!cConfig.isAsRange() && !cConfig.isExclusive() && r.nextBoolean()) {
- // if the column still allows random values
- return (createRandomCell(cDesc));
-
- } else {
- // use specified values
- ArrayList<String> valueSet = cConfig.getValueSet();
- if (valueSet == null || valueSet.size() == 0)
- throw new Exception("Did you forget to specify value set for " + cDesc.getName());
-
- if (!cConfig.isAsRange()) {
- return (randomPick(valueSet));
- } else {
- if (valueSet.size() != 2)
- throw new Exception("Only two values can be set for range values, the column: " + cDesc.getName());
-
- return (createRandomCell(cDesc, valueSet));
- }
- }
-
- }
- }
-
- private LinkedList<String> createRow(TreeMap<String, String> factTableCol2LookupCol, TreeSet<String> usedCols, TreeSet<String> defaultColumns) throws Exception {
- LinkedList<String> columnValues = new LinkedList<String>();
-
- long currentRowTime = -1;
-
- for (TblColRef col : cube.getModel().getRootFactTable().getColumns()) {
-
- String colName = col.getName();
-
- if (factTableCol2LookupCol.containsKey(colName)) {
-
- // if the current column is a fk column in fact table
- ArrayList<String> candidates = this.feasibleValues.get(factTableCol2LookupCol.get(colName));
-
- columnValues.add(candidates.get(r.nextInt(candidates.size())));
- } else if (usedCols.contains(colName)) {
- // if the current column is a metric or dimension column in fact table
- columnValues.add(createCell(col.getColumnDesc()));
- } else {
-
- // otherwise this column is not useful in OLAP
- columnValues.add(createDefaultsCell(col.getColumnDesc().getTypeName()));
- defaultColumns.add(colName);
- }
-
- if (col.equals(cube.getModel().getPartitionDesc().getPartitionDateColumnRef())) {
- currentRowTime = format.parse(columnValues.get(columnValues.size() - 1)).getTime();
- }
- }
-
- for (Integer index : differentiateColumns) {
- if (r.nextBoolean()) {//only change half of data
- if (currentRowTime >= differentiateBoundary) {
- columnValues.set(index, columnValues.get(index) + "_B");
- } else {
- columnValues.set(index, columnValues.get(index) + "_A");
- }
- }
- }
-
- return columnValues;
- }
-
- /**
- * return the text of table contents(one line one row)
- *
- * @param rowCount
- * @param factTableCol2LookupCol
- * @param lookupCol2FactTableCol
- * @param usedCols
- * @return
- * @throws Exception
- */
- private String createTable(int rowCount, TreeMap<String, String> factTableCol2LookupCol, TreeMap<String, String> lookupCol2FactTableCol, TreeSet<String> usedCols) throws Exception {
- try {
- TreeSet<String> defaultColumns = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
-
- StringBuffer sb = new StringBuffer();
- for (int i = 0; i < rowCount;) {
-
- LinkedList<String> columnValues = createRow(factTableCol2LookupCol, usedCols, defaultColumns);
-
- if (!matchAllCompositeKeys(lookupCol2FactTableCol, columnValues)) {
- if (unlinkableRowCount < unlinkableRowCountMax) {
- unlinkableRowCount++;
- } else {
- continue;
- }
- }
-
- for (String c : columnValues)
- sb.append(c + ",");
- sb.deleteCharAt(sb.length() - 1);
- sb.append(System.getProperty("line.separator"));
-
- i++;
-
- // System.out.println("Just generated the " + i + "th record");
- }
-
- printColumnMappings(factTableCol2LookupCol, usedCols, defaultColumns);
-
- return sb.toString();
-
- } catch (IOException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- return null;
- }
-
- /**
- * Randomly create a fact table and return the table content
- *
- * @param cubeName name of the cube
- * @param rowCount expected row count generated
- * @param linkableRatio the percentage of fact table rows that can be linked with all
- * lookup table by INNER join
- * @param randomSeed random seed
- */
- public static String generate(String cubeName, String rowCount, String linkableRatio, String randomSeed) throws Exception {
-
- if (rowCount == null)
- rowCount = "10000";
- if (linkableRatio == null)
- linkableRatio = "0.6";
-
- //if (randomSeed == null)
- // don't give it value
-
- // String conflictRatio = "5";//this parameter do not allow configuring
- // any more
-
- FactTableGenerator generator = new FactTableGenerator();
- long seed;
- if (randomSeed != null) {
- seed = Long.parseLong(randomSeed);
- } else {
- Random r = new Random();
- seed = r.nextLong();
- }
-
- generator.init(cubeName, Integer.parseInt(rowCount), 5, Double.parseDouble(linkableRatio), seed);
- generator.prepare();
- return generator.cookData();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/assembly/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
deleted file mode 100644
index 5204d2a..0000000
--- a/assembly/src/test/java/org/apache/kylin/job/dataGen/GenConfig.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.dataGen;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import org.apache.kylin.common.util.JsonUtil;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.databind.JsonMappingException;
-
-/**
- */
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class GenConfig {
-
- @JsonProperty("columnConfigs")
- private ArrayList<ColumnConfig> columnConfigs;
-
- @JsonProperty("differentiateBoundary")
- private String differentiateBoundary; //data before and after the provided date will be different, so that different segments will have different segments
-
- private HashMap<String, ColumnConfig> cache = new HashMap<String, ColumnConfig>();
-
- public String getDifferentiateBoundary() {
- return differentiateBoundary;
- }
-
- public void setDifferentiateBoundary(String differentiateBoundary) {
- this.differentiateBoundary = differentiateBoundary;
- }
-
- public ArrayList<ColumnConfig> getColumnConfigs() {
- return columnConfigs;
- }
-
- public void setColumnConfigs(ArrayList<ColumnConfig> columnConfigs) {
- this.columnConfigs = columnConfigs;
- }
-
- public ColumnConfig getColumnConfigByName(String columnName) {
- columnName = columnName.toLowerCase();
-
- if (cache.containsKey(columnName))
- return cache.get(columnName);
-
- for (ColumnConfig cConfig : columnConfigs) {
- if (cConfig.getColumnName().toLowerCase().equals(columnName)) {
- cache.put(columnName, cConfig);
- return cConfig;
- }
- }
- cache.put(columnName, null);
- return null;
- }
-
- public static GenConfig loadConfig(InputStream stream) {
- try {
- GenConfig config = JsonUtil.readValue(stream, GenConfig.class);
- return config;
- } catch (JsonMappingException e) {
- e.printStackTrace();
- } catch (JsonParseException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 2da1f5e..7105ede 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -45,6 +45,10 @@ public class ColumnDesc implements Serializable {
@JsonInclude(JsonInclude.Include.NON_NULL)
private String comment;
+ @JsonProperty("data_gen")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String dataGen;
+
// parsed from data type
private DataType type;
private DataType upgradedType;
@@ -148,6 +152,10 @@ public class ColumnDesc implements Serializable {
public void setNullable(boolean nullable) {
this.isNullable = nullable;
}
+
+ public String getDataGen() {
+ return dataGen;
+ }
public void init(TableDesc table) {
this.table = table;
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index ab8c465..e845da1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -26,6 +26,7 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.StringSplitter;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -47,6 +48,10 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
private int sourceType = ISourceAware.ID_HIVE;
@JsonProperty("table_type")
private String tableType;
+
+ @JsonProperty("data_gen")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private String dataGen;
private DatabaseDesc database = new DatabaseDesc();
@@ -160,6 +165,10 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
return getMaxColumnIndex() + 1;
}
+ public String getDataGen() {
+ return dataGen;
+ }
+
public void init() {
if (name != null)
name = name.toUpperCase();
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
new file mode 100644
index 0000000..6387873
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.datagen;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.metadata.model.ColumnDesc;
+
+public class ColumnGenConfig {
+
+ public static final String FK = "FK";
+ public static final String ID = "ID";
+ public static final String RAND = "RAND";
+ public static final String $RANDOM = "${RANDOM}";
+
+ // discrete values
+ boolean isDiscrete;
+ boolean isFK;
+ List<String> values;
+
+ // random
+ boolean isRandom;
+ String randFormat;
+ int randStart;
+ int randEnd;
+
+ // ID
+ boolean isID;
+ int idStart;
+
+ // general
+ int cardinality;
+ boolean genNull;
+ double genNullPct;
+ String genNullStr;
+ boolean order;
+ boolean unique;
+
+ public ColumnGenConfig(ColumnDesc col, ModelDataGenerator modelGen) throws IOException {
+ init(col, modelGen);
+ }
+
+ private void init(ColumnDesc col, ModelDataGenerator modelGen) throws IOException {
+
+ Map<String, String> config = Util.parseEqualCommaPairs(col.getDataGen(), "values");
+
+ values = Arrays.asList(Util.parseString(config, "values", "").split("[|]"));
+
+ List<String> pkValues = modelGen.getPkValuesIfIsFk(col);
+
+ if (FK.equals(values.get(0)) || (values.get(0).isEmpty() && pkValues != null)) {
+ isFK = true;
+ values = pkValues;
+ } else if (ID.equals(values.get(0))) {
+ isID = true;
+ idStart = (values.size() > 1) ? Integer.parseInt(values.get(1)) : 0;
+ } else if (RAND.equals(values.get(0)) || values.get(0).isEmpty()) {
+ isRandom = true;
+ randFormat = (values.size() > 1) ? values.get(1) : "";
+ randStart = (values.size() > 2) ? Integer.parseInt(values.get(2)) : 0;
+ randEnd = (values.size() > 3) ? Integer.parseInt(values.get(3)) : 0;
+ } else {
+ isDiscrete = true;
+ }
+
+ cardinality = Util.parseInt(config, "card", guessCardinality(col.getName()));
+ genNull = Util.parseBoolean(config, "null", guessGenNull(col.getName()));
+ genNullPct = Util.parseDouble(config, "nullpct", 0.01);
+ genNullStr = Util.parseString(config, "nullstr", "\\N"); // '\N' is null in hive
+ order = Util.parseBoolean(config, "order", false);
+ unique = Util.parseBoolean(config, "uniq", modelGen.isPK(col));
+ }
+
+ private int guessCardinality(String col) {
+ for (String s : col.split("_")) {
+ if (s.startsWith("C")) {
+ try {
+ return Integer.parseInt(s.substring(1));
+ } catch (Exception ex) {
+ // ok
+ }
+ }
+ }
+ return 0;
+ }
+
+ private boolean guessGenNull(String col) {
+ return col.contains("_NULL");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
new file mode 100644
index 0000000..775f4fc
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenerator.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.datagen;
+
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.TreeSet;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.ColumnDesc;
+
+import com.google.common.base.Preconditions;
+
+public class ColumnGenerator {
+
+ final private ColumnGenConfig conf;
+ final private ColumnDesc targetCol;
+ final private int targetRows;
+
+ public ColumnGenerator(ColumnDesc col, int nRows, ModelDataGenerator modelGen) throws IOException {
+ this.conf = new ColumnGenConfig(col, modelGen);
+ this.targetCol = col;
+ this.targetRows = nRows;
+ }
+
+ public Iterator<String> generate(long seed) {
+ Base result;
+ if (conf.isFK) {
+ result = new DiscreteGen(conf.values, seed);
+ } else if (conf.isID) {
+ result = new IDGen(conf.idStart);
+ } else if (conf.isRandom) {
+ result = new RandomGen(targetCol, conf.randFormat, conf.randStart, conf.randEnd, conf.cardinality);
+ } else {
+ result = new DiscreteGen(conf.values);
+ }
+
+ if (conf.cardinality > 0) {
+ result = new CardinalityFilter(result, conf.cardinality);
+ }
+
+ if (conf.genNull) {
+ result = new AddNullFilter(result, conf.genNullPct, conf.genNullStr);
+ }
+
+ if (conf.order || conf.unique) {
+ result = new OrderFilter(result, conf.unique, targetRows);
+ }
+
+ return result;
+ }
+
+ abstract public static class Base implements Iterator<String> {
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private class RandomGen extends Base {
+
+ private DataType type;
+ private String format;
+ private int randStart;
+ private int randEnd;
+ private Random rand;
+
+ public RandomGen(ColumnDesc col, String format, int randStart, int randEnd, int cardinality) {
+ this.type = col.getType();
+
+ if (type.isStringFamily()) {
+ // string
+ if (StringUtils.isBlank(format)) {
+ String name = col.getName();
+ format = name.substring(0, Math.min(4, name.length())) + ColumnGenConfig.$RANDOM;
+ }
+ Preconditions.checkArgument(format.contains(ColumnGenConfig.$RANDOM));
+ initNumberRange(randStart, randEnd, cardinality);
+ } else if (type.isTimeFamily()) {
+ // time
+ format = StringUtil.noBlank(format, DateFormat.DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS);
+ initDateTimeRange(randStart, randEnd, 0);
+ } else if (type.isDateTimeFamily()) {
+ // date
+ format = StringUtil.noBlank(format, DateFormat.DEFAULT_DATE_PATTERN);
+ initDateTimeRange(randStart, randEnd, cardinality);
+ } else if (type.isIntegerFamily()) {
+ // integer
+ initNumberRange(randStart, randEnd, cardinality);
+ format = StringUtil.noBlank(format, "#");
+ } else if (type.isNumberFamily()) {
+ // double
+ initNumberRange(randStart, randEnd, 0);
+ format = StringUtil.noBlank(format, ".##");
+ } else {
+ throw new IllegalArgumentException();
+ }
+
+ this.format = format;
+ this.rand = new Random();
+ }
+
+ private void initDateTimeRange(int randStart, int randEnd, int days) {
+ if (randStart == 0 && randEnd == 0) {
+ randStart = 2010;
+ randEnd = 2015;
+ }
+ randEnd = Math.max(randEnd, randStart + (days / 365) + 1);
+
+ Preconditions.checkArgument(randStart < randEnd);
+ Preconditions.checkArgument((randEnd - randStart) * 365 >= days);
+
+ this.randStart = randStart;
+ this.randEnd = randEnd;
+ }
+
+ private void initNumberRange(int randStart, int randEnd, int cardinality) {
+ if (randStart == 0 && randEnd == 0) {
+ randStart = 0;
+ randEnd = 1000;
+ }
+ randEnd = Math.max(randEnd, randStart + cardinality);
+
+ Preconditions.checkArgument(randStart < randEnd);
+ Preconditions.checkArgument(randEnd - randStart >= cardinality);
+
+ this.randStart = randStart;
+ this.randEnd = randEnd;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return true;
+ }
+
+ @Override
+ public String next() {
+ if (type.isStringFamily()) {
+ // string
+ return format.replace(ColumnGenConfig.$RANDOM, "" + randomInt());
+ } else if (type.isTimeFamily()) {
+ // time
+ return DateFormat.formatToTimeStr(randomMillis(), format);
+ } else if (type.isDateTimeFamily()) {
+ // date
+ return DateFormat.formatToDateStr(randomMillis(), format);
+ } else if (type.isIntegerFamily()) {
+ // integer
+ return formatNumber(randomInt());
+ } else if (type.isNumberFamily()) {
+ // double
+ return formatNumber(randomDouble());
+ } else {
+ throw new IllegalStateException();
+ }
+ }
+
+ private String formatNumber(double i) {
+ return new DecimalFormat(format).format(i);
+ }
+
+ private int randomInt() {
+ return randStart + rand.nextInt(randEnd - randStart);
+ }
+
+ private double randomDouble() {
+ return randomInt() + rand.nextDouble();
+ }
+
+ private long randomMillis() {
+ int secondsInYear = 3600 * 24 * 365;
+ long year = randStart + rand.nextInt(randEnd - randStart) - 1970;
+ long second = year * secondsInYear + rand.nextInt(secondsInYear);
+ return second * 1000;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ private class IDGen extends Base {
+
+ int next;
+
+ public IDGen(int start) {
+ next = start;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return true;
+ }
+
+ @Override
+ public String next() {
+ return "" + (next++);
+ }
+ }
+
+ private class DiscreteGen extends Base {
+
+ private List<String> values;
+ private Random rand;
+
+ public DiscreteGen(List<String> values) {
+ this.values = values;
+ this.rand = new Random();
+ }
+
+ public DiscreteGen(List<String> values, long seed) {
+ this.values = values;
+ this.rand = new Random(seed);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return true;
+ }
+
+ @Override
+ public String next() {
+ if (values.isEmpty())
+ return null;
+ else
+ return values.get(rand.nextInt(values.size()));
+ }
+ }
+
+ private class CardinalityFilter extends Base {
+
+ private Iterator<String> input;
+ private int card;
+ private TreeSet<String> cache;
+
+ public CardinalityFilter(Iterator<String> input, int card) {
+ assert card > 0;
+ this.input = input;
+ this.card = card;
+ this.cache = new TreeSet<String>();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return input.hasNext();
+ }
+
+ @Override
+ public String next() {
+ String r = input.next();
+
+ if (cache.size() < card) {
+ cache.add(r);
+ return r;
+ }
+
+ r = cache.floor(r);
+ return r == null ? cache.first() : r;
+ }
+ }
+
+ private class AddNullFilter extends Base {
+
+ private Iterator<String> input;
+ private double nullPct;
+ private String nullStr;
+ private Random rand;
+
+ public AddNullFilter(Iterator<String> input, double nullPct, String nullStr) {
+ this.input = input;
+ this.nullPct = nullPct;
+ this.nullStr = nullStr;
+ this.rand = new Random();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return true;
+ }
+
+ @Override
+ public String next() {
+ return rand.nextDouble() < nullPct || !input.hasNext() ? nullStr : input.next();
+ }
+ }
+
+ final private Comparator<String> comp = new Comparator<String>() {
+ @Override
+ public int compare(String s1, String s2) {
+ if (s1 == null) {
+ return s2 == null ? 0 : -1;
+ } else if (s2 == null) {
+ return 1;
+ } else {
+ if (targetCol.getType().isNumberFamily())
+ return Double.compare(Double.parseDouble(s1), Double.parseDouble(s2));
+ else
+ return s1.compareTo(s2);
+ }
+ }
+ };
+
+ private class OrderFilter extends Base {
+
+ private Iterator<String> iter;
+
+ public OrderFilter(Iterator<String> input, boolean unique, int targetRows) {
+ Collection<String> cache = unique ? new TreeSet<String>(comp) : new ArrayList<String>(targetRows);
+ int cap = targetRows * 100;
+ for (int i = 0; cache.size() < targetRows; i++) {
+ cache.add(input.next());
+ if (i >= cap)
+ throw new IllegalStateException();
+ }
+
+ if (cache instanceof List) {
+ Collections.sort((List<String>) cache, comp);
+ }
+
+ iter = cache.iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public String next() {
+ return iter.next();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
new file mode 100644
index 0000000..c325ab0
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.datagen;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.JoinDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.base.Preconditions;
+
+public class ModelDataGenerator {
+
+ final private DataModelDesc model;
+ final private int targetRows;
+ final private ResourceStore outputStore;
+ final private String outputPath;
+
+ boolean outprint = false; // for debug
+
+ public ModelDataGenerator(DataModelDesc model, int nRows) {
+ this(model, nRows, ResourceStore.getStore(model.getConfig()), "/data");
+ }
+
+ public ModelDataGenerator(DataModelDesc model, int nRows, ResourceStore outputStore, String outputPath) {
+ this.model = model;
+ this.targetRows = nRows;
+ this.outputStore = outputStore;
+ this.outputPath = outputPath;
+ }
+
+ public void generate() throws IOException {
+ Set<TableDesc> generated = new HashSet<>();
+ Set<TableDesc> allTableDesc = new LinkedHashSet<>();
+
+ JoinTableDesc[] allTables = model.getJoinTables();
+ for (int i = allTables.length - 1; i >= -1; i--) {
+ TableDesc table = (i == -1) ? model.getRootFactTable().getTableDesc() : allTables[i].getTableRef().getTableDesc();
+ allTableDesc.add(table);
+
+ if (generated.contains(table))
+ continue;
+
+ boolean gen = generateTable(table);
+
+ if (gen)
+ generated.add(table);
+ }
+
+ generateDDL(allTableDesc);
+ }
+
+ private boolean generateTable(TableDesc table) throws IOException {
+ TableGenConfig config = new TableGenConfig(table, this);
+ if (!config.needGen)
+ return false;
+
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ PrintWriter pout = new PrintWriter(new OutputStreamWriter(bout, "UTF-8"));
+
+ generateTableInternal(table, config, pout);
+
+ pout.close();
+ bout.close();
+
+ saveResource(bout.toByteArray(), path(table));
+ return true;
+ }
+
+ private void generateTableInternal(TableDesc table, TableGenConfig config, PrintWriter out) throws IOException {
+ ColumnDesc[] columns = table.getColumns();
+ ColumnGenerator[] colGens = new ColumnGenerator[columns.length];
+ Iterator<String>[] colIters = new Iterator[columns.length];
+
+ // config.rows is either a multiplier (0,1] or an absolute row number
+ int tableRows = (int) ((config.rows > 1) ? config.rows : targetRows * config.rows);
+ tableRows = Math.max(1, tableRows);
+
+ // same seed for all columns, to ensure composite FK columns generate correct pairs
+ long seed = System.currentTimeMillis();
+
+ for (int i = 0; i < columns.length; i++) {
+ colGens[i] = new ColumnGenerator(columns[i], tableRows, this);
+ colIters[i] = colGens[i].generate(seed);
+ }
+
+ for (int i = 0; i < tableRows; i++) {
+ for (int c = 0; c < columns.length; c++) {
+ if (c > 0)
+ out.print(",");
+
+ String v = colIters[c].next();
+ Preconditions.checkState(v == null || !v.contains(","));
+
+ out.print(v);
+ }
+ out.print("\n");
+ }
+ }
+
+ private void generateDDL(Set<TableDesc> tables) throws IOException {
+
+ ByteArrayOutputStream bout = new ByteArrayOutputStream();
+ PrintWriter pout = new PrintWriter(new OutputStreamWriter(bout, "UTF-8"));
+
+ generateDatabaseDDL(tables, pout);
+ generateCreateTableDDL(tables, pout);
+ generateLoadDataDDL(tables, pout);
+
+ pout.close();
+ bout.close();
+
+ saveResource(bout.toByteArray(), path(model));
+ }
+
+ private void generateDatabaseDDL(Set<TableDesc> tables, PrintWriter out) {
+ Set<String> dbs = new HashSet<>();
+ for (TableDesc t : tables) {
+ String db = t.getDatabase();
+ if (StringUtils.isBlank(db) == false && "DEFAULT".equals(db) == false)
+ dbs.add(db);
+ }
+
+ for (String db : dbs) {
+ out.print("CREATE DATABASE IF NOT EXISTS " + db + ";\n");
+ }
+ out.print("\n");
+ }
+
+ private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) {
+ for (TableDesc t : tables) {
+ out.print("DROP TABLE IF EXISTS " + t.getIdentity() + ";\n");
+
+ out.print("CREATE TABLE " + t.getIdentity() + "(" + "\n");
+
+ for (int i = 0; i < t.getColumns().length; i++) {
+ ColumnDesc col = t.getColumns()[i];
+ out.print(" ");
+ if (i > 0) {
+ out.print(",");
+ }
+ out.print(col.getName() + " " + hiveType(col.getType()) + "\n");
+ }
+
+ out.print(")" + "\n");
+ out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
+ out.print("STORED AS TEXTFILE" + ";\n");
+ out.print("\n");
+ }
+ }
+
+ private String hiveType(DataType type) {
+ String t = type.toString();
+ if (t.startsWith("varchar"))
+ return "string";
+ else if (t.startsWith("integer"))
+ return "int";
+ else
+ return t;
+ }
+
+ private void generateLoadDataDDL(Set<TableDesc> tables, PrintWriter out) {
+ for (TableDesc t : tables) {
+ out.print("LOAD DATA LOCAL INPATH '" + t.getIdentity() + ".csv' OVERWRITE INTO TABLE " + t.getIdentity() + ";\n");
+ }
+ }
+
+ public boolean existsInStore(TableDesc table) throws IOException {
+ return outputStore.exists(path(table));
+ }
+
+ public boolean isPK(ColumnDesc col) {
+ for (JoinTableDesc joinTable : model.getJoinTables()) {
+ JoinDesc join = joinTable.getJoin();
+ for (TblColRef pk : join.getPrimaryKeyColumns()) {
+ if (pk.getColumnDesc().equals(col))
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public List<String> getPkValuesIfIsFk(ColumnDesc fk) throws IOException {
+ JoinTableDesc[] joinTables = model.getJoinTables();
+ for (int i = 0; i < joinTables.length; i++) {
+ JoinTableDesc joinTable = joinTables[i];
+ ColumnDesc pk = findPk(joinTable, fk);
+ if (pk == null)
+ continue;
+
+ List<String> pkValues = getPkValues(pk);
+ if (pkValues != null)
+ return pkValues;
+ }
+ return null;
+ }
+
+ private ColumnDesc findPk(JoinTableDesc joinTable, ColumnDesc fk) {
+ TblColRef[] fkCols = joinTable.getJoin().getForeignKeyColumns();
+ for (int i = 0; i < fkCols.length; i++) {
+ if (fkCols[i].getColumnDesc().equals(fk))
+ return joinTable.getJoin().getPrimaryKeyColumns()[i].getColumnDesc();
+ }
+ return null;
+ }
+
+ private List<String> getPkValues(ColumnDesc pk) throws IOException {
+ if (existsInStore(pk.getTable()) == false)
+ return null;
+
+ List<String> r = new ArrayList<>();
+
+ BufferedReader in = new BufferedReader(new InputStreamReader(outputStore.getResource(path(pk.getTable())).inputStream, "UTF-8"));
+ try {
+ String line;
+ while ((line = in.readLine()) != null) {
+ r.add(line.split(",")[pk.getZeroBasedIndex()]);
+ }
+ } finally {
+ IOUtils.closeQuietly(in);
+ }
+ return r;
+ }
+
+ private void saveResource(byte[] content, String path) throws IOException {
+ if (outprint) {
+ System.out.println("Generated " + path);
+ System.out.println(Bytes.toString(content));
+ }
+ outputStore.putResource(path, new ByteArrayInputStream(content), System.currentTimeMillis());
+ }
+
+ private String path(TableDesc table) {
+ return outputPath + "/" + table.getIdentity() + ".csv";
+ }
+
+ private String path(DataModelDesc model) {
+ return outputPath + "/" + "ddl_" + model.getName() + ".sql";
+ }
+
+ public DataModelDesc getModle() {
+ return model;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java
new file mode 100644
index 0000000..be948c1
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/TableGenConfig.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.datagen;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.kylin.metadata.model.TableDesc;
+
+public class TableGenConfig {
+
+ boolean needGen;
+ double rows;
+
+ public TableGenConfig(TableDesc table, ModelDataGenerator modelGen) throws IOException {
+ String dataGen = table.getDataGen();
+ if (dataGen == null && modelGen.existsInStore(table) == false) {
+ dataGen = "";
+ }
+
+ if (dataGen == null)
+ return;
+
+ needGen = true;
+
+ Map<String, String> config = Util.parseEqualCommaPairs(dataGen, "rows");
+
+ // config.rows is either a multiplier (0,1] or an absolute row number
+ rows = Util.parseDouble(config, "rows", modelGen.getModle().isFactTable(table.getIdentity()) ? 1.0 : 20);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/main/java/org/apache/kylin/source/datagen/Util.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/Util.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/Util.java
new file mode 100644
index 0000000..ca27bbf
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/Util.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.source.datagen;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+
+public class Util {
+
+ static Map<String, String> parseEqualCommaPairs(String equalCommaPairs, String defaultKey) {
+ Map<String, String> r = new LinkedHashMap<>();
+
+ if (StringUtils.isBlank(equalCommaPairs))
+ return r;
+
+ for (String s : equalCommaPairs.split(",")) {
+ int equal = s.indexOf("=");
+ if (equal < 0) {
+ if (r.containsKey(defaultKey))
+ r.put(s.trim(), "true");
+ else
+ r.put(defaultKey, s.trim());
+ } else {
+ r.put(s.substring(0, equal).trim(), s.substring(equal + 1).trim());
+ }
+ }
+ return r;
+ }
+
+ static double parseDouble(Map<String, String> config, String key, double dft) {
+ if (config.containsKey(key))
+ return Double.parseDouble(config.get(key));
+ else
+ return dft;
+ }
+
+ static boolean parseBoolean(Map<String, String> config, String key, boolean dft) {
+ if (config.containsKey(key))
+ return Boolean.parseBoolean(config.get(key));
+ else
+ return dft;
+ }
+
+ public static int parseInt(Map<String, String> config, String key, int dft) {
+ if (config.containsKey(key))
+ return Integer.parseInt(config.get(key));
+ else
+ return dft;
+ }
+
+ public static String parseString(Map<String, String> config, String key, String dft) {
+ if (config.containsKey(key))
+ return config.get(key);
+ else
+ return dft;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
deleted file mode 100644
index 1ce17fe..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/aggregation/topn/TopNCounterSerializerTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.aggregation.topn;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.measure.topn.TopNCounter;
-import org.apache.kylin.measure.topn.TopNCounterSerializer;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TopNCounterSerializerTest extends LocalFileMetadataTestCase {
-
- private static TopNCounterSerializer serializer;
-
- @BeforeClass
- public static void setUp() throws Exception {
- staticCreateTestMetadata();
-
- DataType.register("topn");
- serializer = new TopNCounterSerializer(DataType.getType("topn(10)"));
- }
-
- @AfterClass
- public static void after() throws Exception {
- cleanAfterClass();
- }
-
- @Test
- public void testSerialization() {
- TopNCounter<ByteArray> vs = new TopNCounter<ByteArray>(50);
- Integer[] stream = { 1, 1, 2, 9, 1, 2, 3, 7, 7, 1, 3, 1, 1 };
- for (Integer i : stream) {
- vs.offer(new ByteArray(Bytes.toBytes(i)));
- }
- vs.sortAndRetain();
- ByteBuffer out = ByteBuffer.allocate(1024);
- serializer.serialize(vs, out);
-
- byte[] copyBytes = new byte[out.position()];
- System.arraycopy(out.array(), 0, copyBytes, 0, out.position());
-
- ByteBuffer in = ByteBuffer.wrap(copyBytes);
- TopNCounter<ByteArray> vsNew = serializer.deserialize(in);
-
- Assert.assertEquals(vs.toString(), vsNew.toString());
-
- }
-
- @Test
- public void testValueOf() {
- // FIXME need a good unit test for valueOf()
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/test/java/org/apache/kylin/measure/hllc/NewHyperLogLogBenchmarkTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/NewHyperLogLogBenchmarkTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/NewHyperLogLogBenchmarkTest.java
index 586c007..5de2a3a 100644
--- a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/NewHyperLogLogBenchmarkTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/NewHyperLogLogBenchmarkTest.java
@@ -20,6 +20,7 @@ package org.apache.kylin.measure.hllc;
import org.apache.kylin.measure.hllc.HLLCounterOld;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.measure.hllc.RegisterType;
+import org.junit.Ignore;
import org.junit.Test;
import java.nio.ByteBuffer;
@@ -30,6 +31,7 @@ import static org.junit.Assert.assertEquals;
/**
* Created by xiefan on 16-12-12.
*/
+@Ignore("Save UT time")
@SuppressWarnings("deprecation")
public class NewHyperLogLogBenchmarkTest {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d1175d2c/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterSerializerTest.java
new file mode 100644
index 0000000..2daf3b4
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/topn/TopNCounterSerializerTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.topn;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.measure.topn.TopNCounter;
+import org.apache.kylin.measure.topn.TopNCounterSerializer;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TopNCounterSerializerTest extends LocalFileMetadataTestCase {
+
+ private static TopNCounterSerializer serializer;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ staticCreateTestMetadata();
+
+ DataType.register("topn");
+ serializer = new TopNCounterSerializer(DataType.getType("topn(10)"));
+ }
+
+ @AfterClass
+ public static void after() throws Exception {
+ cleanAfterClass();
+ }
+
+ @Test
+ public void testSerialization() {
+ TopNCounter<ByteArray> vs = new TopNCounter<ByteArray>(50);
+ Integer[] stream = { 1, 1, 2, 9, 1, 2, 3, 7, 7, 1, 3, 1, 1 };
+ for (Integer i : stream) {
+ vs.offer(new ByteArray(Bytes.toBytes(i)));
+ }
+ vs.sortAndRetain();
+ ByteBuffer out = ByteBuffer.allocate(1024);
+ serializer.serialize(vs, out);
+
+ byte[] copyBytes = new byte[out.position()];
+ System.arraycopy(out.array(), 0, copyBytes, 0, out.position());
+
+ ByteBuffer in = ByteBuffer.wrap(copyBytes);
+ TopNCounter<ByteArray> vsNew = serializer.deserialize(in);
+
+ Assert.assertEquals(vs.toString(), vsNew.toString());
+
+ }
+
+ @Test
+ public void testValueOf() {
+ // FIXME need a good unit test for valueOf()
+ }
+}
[23/50] [abbrv] kylin git commit: KYLIN 1875 model visualization
Posted by li...@apache.org.
KYLIN 1875 model visualization
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/36a42d86
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/36a42d86
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/36a42d86
Branch: refs/heads/master-cdh5.7
Commit: 36a42d86f84e0b0ba69b44e809e68fcee5437d65
Parents: d5b8c7f
Author: chenzhx <34...@qq.com>
Authored: Mon Dec 12 10:56:17 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 15 18:57:36 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/modelDataModel.js | 10 ++--
webapp/app/js/controllers/models.js | 1 +
webapp/app/js/services/tree.js | 60 ++++++--------------
.../app/partials/modelDesigner/data_model.html | 8 +--
webapp/app/partials/models/model_detail.html | 18 +-----
5 files changed, 26 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/36a42d86/webapp/app/js/controllers/modelDataModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelDataModel.js b/webapp/app/js/controllers/modelDataModel.js
index d17af91..95a4c70 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -21,7 +21,7 @@
KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cubeConfig,MetaModel,SweetAlert,ModelGraphService,$log,TableModel,ModelService,loadingRequest,modelsManager,VdmUtil) {
$scope.modelsManager = modelsManager;
$scope.init = function (){
- $scope.rootFactTable=$scope.modelsManager.selectedModel.fact_table;
+ $scope.FactTable={root:$scope.modelsManager.selectedModel.fact_table};
$scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)]=$scope.modelsManager.selectedModel.fact_table;
$scope.tableAliasMap[$scope.modelsManager.selectedModel.fact_table]=VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table);
$scope.aliasName.push(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table));
@@ -137,12 +137,12 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
};
$scope.changeFactTable = function () {
delete $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)];
- $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.rootFactTable)]=$scope.rootFactTable;
+ $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.FactTable.root)]=$scope.FactTable.root;
delete $scope.tableAliasMap[$scope.modelsManager.selectedModel.fact_table];
- $scope.tableAliasMap[$scope.rootFactTable]=VdmUtil.removeNameSpace($scope.rootFactTable);
+ $scope.tableAliasMap[$scope.FactTable.root]=VdmUtil.removeNameSpace($scope.FactTable.root);
$scope.aliasName.splice($scope.aliasName.indexOf(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)),1);
- $scope.aliasName.push(VdmUtil.removeNameSpace($scope.rootFactTable));
- $scope.modelsManager.selectedModel.fact_table=$scope.rootFactTable;
+ $scope.aliasName.push(VdmUtil.removeNameSpace($scope.FactTable.root));
+ $scope.modelsManager.selectedModel.fact_table=$scope.FactTable.root;
}
$scope.changeJoinTable = function () {
$scope.newLookup.alias=$scope.newLookup.table;
http://git-wip-us.apache.org/repos/asf/kylin/blob/36a42d86/webapp/app/js/controllers/models.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/models.js b/webapp/app/js/controllers/models.js
index 6fe2598..fb2c6d1 100644
--- a/webapp/app/js/controllers/models.js
+++ b/webapp/app/js/controllers/models.js
@@ -177,6 +177,7 @@ KylinApp.controller('ModelsCtrl', function ($scope, $q, $routeParams, $location,
};
var ModelDetailModalCtrl = function ($scope, $location, $modalInstance, scope) {
+ modelsManager.selectedModel.visiblePage='metadata';
$scope.cancel = function () {
$modalInstance.dismiss('cancel');
};
http://git-wip-us.apache.org/repos/asf/kylin/blob/36a42d86/webapp/app/js/services/tree.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/tree.js b/webapp/app/js/services/tree.js
index 3b444c6..08e290b 100755
--- a/webapp/app/js/services/tree.js
+++ b/webapp/app/js/services/tree.js
@@ -17,8 +17,8 @@
*/
KylinApp.service('ModelGraphService', function (VdmUtil) {
- var tablesNodeList=[];
- var aliasList=[];
+ var tablesNodeList={};
+ // var aliasList=[];
var margin = {top: 20, right: 100, bottom: 20, left: 100},
width = 1100 - margin.right - margin.left,
height = 600;
@@ -42,12 +42,12 @@ KylinApp.service('ModelGraphService', function (VdmUtil) {
"name": VdmUtil.removeNameSpace(model.fact_table),
"children": []
};
- tablesNodeList.push(graphData);
- aliasList.push(graphData.name);
+ tablesNodeList[graphData.name]=graphData;
+ // aliasList.push(graphData.name);
model.graph = (!!model.graph) ? model.graph : {};
- angular.forEach(model.lookups, function (lookup, index) {
- if (lookup.join && aliasList.indexOf(lookup.alias)==-1) {
+ angular.forEach(model.lookups,function (lookup, index) {
+ if (lookup.join && lookup.join.primary_key.length > 0) {
var dimensionNode={
"type": lookup.kind,
"name": lookup.alias,
@@ -55,49 +55,21 @@ KylinApp.service('ModelGraphService', function (VdmUtil) {
"children": [],
"_children": []
}
- aliasList.push(dimensionNode.name);
- tablesNodeList.push(dimensionNode);
+ // aliasList.push(dimensionNode.name);
+ tablesNodeList[dimensionNode.name]=dimensionNode;
}
});
- angular.forEach(model.lookups, function (lookup) {
- if (lookup.join && lookup.join.primary_key.length > 0) {
- VdmUtil.getNameSpaceAliasName(lookup.join.primary_key[0]);
- VdmUtil.getNameSpaceAliasName(lookup.join.foreign_key[0]);
- }
- });
-
-
-/* Loop through the graphData.children array to find out: If the LKP table is already existed *//*
-
- for(var j = 0; j < graphData.children.length; j++ ) {
- if(graphData.children[j].name == lookup.alias){
- dimensionNode = graphData.children[j];
- break;
- }
- }
- */
-/* If not existed, create dimensionNode and push it *//*
-
- if(j == graphData.children.length) {
- dimensionNode = {
- "type": "dimension",
- "name": lookup.alias,
- "join": lookup.join,
- "children": [],
- "_children": []
- };
+ angular.forEach(model.lookups,function(joinTable){
+ if (joinTable.join && joinTable.join.primary_key.length > 0) {
+ var fkAliasName=VdmUtil.getNameSpaceAliasName(joinTable.join.foreign_key[0]);
+ var pkAliasName=VdmUtil.getNameSpaceAliasName(joinTable.join.primary_key[0]);
+ tablesNodeList[fkAliasName].children.push(tablesNodeList[pkAliasName]);
}
+ });
- if(j == graphData.children.length) {
- graphData.children.push(dimensionNode);
- }
-
- }
-*/
-
- angular.forEach(model.dimensions, function (dimension, index) {
+/* angular.forEach(model.dimensions, function (dimension, index) {
// for dimension on lookup table
if(model.fact_table!==dimension.table){
var lookup = _.find(graphData.children,function(item){
@@ -117,7 +89,7 @@ KylinApp.service('ModelGraphService', function (VdmUtil) {
}
});
};
- });
+ });*/
model.graph.columnsCount = 0;
model.graph.tree = tree;
model.graph.root = graphData;
http://git-wip-us.apache.org/repos/asf/kylin/blob/36a42d86/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index 82be964..f1192f4 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -23,10 +23,10 @@
<div class="form-group" ng-class="{'required':state.mode=='edit'}">
<div class="row">
<label class="col-xs-12 col-sm-2 control-label concube.detailtrol-label no-padding-right font-color-default">
- <b>Fact Table</b>
+ <b>Root Fact Table</b>
</label>
<div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.data_model_form.table_name.$invalid && (forms.data_model_form.table_name.$dirty||forms.data_model_form.$submitted)}">
- <select chosen ng-model="$parent.rootFactTable" ng-if="state.mode=='edit'"
+ <select chosen ng-model="FactTable.root" ng-if="state.mode=='edit'"
ng-options="table.name as table.name for table in tableModel.selectProjectTables"
style="width:100%;" ng-change="changeFactTable()"
name="table_name"
@@ -35,9 +35,7 @@
class="chosen-select">
<option value=""> -- Select Fact Table -- </option>
</select>
-
- <small class="help-block" ng-show="forms.data_model_form.innerform.typeahead.$error.required && (forms.data_model_form.innerform.typeahead.$dirty||forms.data_model_form.$submitted)">The fact table is required</small>
- <span ng-if="state.mode=='view'">{{$parent.rootFactTable}}</span>
+ <span ng-if="state.mode=='view'">{{modelsManager.selectedModel.fact_table}}</span>
</div>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/36a42d86/webapp/app/partials/models/model_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/models/model_detail.html b/webapp/app/partials/models/model_detail.html
index dcf782a..2549da2 100644
--- a/webapp/app/partials/models/model_detail.html
+++ b/webapp/app/partials/models/model_detail.html
@@ -32,23 +32,7 @@
<li class="{{modelsManager.selectedModel.visiblePage=='json_model'? 'active':''}}">
<a href="" ng-click="modelsManager.selectedModel.visiblePage='json_model';">JSON</a>
</li>
- <li class="dropdown" ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission(modelsManager.selectedModel, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask)">
- <a class="dropdown-toggle highlight-bule" data-toggle="dropdown" href="#" aria-expanded="true">
- <i class="fa fa-star"> Action </i> <span class="caret"></span>
- </a>
- <ul class="dropdown-menu">
- <li ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission(modelsManager.selectedModel, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask)">
- <a href="models/edit/{{modelsManager.selectedModel.name}}" data-widget="collapse">Edit</a>
- </li>
- <li ng-if="userService.hasRole('ROLE_ADMIN')">
- <a ng-click="dropModel(modelsManager.selectedModel)">Drop</a>
- </li>
- <li role="presentation" class="divider"></li>
- <li ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission(modelsManager.selectedModel, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask)">
- <a href="cubes/add/{{modelsManager.selectedModel.name}}" data-widget="collapse">Create Cube</a>
- </li>
- </ul>
- </li>
+
</ul>
<div class="model-detail" ng-if="!modelsManager.selectedModel.visiblePage || modelsManager.selectedModel.visiblePage=='metadata'">
[10/50] [abbrv] kylin git commit: minor,
fix cubedesc and datamodeldesc UT
Posted by li...@apache.org.
minor, fix cubedesc and datamodeldesc UT
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/53036513
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/53036513
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/53036513
Branch: refs/heads/master-cdh5.7
Commit: 5303651316ee6911e312130624534315267ef99a
Parents: 05f35f0
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Dec 13 13:03:22 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Dec 13 13:05:51 2016 +0800
----------------------------------------------------------------------
core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java | 1 -
core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java | 1 +
.../main/java/org/apache/kylin/metadata/model/DataModelDesc.java | 1 -
.../java/org/apache/kylin/metadata/model/DataModelDescTest.java | 1 +
4 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/53036513/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 6088bb4..f6b68af 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -1129,7 +1129,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
newCubeDesc.setAggregationGroups(cubeDesc.getAggregationGroups());
newCubeDesc.setOverrideKylinProps(cubeDesc.getOverrideKylinProps());
newCubeDesc.setConfig((KylinConfigExt) cubeDesc.getConfig());
- newCubeDesc.setLastModified(cubeDesc.getLastModified());
newCubeDesc.setPartitionOffsetStart(cubeDesc.getPartitionOffsetStart());
newCubeDesc.updateRandomUuid();
return newCubeDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/53036513/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 78cf222..1daa748 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -262,6 +262,7 @@ public class CubeDescTest extends LocalFileMetadataTestCase {
// uuid is different, set to equals for json comparison
copyDesc.setUuid(desc.getUuid());
+ copyDesc.setLastModified(desc.getLastModified());
String descStr = JsonUtil.writeValueAsIndentString(desc);
String copyStr = JsonUtil.writeValueAsIndentString(copyDesc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/53036513/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 99636d7..0be240e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -526,7 +526,6 @@ public class DataModelDesc extends RootPersistentEntity {
copy.filterCondition = orig.filterCondition;
copy.partitionDesc = PartitionDesc.getCopyOf(orig.getPartitionDesc());
copy.capacity = orig.capacity;
- copy.lastModified = orig.lastModified;
copy.updateRandomUuid();
return copy;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/53036513/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
index 15bd29c..0cfa1f4 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
@@ -47,6 +47,7 @@ public class DataModelDescTest extends LocalFileMetadataTestCase {
// uuid is different, set to equals for json comparison
copyDesc.setUuid(desc.getUuid());
+ copyDesc.setLastModified(desc.getLastModified());
String descStr = JsonUtil.writeValueAsIndentString(desc);
String copyStr = JsonUtil.writeValueAsIndentString(copyDesc);
[29/50] [abbrv] kylin git commit: minor, sub_query/query02.sql fix CI
Posted by li...@apache.org.
minor, sub_query/query02.sql fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f2377db9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f2377db9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f2377db9
Branch: refs/heads/master-cdh5.7
Commit: f2377db9ff2687098c0bd8a805449547b7dcd436
Parents: 911bdd0
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 16 15:53:30 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Dec 16 15:53:30 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/dimension/DimensionEncodingFactory.java | 2 +-
.../java/org/apache/kylin/metadata/model/DataModelDesc.java | 4 ++--
kylin-it/src/test/resources/query/sql_subquery/query02.sql | 8 ++++++--
3 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2377db9/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 4954ead..3eddba7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -71,7 +71,7 @@ public abstract class DimensionEncodingFactory {
if (factoryMap == null)
initFactoryMap();
- Map<String, Integer> result = Maps.newHashMap();
+ Map<String, Integer> result = Maps.newTreeMap();
for (Pair<String, Integer> p : factoryMap.keySet()) {
if (result.containsKey(p.getFirst())) {
if (result.get(p.getFirst()) > p.getSecond()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2377db9/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 0b494e3..571b196 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -393,7 +393,7 @@ public class DataModelDesc extends RootPersistentEntity {
col = findColumn(pks[i]);
}
if (col == null || col.getTableRef().equals(dimTable) == false) {
- throw new IllegalStateException("Can't find column " + pks[i] + " in table " + dimTable.getTableIdentity());
+ throw new IllegalStateException("Can't find PK column " + pks[i] + " in table " + dimTable);
}
pks[i] = col.getIdentity();
pkCols[i] = col;
@@ -406,7 +406,7 @@ public class DataModelDesc extends RootPersistentEntity {
for (int i = 0; i < fks.length; i++) {
TblColRef col = findColumn(fks[i]);
if (col == null) {
- throw new IllegalStateException("Can't find column " + fks[i] + " in table " + this.getRootFactTable());
+ throw new IllegalStateException("Can't find FK column " + fks[i]);
}
fks[i] = col.getIdentity();
fkCols[i] = col;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f2377db9/kylin-it/src/test/resources/query/sql_subquery/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_subquery/query02.sql b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
index e6751b7..b0dcf73 100644
--- a/kylin-it/src/test/resources/query/sql_subquery/query02.sql
+++ b/kylin-it/src/test/resources/query/sql_subquery/query02.sql
@@ -25,7 +25,11 @@ FROM
test_cal_dt.week_beg_dt
,test_kylin_fact.price
from test_kylin_fact
- inner join edw.test_cal_dt as test_cal_dt
- on test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ inner JOIN edw.test_sites as test_sites
+ ON test_kylin_fact.lstg_site_id = test_sites.site_id inner join edw.test_cal_dt as test_cal_dt
) t
group by week_beg_dt
[13/50] [abbrv] kylin git commit: KYLIN-1832 code review
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 76212c8..6e894dd 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -83,7 +83,7 @@ import org.apache.kylin.engine.spark.cube.DefaultTupleConverter;
import org.apache.kylin.engine.spark.util.IteratorUtils;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.MeasureAggregators;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -241,15 +241,15 @@ public class SparkCubing extends AbstractApplication {
}
}
- private Map<Long, HyperLogLogPlusCounterNew> sampling(final JavaRDD<List<String>> rowJavaRDD, final String cubeName, String segmentId) throws Exception {
+ private Map<Long, HLLCounter> sampling(final JavaRDD<List<String>> rowJavaRDD, final String cubeName, String segmentId) throws Exception {
CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).reloadCubeLocal(cubeName);
CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
CubeDesc cubeDesc = cubeInstance.getDescriptor();
CuboidScheduler cuboidScheduler = new CuboidScheduler(cubeDesc);
List<Long> allCuboidIds = cuboidScheduler.getAllCuboidIds();
- final HashMap<Long, HyperLogLogPlusCounterNew> zeroValue = Maps.newHashMap();
+ final HashMap<Long, HLLCounter> zeroValue = Maps.newHashMap();
for (Long id : allCuboidIds) {
- zeroValue.put(id, new HyperLogLogPlusCounterNew(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
+ zeroValue.put(id, new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision()));
}
CubeJoinedFlatTableEnrich flatDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
@@ -278,12 +278,12 @@ public class SparkCubing extends AbstractApplication {
row_hashcodes[i] = new ByteArray();
}
- final HashMap<Long, HyperLogLogPlusCounterNew> samplingResult = rowJavaRDD.aggregate(zeroValue, new Function2<HashMap<Long, HyperLogLogPlusCounterNew>, List<String>, HashMap<Long, HyperLogLogPlusCounterNew>>() {
+ final HashMap<Long, HLLCounter> samplingResult = rowJavaRDD.aggregate(zeroValue, new Function2<HashMap<Long, HLLCounter>, List<String>, HashMap<Long, HLLCounter>>() {
final HashFunction hashFunction = Hashing.murmur3_128();
@Override
- public HashMap<Long, HyperLogLogPlusCounterNew> call(HashMap<Long, HyperLogLogPlusCounterNew> v1, List<String> v2) throws Exception {
+ public HashMap<Long, HLLCounter> call(HashMap<Long, HLLCounter> v1, List<String> v2) throws Exception {
for (int i = 0; i < nRowKey; i++) {
Hasher hc = hashFunction.newHasher();
String colValue = v2.get(rowKeyColumnIndexes[i]);
@@ -296,7 +296,7 @@ public class SparkCubing extends AbstractApplication {
for (Map.Entry<Long, Integer[]> entry : allCuboidsBitSet.entrySet()) {
Hasher hc = hashFunction.newHasher();
- HyperLogLogPlusCounterNew counter = v1.get(entry.getKey());
+ HLLCounter counter = v1.get(entry.getKey());
final Integer[] cuboidBitSet = entry.getValue();
for (int position = 0; position < cuboidBitSet.length; position++) {
hc.putBytes(row_hashcodes[cuboidBitSet[position]].array());
@@ -305,14 +305,14 @@ public class SparkCubing extends AbstractApplication {
}
return v1;
}
- }, new Function2<HashMap<Long, HyperLogLogPlusCounterNew>, HashMap<Long, HyperLogLogPlusCounterNew>, HashMap<Long, HyperLogLogPlusCounterNew>>() {
+ }, new Function2<HashMap<Long, HLLCounter>, HashMap<Long, HLLCounter>, HashMap<Long, HLLCounter>>() {
@Override
- public HashMap<Long, HyperLogLogPlusCounterNew> call(HashMap<Long, HyperLogLogPlusCounterNew> v1, HashMap<Long, HyperLogLogPlusCounterNew> v2) throws Exception {
+ public HashMap<Long, HLLCounter> call(HashMap<Long, HLLCounter> v1, HashMap<Long, HLLCounter> v2) throws Exception {
Preconditions.checkArgument(v1.size() == v2.size());
Preconditions.checkArgument(v1.size() > 0);
- for (Map.Entry<Long, HyperLogLogPlusCounterNew> entry : v1.entrySet()) {
- final HyperLogLogPlusCounterNew counter1 = entry.getValue();
- final HyperLogLogPlusCounterNew counter2 = v2.get(entry.getKey());
+ for (Map.Entry<Long, HLLCounter> entry : v1.entrySet()) {
+ final HLLCounter counter1 = entry.getValue();
+ final HLLCounter counter2 = v2.get(entry.getKey());
counter1.merge(Preconditions.checkNotNull(counter2, "counter cannot be null"));
}
return v1;
@@ -470,7 +470,7 @@ public class SparkCubing extends AbstractApplication {
ClassUtil.addClasspath(confPath);
}
- private byte[][] createHTable(String cubeName, String segmentId, Map<Long, HyperLogLogPlusCounterNew> samplingResult) throws Exception {
+ private byte[][] createHTable(String cubeName, String segmentId, Map<Long, HLLCounter> samplingResult) throws Exception {
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName);
final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
@@ -614,7 +614,7 @@ public class SparkCubing extends AbstractApplication {
}
});
- final Map<Long, HyperLogLogPlusCounterNew> samplingResult = sampling(rowJavaRDD, cubeName, segmentId);
+ final Map<Long, HLLCounter> samplingResult = sampling(rowJavaRDD, cubeName, segmentId);
final byte[][] splitKeys = createHTable(cubeName, segmentId, samplingResult);
final String hfile = build(rowJavaRDD, cubeName, segmentId, splitKeys);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
index 230249f..f046f78 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
@@ -35,7 +35,7 @@ import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -46,7 +46,7 @@ import org.apache.kylin.metadata.model.TableDesc;
*/
public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritable, BytesWritable> {
- private Map<Integer, HyperLogLogPlusCounterNew> hllcMap = new HashMap<Integer, HyperLogLogPlusCounterNew>();
+ private Map<Integer, HLLCounter> hllcMap = new HashMap<Integer, HLLCounter>();
public static final String DEFAULT_DELIM = ",";
private int counter = 0;
@@ -87,9 +87,9 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab
counter++;
}
- private HyperLogLogPlusCounterNew getHllc(Integer key) {
+ private HLLCounter getHllc(Integer key) {
if (!hllcMap.containsKey(key)) {
- hllcMap.put(key, new HyperLogLogPlusCounterNew());
+ hllcMap.put(key, new HLLCounter());
}
return hllcMap.get(key);
}
@@ -100,7 +100,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab
ByteBuffer buf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
while (it.hasNext()) {
int key = it.next();
- HyperLogLogPlusCounterNew hllc = hllcMap.get(key);
+ HLLCounter hllc = hllcMap.get(key);
buf.clear();
hllc.writeRegisters(buf);
buf.flip();
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
index 32cc6d9..0648960 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducer.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
/**
* @author Jack
@@ -41,7 +41,7 @@ import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWritable, IntWritable, LongWritable> {
public static final int ONE = 1;
- private Map<Integer, HyperLogLogPlusCounterNew> hllcMap = new HashMap<Integer, HyperLogLogPlusCounterNew>();
+ private Map<Integer, HLLCounter> hllcMap = new HashMap<Integer, HLLCounter>();
@Override
protected void setup(Context context) throws IOException {
@@ -53,16 +53,16 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri
int skey = key.get();
for (BytesWritable v : values) {
ByteBuffer buffer = ByteBuffer.wrap(v.getBytes());
- HyperLogLogPlusCounterNew hll = new HyperLogLogPlusCounterNew();
+ HLLCounter hll = new HLLCounter();
hll.readRegisters(buffer);
getHllc(skey).merge(hll);
hll.clear();
}
}
- private HyperLogLogPlusCounterNew getHllc(Integer key) {
+ private HLLCounter getHllc(Integer key) {
if (!hllcMap.containsKey(key)) {
- hllcMap.put(key, new HyperLogLogPlusCounterNew());
+ hllcMap.put(key, new HLLCounter());
}
return hllcMap.get(key);
}
@@ -78,7 +78,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri
it = keys.iterator();
while (it.hasNext()) {
int key = it.next();
- HyperLogLogPlusCounterNew hllc = hllcMap.get(key);
+ HLLCounter hllc = hllcMap.get(key);
ByteBuffer buf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
buf.clear();
hllc.writeRegisters(buf);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/source-hive/src/test/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducerTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducerTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducerTest.java
index 410543a..c32e76d 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducerTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityReducerTest.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.apache.hadoop.mrunit.types.Pair;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.junit.Before;
import org.junit.Test;
@@ -57,7 +57,7 @@ public class ColumnCardinalityReducerTest {
}
private byte[] getBytes(String str) throws IOException {
- HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew();
+ HLLCounter hllc = new HLLCounter();
StringTokenizer tokenizer = new StringTokenizer(str, ColumnCardinalityMapper.DEFAULT_DELIM);
int i = 0;
while (tokenizer.hasMoreTokens()) {
[37/50] [abbrv] kylin git commit: minor,
add equals and hashcode to RowKeyColDesc
Posted by li...@apache.org.
minor, add equals and hashcode to RowKeyColDesc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7446c56b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7446c56b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7446c56b
Branch: refs/heads/master-cdh5.7
Commit: 7446c56b1aca87fce9868c9e1f0674299b05c405
Parents: 1d53ce9
Author: lidongsjtu <li...@apache.org>
Authored: Sat Dec 17 13:46:22 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Dec 19 12:57:14 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/cube/model/RowKeyColDesc.java | 26 ++++++++++++++++++++
1 file changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7446c56b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 3b49323..ef34a9b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -155,8 +155,34 @@ public class RowKeyColDesc {
}
@Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((column == null) ? 0 : column.hashCode());
+ return result;
+ }
+
+ @Override
public String toString() {
return Objects.toStringHelper(this).add("column", column).add("encoding", encoding).toString();
}
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ RowKeyColDesc that = (RowKeyColDesc) o;
+
+ if (column != null ? !column.equals(that.column) : that.column != null) {
+ return false;
+ }
+
+ return true;
+ }
}
\ No newline at end of file
[27/50] [abbrv] kylin git commit: minor, fix NPE in OLAPTable
Posted by li...@apache.org.
minor, fix NPE in OLAPTable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3003d9fb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3003d9fb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3003d9fb
Branch: refs/heads/master-cdh5.7
Commit: 3003d9fba7c0b20a74785cbc431f9e708e775418
Parents: 1c2c43c
Author: Yang Li <li...@apache.org>
Authored: Fri Dec 16 07:57:33 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Fri Dec 16 07:57:33 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/query/schema/OLAPTable.java | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3003d9fb/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 06ffdba..c33c4ed 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -43,13 +43,13 @@ import org.apache.calcite.schema.impl.AbstractTableQueryable;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.SqlTypeUtil;
import org.apache.calcite.util.ImmutableBitSet;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -198,14 +198,13 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
}
}
//2. All integer measures in non-cube realizations
- MetadataManager metadataManager = MetadataManager.getInstance(olapSchema.getConfig());
for (IRealization realization : mgr.listAllRealizations(olapSchema.getProjectName())) {
if (realization.getType() == RealizationType.INVERTED_INDEX && realization.getModel().isFactTable(sourceTable.getIdentity())) {
- DataModelDesc dataModelDesc = realization.getModel();
- for (String metricColumn : dataModelDesc.getMetrics()) {
- ColumnDesc columnDesc = metadataManager.getColumnDesc(dataModelDesc.getRootFactTable().getTableIdentity() + "." + metricColumn);
- if (columnDesc.getType().isIntegerFamily() && !columnDesc.getType().isBigInt())
- updateColumns.add(columnDesc);
+ DataModelDesc model = realization.getModel();
+ for (String metricColumn : model.getMetrics()) {
+ TblColRef col = model.findColumn(metricColumn);
+ if (col.getType().isIntegerFamily() && !col.getType().isBigInt())
+ updateColumns.add(col.getColumnDesc());
}
}
}
[49/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
Posted by li...@apache.org.
KYLIN-1528 Create a branch for v1.5 with HBase 1.x API
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9690649b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9690649b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9690649b
Branch: refs/heads/master-cdh5.7
Commit: 9690649b858718b7314900e457272d6d8b626d31
Parents: c72e232
Author: shaofengshi <sh...@apache.org>
Authored: Wed Mar 23 17:07:05 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Dec 20 19:23:06 2016 +0800
----------------------------------------------------------------------
examples/test_case_data/sandbox/hbase-site.xml | 19 +---
.../kylin/provision/BuildCubeWithEngine.java | 12 +--
pom.xml | 12 +--
.../kylin/rest/security/AclHBaseStorage.java | 4 +-
.../rest/security/MockAclHBaseStorage.java | 8 +-
.../apache/kylin/rest/security/MockHTable.java | 95 ++++----------------
.../rest/security/RealAclHBaseStorage.java | 9 +-
.../apache/kylin/rest/service/AclService.java | 25 +++---
.../apache/kylin/rest/service/CubeService.java | 35 +++-----
.../apache/kylin/rest/service/QueryService.java | 24 +++--
.../apache/kylin/rest/service/UserService.java | 17 ++--
.../kylin/storage/hbase/HBaseConnection.java | 44 ++++-----
.../kylin/storage/hbase/HBaseResourceStore.java | 31 +++----
.../storage/hbase/cube/SimpleHBaseStore.java | 20 ++---
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 13 +--
.../storage/hbase/cube/v2/CubeHBaseScanRPC.java | 9 +-
.../coprocessor/endpoint/CubeVisitService.java | 4 +-
.../storage/hbase/steps/CubeHTableUtil.java | 16 ++--
.../storage/hbase/steps/DeprecatedGCStep.java | 23 ++---
.../storage/hbase/steps/HBaseCuboidWriter.java | 7 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 23 ++---
.../storage/hbase/util/CleanHtableCLI.java | 12 +--
.../storage/hbase/util/CubeMigrationCLI.java | 36 ++++----
.../hbase/util/CubeMigrationCheckCLI.java | 17 ++--
.../hbase/util/DeployCoprocessorCLI.java | 27 +++---
.../hbase/util/ExtendCubeToHybridCLI.java | 8 +-
.../hbase/util/GridTableHBaseBenchmark.java | 34 +++----
.../kylin/storage/hbase/util/HBaseClean.java | 18 ++--
.../hbase/util/HBaseRegionSizeCalculator.java | 35 ++++----
.../kylin/storage/hbase/util/HBaseUsage.java | 9 +-
.../storage/hbase/util/HbaseStreamingInput.java | 30 +++----
.../hbase/util/HtableAlterMetadataCLI.java | 9 +-
.../storage/hbase/util/OrphanHBaseCleanJob.java | 19 ++--
.../kylin/storage/hbase/util/PingHBaseCLI.java | 15 ++--
.../kylin/storage/hbase/util/RowCounterCLI.java | 11 +--
.../storage/hbase/util/StorageCleanupJob.java | 20 +++--
.../storage/hbase/util/UpdateHTableHostCLI.java | 17 ++--
.../org/apache/kylin/tool/CubeMigrationCLI.java | 19 ++--
.../kylin/tool/ExtendCubeToHybridCLI.java | 8 +-
39 files changed, 363 insertions(+), 431 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 46d5345..734908e 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -190,22 +190,5 @@
<name>zookeeper.znode.parent</name>
<value>/hbase-unsecure</value>
</property>
- <property>
- <name>hbase.client.pause</name>
- <value>100</value>
- <description>General client pause value. Used mostly as value to wait
- before running a retry of a failed get, region lookup, etc.
- See hbase.client.retries.number for description of how we backoff from
- this initial pause amount and how this pause works w/ retries.</description>
- </property>
- <property>
- <name>hbase.client.retries.number</name>
- <value>5</value>
- <description>Maximum retries. Used as maximum for all retryable
- operations such as the getting of a cell's value, starting a row update,
- etc. Retry interval is a rough function based on hbase.client.pause. At
- first we retry at this interval but then with backoff, we pretty quickly reach
- retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
- ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
- </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
index bfbeb70..cecece5 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithEngine.java
@@ -35,8 +35,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
@@ -58,6 +57,7 @@ import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.source.ISource;
import org.apache.kylin.source.SourceFactory;
import org.apache.kylin.source.SourcePartition;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.apache.kylin.tool.StorageCleanupJob;
@@ -388,10 +388,10 @@ public class BuildCubeWithEngine {
@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
- Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
- String tableName = segment.getStorageLocationIdentifier();
- try (HTable table = new HTable(conf, tableName)) {
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
+ try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
+ String tableName = segment.getStorageLocationIdentifier();
+
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
long totalSize = 0;
for (Long size : sizeMap.values()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 51479c8..6d3425e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,20 +46,20 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.6.0</hadoop2.version>
- <yarn.version>2.6.0</yarn.version>
+ <hadoop2.version>2.7.1</hadoop2.version>
+ <yarn.version>2.7.1</yarn.version>
<!-- Hive versions -->
- <hive.version>0.14.0</hive.version>
- <hive-hcatalog.version>0.14.0</hive-hcatalog.version>
+ <hive.version>1.2.1</hive.version>
+ <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>0.98.8-hadoop2</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
<kafka.version>0.10.0.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
<zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.6.0</curator.version>
+ <curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
<guava.version>14.0</guava.version>
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
index ea68855..8095bf8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/AclHBaseStorage.java
@@ -20,7 +20,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
/**
*/
@@ -36,6 +36,6 @@ public interface AclHBaseStorage {
String prepareHBaseTable(Class<?> clazz) throws IOException;
- HTableInterface getTable(String tableName) throws IOException;
+ Table getTable(String tableName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
index d9326f5..cc76b87 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java
@@ -21,7 +21,7 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -34,8 +34,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
private static final String aclTableName = "MOCK-ACL-TABLE";
private static final String userTableName = "MOCK-USER-TABLE";
- private HTableInterface mockedAclTable;
- private HTableInterface mockedUserTable;
+ private Table mockedAclTable;
+ private Table mockedUserTable;
private RealAclHBaseStorage realAcl;
public MockAclHBaseStorage() {
@@ -65,7 +65,7 @@ public class MockAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (realAcl != null) {
return realAcl.getTable(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
index d0aa0ed..972eea9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockHTable.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -91,7 +91,7 @@ import com.google.protobuf.ServiceException;
* <li>remove some methods for loading data, checking values ...</li>
* </ul>
*/
-public class MockHTable implements HTableInterface {
+public class MockHTable implements Table {
private final String tableName;
private final List<String> columnFamilies = new ArrayList<>();
@@ -114,14 +114,6 @@ public class MockHTable implements HTableInterface {
this.columnFamilies.add(columnFamily);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public byte[] getTableName() {
- return tableName.getBytes();
- }
-
@Override
public TableName getName() {
return null;
@@ -200,8 +192,8 @@ public class MockHTable implements HTableInterface {
}
@Override
- public Boolean[] exists(List<Get> gets) throws IOException {
- return new Boolean[0];
+ public boolean[] existsAll(List<Get> list) throws IOException {
+ return new boolean[0];
}
/**
@@ -306,15 +298,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public Result getRowOrBefore(byte[] row, byte[] family) throws IOException {
- // FIXME: implement
- return null;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public ResultScanner getScanner(Scan scan) throws IOException {
final List<Result> ret = new ArrayList<Result>();
byte[] st = scan.getStartRow();
@@ -446,7 +429,7 @@ public class MockHTable implements HTableInterface {
*/
}
if (filter.hasFilterRow() && !filteredOnRowKey) {
- filter.filterRow(nkvs);
+ filter.filterRow();
}
if (filter.filterRow() || filteredOnRowKey) {
nkvs.clear();
@@ -535,6 +518,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndPut(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Put put) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -555,7 +543,7 @@ public class MockHTable implements HTableInterface {
continue;
}
for (KeyValue kv : delete.getFamilyMap().get(family)) {
- if (kv.isDeleteFamily()) {
+ if (kv.isDelete()) {
data.get(row).get(kv.getFamily()).clear();
} else {
data.get(row).get(kv.getFamily()).remove(kv.getQualifier());
@@ -592,6 +580,11 @@ public class MockHTable implements HTableInterface {
return false;
}
+ @Override
+ public boolean checkAndDelete(byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, byte[] bytes3, Delete delete) throws IOException {
+ return false;
+ }
+
/**
* {@inheritDoc}
*/
@@ -605,7 +598,7 @@ public class MockHTable implements HTableInterface {
*/
@Override
public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount) throws IOException {
- return incrementColumnValue(row, family, qualifier, amount, true);
+ return incrementColumnValue(row, family, qualifier, amount, null);
}
@Override
@@ -617,37 +610,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if (check(row, family, qualifier, null)) {
- Put put = new Put(row);
- put.add(family, qualifier, Bytes.toBytes(amount));
- put(put);
- return amount;
- }
- long newValue = Bytes.toLong(data.get(row).get(family).get(qualifier).lastEntry().getValue()) + amount;
- data.get(row).get(family).get(qualifier).put(System.currentTimeMillis(), Bytes.toBytes(newValue));
- return newValue;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public boolean isAutoFlush() {
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void flushCommits() throws IOException {
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public void close() throws IOException {
}
@@ -673,29 +635,6 @@ public class MockHTable implements HTableInterface {
* {@inheritDoc}
*/
@Override
- public void setAutoFlush(boolean autoFlush) {
- throw new NotImplementedException();
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void setAutoFlush(boolean autoFlush, boolean clearBufferOnFail) {
- throw new NotImplementedException();
-
- }
-
- @Override
- public void setAutoFlushTo(boolean autoFlush) {
- throw new NotImplementedException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
public long getWriteBufferSize() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index 1d520c4..d1a1384 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -21,7 +21,8 @@ package org.apache.kylin.rest.security;
import java.io.IOException;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.QueryService;
@@ -58,11 +59,11 @@ public class RealAclHBaseStorage implements AclHBaseStorage {
}
@Override
- public HTableInterface getTable(String tableName) throws IOException {
+ public Table getTable(String tableName) throws IOException {
if (StringUtils.equals(tableName, aclTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(aclTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(aclTableName));
} else if (StringUtils.equals(tableName, userTableName)) {
- return HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ return HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
} else {
throw new IllegalStateException("getTable failed" + tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index d693a67..3e3efec 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -33,7 +33,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
@@ -124,7 +124,7 @@ public class AclService implements MutableAclService {
@Override
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -173,7 +173,7 @@ public class AclService implements MutableAclService {
@Override
public Map<ObjectIdentity, Acl> readAclsById(List<ObjectIdentity> oids, List<Sid> sids) throws NotFoundException {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
- HTableInterface htable = null;
+ Table htable = null;
Result result = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -226,17 +226,16 @@ public class AclService implements MutableAclService {
Authentication auth = SecurityContextHolder.getContext().getAuthentication();
PrincipalSid sid = new PrincipalSid(auth);
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
Put put = new Put(Bytes.toBytes(String.valueOf(objectIdentity.getIdentifier())));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_TYPE_COLUMN), Bytes.toBytes(objectIdentity.getType()));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_OWNER_COLUMN), sidSerializer.serialize(new SidInfo(sid)));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN), Bytes.toBytes(true));
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
@@ -250,7 +249,7 @@ public class AclService implements MutableAclService {
@Override
public void deleteAcl(ObjectIdentity objectIdentity, boolean deleteChildren) throws ChildrenExistException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -266,7 +265,6 @@ public class AclService implements MutableAclService {
}
htable.delete(delete);
- htable.flushCommits();
logger.debug("ACL of " + objectIdentity + " deleted successfully.");
} catch (IOException e) {
@@ -284,7 +282,7 @@ public class AclService implements MutableAclService {
throw e;
}
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(aclTableName);
@@ -295,17 +293,16 @@ public class AclService implements MutableAclService {
Put put = new Put(Bytes.toBytes(String.valueOf(acl.getObjectIdentity().getIdentifier())));
if (null != acl.getParentAcl()) {
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_INFO_FAMILY), Bytes.toBytes(ACL_INFO_FAMILY_PARENT_COLUMN), domainObjSerializer.serialize(new DomainObjectInfo(acl.getParentAcl().getObjectIdentity())));
}
for (AccessControlEntry ace : acl.getEntries()) {
AceInfo aceInfo = new AceInfo(ace);
- put.add(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.ACL_ACES_FAMILY), Bytes.toBytes(aceInfo.getSidInfo().getSid()), aceSerializer.serialize(aceInfo));
}
if (!put.isEmpty()) {
htable.put(put);
- htable.flushCommits();
logger.debug("ACL of " + acl.getObjectIdentity() + " updated successfully.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 85c9284..c0f8e6f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -28,9 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.WeakHashMap;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
@@ -406,33 +404,24 @@ public class CubeService extends BasicService {
if (htableInfoCache.containsKey(tableName)) {
return htableInfoCache.get(tableName);
}
-
- Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
- HTable table = null;
+ Connection conn = HBaseConnection.get(this.getConfig().getStorageUrl());
HBaseResponse hr = null;
long tableSize = 0;
int regionCount = 0;
- try {
- table = new HTable(hconf, tableName);
-
- HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(table);
- Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+ HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+ Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
- for (long s : sizeMap.values()) {
- tableSize += s;
- }
-
- regionCount = sizeMap.size();
-
- // Set response.
- hr = new HBaseResponse();
- hr.setTableSize(tableSize);
- hr.setRegionCount(regionCount);
- } finally {
- IOUtils.closeQuietly(table);
+ for (long s : sizeMap.values()) {
+ tableSize += s;
}
+ regionCount = sizeMap.size();
+
+ // Set response.
+ hr = new HBaseResponse();
+ hr.setTableSize(tableSize);
+ hr.setRegionCount(regionCount);
htableInfoCache.put(tableName, hr);
return hr;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 8810c85..2c031cf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -47,11 +47,11 @@ import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.Bytes;
@@ -161,14 +161,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -194,14 +193,13 @@ public class QueryService extends BasicService {
Query[] queryArray = new Query[queries.size()];
byte[] bytes = querySerializer.serialize(queries.toArray(queryArray));
- HTableInterface htable = null;
+ Table htable = null;
try {
- htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Put put = new Put(Bytes.toBytes(creator));
- put.add(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
+ put.addColumn(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN), bytes);
htable.put(put);
- htable.flushCommits();
} finally {
IOUtils.closeQuietly(htable);
}
@@ -213,12 +211,12 @@ public class QueryService extends BasicService {
}
List<Query> queries = new ArrayList<Query>();
- HTableInterface htable = null;
+ Table htable = null;
try {
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ org.apache.hadoop.hbase.client.Connection conn = HBaseConnection.get(hbaseUrl);
HBaseConnection.createHTableIfNeeded(conn, userTableName, USER_QUERY_FAMILY);
- htable = conn.getTable(userTableName);
+ htable = HBaseConnection.get(hbaseUrl).getTable(TableName.valueOf(userTableName));
Get get = new Get(Bytes.toBytes(creator));
get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY));
Result result = htable.get(get);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 07c7c6f..ab54882 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,11 +30,11 @@ import javax.annotation.PostConstruct;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.rest.security.AclHBaseStorage;
@@ -72,7 +72,7 @@ public class UserService implements UserDetailsManager {
@Override
public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -144,16 +144,16 @@ public class UserService implements UserDetailsManager {
@Override
public void updateUser(UserDetails user) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Pair<byte[], byte[]> pair = userToHBaseRow(user);
Put put = new Put(pair.getKey());
- put.add(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
+
+ put.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN), pair.getSecond());
htable.put(put);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -163,14 +163,13 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
Delete delete = new Delete(Bytes.toBytes(username));
htable.delete(delete);
- htable.flushCommits();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
} finally {
@@ -185,7 +184,7 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String username) {
- HTableInterface htable = null;
+ Table htable = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
@@ -216,7 +215,7 @@ public class UserService implements UserDetailsManager {
s.addColumn(Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclHBaseStorage.USER_AUTHORITY_COLUMN));
List<UserDetails> all = new ArrayList<UserDetails>();
- HTableInterface htable = null;
+ Table htable = null;
ResultScanner scanner = null;
try {
htable = aclHBaseStorage.getTable(userTableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..b769391 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.kylin.common.KylinConfig;
@@ -64,7 +64,7 @@ public class HBaseConnection {
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>();
- private static final Map<String, HConnection> connPool = new ConcurrentHashMap<String, HConnection>();
+ private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>();
private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>();
private static ExecutorService coprocessorPool = null;
@@ -75,7 +75,7 @@ public class HBaseConnection {
public void run() {
closeCoprocessorPool();
- for (HConnection conn : connPool.values()) {
+ for (Connection conn : connPool.values()) {
try {
conn.close();
} catch (IOException e) {
@@ -144,7 +144,7 @@ public class HBaseConnection {
// using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath
if (!(StringUtils.isEmpty(url) || "hbase".equals(url)))
throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties");
-
+
Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration());
addHBaseClusterNNHAConfiguration(conf);
@@ -213,9 +213,9 @@ public class HBaseConnection {
// ============================================================================
- // returned HConnection can be shared by multiple threads and does not require close()
+ // returned Connection can be shared by multiple threads and does not require close()
@SuppressWarnings("resource")
- public static HConnection get(String url) {
+ public static Connection get(String url) {
// find configuration
Configuration conf = configCache.get(url);
if (conf == null) {
@@ -223,13 +223,13 @@ public class HBaseConnection {
configCache.put(url, conf);
}
- HConnection connection = connPool.get(url);
+ Connection connection = connPool.get(url);
try {
while (true) {
// I don't use DCL since recreate a connection is not a big issue.
if (connection == null || connection.isClosed()) {
logger.info("connection is null or closed, creating a new one");
- connection = HConnectionManager.createConnection(conf);
+ connection = ConnectionFactory.createConnection(conf);
connPool.put(url, connection);
}
@@ -248,8 +248,8 @@ public class HBaseConnection {
return connection;
}
- public static boolean tableExists(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static boolean tableExists(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
return hbase.tableExists(TableName.valueOf(tableName));
} finally {
@@ -269,18 +269,18 @@ public class HBaseConnection {
deleteTable(HBaseConnection.get(hbaseUrl), tableName);
}
- public static void createHTableIfNeeded(HConnection conn, String table, String... families) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
-
+ public static void createHTableIfNeeded(Connection conn, String table, String... families) throws IOException {
+ Admin hbase = conn.getAdmin();
+ TableName tableName = TableName.valueOf(table);
try {
if (tableExists(conn, table)) {
logger.debug("HTable '" + table + "' already exists");
- Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(TableName.valueOf(table)));
+ Set<String> existingFamilies = getFamilyNames(hbase.getTableDescriptor(tableName));
boolean wait = false;
for (String family : families) {
if (existingFamilies.contains(family) == false) {
logger.debug("Adding family '" + family + "' to HTable '" + table + "'");
- hbase.addColumn(table, newFamilyDescriptor(family));
+ hbase.addColumn(tableName, newFamilyDescriptor(family));
// addColumn() is async, is there a way to wait it finish?
wait = true;
}
@@ -333,8 +333,8 @@ public class HBaseConnection {
return fd;
}
- public static void deleteTable(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ public static void deleteTable(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
if (!tableExists(conn, tableName)) {
@@ -344,10 +344,10 @@ public class HBaseConnection {
logger.debug("delete HTable '" + tableName + "'");
- if (hbase.isTableEnabled(tableName)) {
- hbase.disableTable(tableName);
+ if (hbase.isTableEnabled(TableName.valueOf(tableName))) {
+ hbase.disableTable(TableName.valueOf(tableName));
}
- hbase.deleteTable(tableName);
+ hbase.deleteTable(TableName.valueOf(tableName));
logger.debug("HTable '" + tableName + "' deleted");
} finally {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 1d19983..714a265 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -31,14 +31,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -69,7 +70,7 @@ public class HBaseResourceStore extends ResourceStore {
final String tableNameBase;
final String hbaseUrl;
- HConnection getConnection() throws IOException {
+ Connection getConnection() throws IOException {
return HBaseConnection.get(hbaseUrl);
}
@@ -120,7 +121,7 @@ public class HBaseResourceStore extends ResourceStore {
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
Scan scan = new Scan(startRow, endRow);
if ((filter != null && filter instanceof KeyOnlyFilter) == false) {
scan.addColumn(B_FAMILY, B_COLUMN_TS);
@@ -238,13 +239,12 @@ public class HBaseResourceStore extends ResourceStore {
IOUtils.copy(content, bout);
bout.close();
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
Put put = buildPut(resPath, ts, row, bout.toByteArray(), table);
table.put(put);
- table.flushCommits();
} finally {
IOUtils.closeQuietly(table);
}
@@ -252,7 +252,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS) throws IOException, IllegalStateException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
byte[] row = Bytes.toBytes(resPath);
byte[] bOldTS = oldTS == 0 ? null : Bytes.toBytes(oldTS);
@@ -265,8 +265,6 @@ public class HBaseResourceStore extends ResourceStore {
throw new IllegalStateException("Overwriting conflict " + resPath + ", expect old TS " + oldTS + ", but it is " + real);
}
- table.flushCommits();
-
return newTS;
} finally {
IOUtils.closeQuietly(table);
@@ -275,7 +273,7 @@ public class HBaseResourceStore extends ResourceStore {
@Override
protected void deleteResourceImpl(String resPath) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
boolean hdfsResourceExist = false;
Result result = internalGetFromHTable(table, resPath, true, false);
@@ -288,7 +286,6 @@ public class HBaseResourceStore extends ResourceStore {
Delete del = new Delete(Bytes.toBytes(resPath));
table.delete(del);
- table.flushCommits();
if (hdfsResourceExist) { // remove hdfs cell value
Path redirectPath = bigCellHDFSPath(resPath);
@@ -310,7 +307,7 @@ public class HBaseResourceStore extends ResourceStore {
}
private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
- HTableInterface table = getConnection().getTable(getAllInOneTableName());
+ Table table = getConnection().getTable(TableName.valueOf(getAllInOneTableName()));
try {
return internalGetFromHTable(table, path, fetchContent, fetchTimestamp);
} finally {
@@ -318,7 +315,7 @@ public class HBaseResourceStore extends ResourceStore {
}
}
- private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
+ private Result internalGetFromHTable(Table table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException {
byte[] rowkey = Bytes.toBytes(path);
Get get = new Get(rowkey);
@@ -337,7 +334,7 @@ public class HBaseResourceStore extends ResourceStore {
return exists ? result : null;
}
- private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, HTableInterface table) throws IOException {
+ private Path writeLargeCellToHdfs(String resPath, byte[] largeColumn, Table table) throws IOException {
Path redirectPath = bigCellHDFSPath(resPath);
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
@@ -363,7 +360,7 @@ public class HBaseResourceStore extends ResourceStore {
return redirectPath;
}
- private Put buildPut(String resPath, long ts, byte[] row, byte[] content, HTableInterface table) throws IOException {
+ private Put buildPut(String resPath, long ts, byte[] row, byte[] content, Table table) throws IOException {
int kvSizeLimit = Integer.parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
if (content.length > kvSizeLimit) {
writeLargeCellToHdfs(resPath, content, table);
@@ -371,8 +368,8 @@ public class HBaseResourceStore extends ResourceStore {
}
Put put = new Put(row);
- put.add(B_FAMILY, B_COLUMN, content);
- put.add(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
+ put.addColumn(B_FAMILY, B_COLUMN, content);
+ put.addColumn(B_FAMILY, B_COLUMN_TS, Bytes.toBytes(ts));
return put;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
index b141190..f63d9c2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/SimpleHBaseStore.java
@@ -26,12 +26,13 @@ import java.util.NoSuchElementException;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.kv.RowConstants;
@@ -86,14 +87,13 @@ public class SimpleHBaseStore implements IGTStore {
}
private class Writer implements IGTWriter {
- final HTableInterface table;
+ final BufferedMutator table;
final ByteBuffer rowkey = ByteBuffer.allocate(50);
final ByteBuffer value = ByteBuffer.allocate(50);
Writer() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
- table = conn.getTable(htableName);
- table.setAutoFlush(false, true);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ table = conn.getBufferedMutator(htableName);
}
@Override
@@ -113,24 +113,24 @@ public class SimpleHBaseStore implements IGTStore {
Put put = new Put(rowkey);
put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value);
- table.put(put);
+ table.mutate(put);
}
@Override
public void close() throws IOException {
- table.flushCommits();
+ table.flush();
table.close();
}
}
class Reader implements IGTScanner {
- final HTableInterface table;
+ final Table table;
final ResultScanner scanner;
int count = 0;
Reader() throws IOException {
- HConnection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
table = conn.getTable(htableName);
Scan scan = new Scan();
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index d99f80e..f879e2b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -26,8 +26,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.zip.DataFormatException;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
@@ -50,10 +51,10 @@ import org.apache.kylin.storage.gtrecord.StorageResponseGTScatter;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
-import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
+import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -116,7 +117,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Connection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList();
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
@@ -171,7 +172,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC {
final boolean[] abnormalFinish = new boolean[1];
try {
- HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool());
+ Table table = conn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()), HBaseConnection.getCoprocessorPool());
final CubeVisitRequest request = builder.build();
final byte[] startKey = epRange.getFirst();
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
index 3cefc5f..a52af90 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseScanRPC.java
@@ -24,11 +24,12 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.ShardingHash;
@@ -154,8 +155,8 @@ public class CubeHBaseScanRPC extends CubeHBaseRPC {
// primary key (also the 0th column block) is always selected
final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0);
// globally shared connection, does not require close
- HConnection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
- final HTableInterface hbaseTable = hbaseConn.getTable(cubeSeg.getStorageLocationIdentifier());
+ Connection hbaseConn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl());
+ final Table hbaseTable = hbaseConn.getTable(TableName.valueOf(cubeSeg.getStorageLocationIdentifier()));
List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks);
List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
index 38efecc..2a315f0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/coprocessor/endpoint/CubeVisitService.java
@@ -142,7 +142,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
if (shardLength == 0) {
return;
}
- byte[] regionStartKey = ArrayUtils.isEmpty(region.getStartKey()) ? new byte[shardLength] : region.getStartKey();
+ byte[] regionStartKey = ArrayUtils.isEmpty(region.getRegionInfo().getStartKey()) ? new byte[shardLength] : region.getRegionInfo().getStartKey();
Bytes.putBytes(rawScan.startKey, 0, regionStartKey, 0, shardLength);
Bytes.putBytes(rawScan.endKey, 0, regionStartKey, 0, shardLength);
}
@@ -179,7 +179,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) {
this.serviceStartTime = System.currentTimeMillis();
- region = env.getRegion();
+ region = (HRegion)env.getRegion();
region.startRegionOperation();
// if user change kylin.properties on kylin server, need to manually redeploy coprocessor jar to update KylinConfig of Env.
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 2814ad6..feb4842 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -26,7 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -79,7 +80,8 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin admin = conn.getAdmin();
try {
if (User.isHBaseSecurityEnabled(conf)) {
@@ -92,7 +94,7 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(tableName)) {
+ if (admin.tableExists(TableName.valueOf(tableName))) {
// admin.disableTable(tableName);
// admin.deleteTable(tableName);
throw new RuntimeException("HBase table " + tableName + " exists!");
@@ -101,7 +103,7 @@ public class CubeHTableUtil {
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
+ Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
IOUtils.closeQuietly(admin);
@@ -110,8 +112,7 @@ public class CubeHTableUtil {
}
public static void deleteHTable(TableName tableName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
@@ -126,8 +127,7 @@ public class CubeHTableUtil {
/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = new HBaseAdmin(conf);
+ Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
try {
if (admin.tableExists(tableName)) {
logger.info("disabling hbase table " + tableName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
index 46a828e..2d1c03a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/DeprecatedGCStep.java
@@ -29,9 +29,10 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.exception.ExecuteException;
@@ -100,19 +101,21 @@ public class DeprecatedGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf(table));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped HBase table " + table);
output.append("Dropped HBase table " + table + " \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
index d5b36df..6587d4e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseCuboidWriter.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.cuboid.Cuboid;
@@ -49,7 +49,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private final List<KeyValueCreator> keyValueCreators;
private final int nColumns;
- private final HTableInterface hTable;
+ private final Table hTable;
private final CubeDesc cubeDesc;
private final CubeSegment cubeSegment;
private final Object[] measureValues;
@@ -58,7 +58,7 @@ public class HBaseCuboidWriter implements ICuboidWriter {
private AbstractRowKeyEncoder rowKeyEncoder;
private byte[] keybuf;
- public HBaseCuboidWriter(CubeSegment segment, HTableInterface hTable) {
+ public HBaseCuboidWriter(CubeSegment segment, Table hTable) {
this.keyValueCreators = Lists.newArrayList();
this.cubeSegment = segment;
this.cubeDesc = cubeSegment.getCubeDesc();
@@ -117,7 +117,6 @@ public class HBaseCuboidWriter implements ICuboidWriter {
long t = System.currentTimeMillis();
if (hTable != null) {
hTable.put(puts);
- hTable.flushCommits();
}
logger.info("commit total " + puts.size() + " puts, totally cost:" + (System.currentTimeMillis() - t) + "ms");
puts.clear();
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
index 5b2441c..2f7e164 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/MergeGCStep.java
@@ -24,11 +24,11 @@ import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
@@ -69,19 +69,20 @@ public class MergeGCStep extends AbstractExecutable {
List<String> oldTables = getOldHTables();
if (oldTables != null && oldTables.size() > 0) {
String metadataUrlPrefix = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin admin = null;
+ Admin admin = null;
try {
- admin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ admin = conn.getAdmin();
+
for (String table : oldTables) {
- if (admin.tableExists(table)) {
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(Bytes.toBytes(table));
+ if (admin.tableExists(TableName.valueOf(table))) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(TableName.valueOf((table)));
String host = tableDescriptor.getValue(IRealizationConstants.HTableTag);
if (metadataUrlPrefix.equalsIgnoreCase(host)) {
- if (admin.isTableEnabled(table)) {
- admin.disableTable(table);
+ if (admin.isTableEnabled(TableName.valueOf(table))) {
+ admin.disableTable(TableName.valueOf(table));
}
- admin.deleteTable(table);
+ admin.deleteTable(TableName.valueOf(table));
logger.debug("Dropped htable: " + table);
output.append("HBase table " + table + " is dropped. \n");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
index a150607..56f867a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CleanHtableCLI.java
@@ -21,9 +21,11 @@ package org.apache.kylin.storage.hbase.util;
import java.io.IOException;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -38,8 +40,8 @@ public class CleanHtableCLI extends AbstractApplication {
protected static final Logger logger = LoggerFactory.getLogger(CleanHtableCLI.class);
private void clean() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
String name = descriptor.getNameAsString().toLowerCase();
@@ -50,7 +52,7 @@ public class CleanHtableCLI extends AbstractApplication {
System.out.println();
descriptor.setValue(IRealizationConstants.HTableOwner, "DL-eBay-Kylin@ebay.com");
- hbaseAdmin.modifyTable(descriptor.getNameAsString(), descriptor);
+ hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
}
}
hbaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 2e682b1..f47bf31 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -33,12 +33,13 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -88,7 +89,7 @@ public class CubeMigrationCLI {
private static ResourceStore srcStore;
private static ResourceStore dstStore;
private static FileSystem hdfsFS;
- private static HBaseAdmin hbaseAdmin;
+ private static Admin hbaseAdmin;
public static final String ACL_INFO_FAMILY = "i";
private static final String ACL_TABLE_NAME = "_acl";
@@ -133,8 +134,8 @@ public class CubeMigrationCLI {
checkAndGetHbaseUrl();
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
hdfsFS = FileSystem.get(new Configuration());
@@ -232,6 +233,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -325,8 +327,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, dstConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -448,11 +450,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -472,7 +474,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -503,8 +504,8 @@ public class CubeMigrationCLI {
switch (opt.type) {
case CHANGE_HTABLE_HOST: {
- String tableName = (String) opt.params[0];
- HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
+ TableName tableName = TableName.valueOf((String) opt.params[0]);
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName);
hbaseAdmin.disableTable(tableName);
desc.setValue(IRealizationConstants.HTableTag, srcConfig.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(tableName, desc);
@@ -538,13 +539,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -561,7 +561,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
index 8bd4abf..20d0f7d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -26,10 +26,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
@@ -61,7 +61,7 @@ public class CubeMigrationCheckCLI {
private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
private KylinConfig dstCfg;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private List<String> issueExistHTables;
private List<String> inconsistentHTables;
@@ -130,9 +130,8 @@ public class CubeMigrationCheckCLI {
this.dstCfg = kylinConfig;
this.ifFix = isFix;
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- hbaseAdmin = new HBaseAdmin(conf);
-
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ hbaseAdmin = conn.getAdmin();
issueExistHTables = Lists.newArrayList();
inconsistentHTables = Lists.newArrayList();
}
@@ -189,10 +188,10 @@ public class CubeMigrationCheckCLI {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info("Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(sepNameList[0]);
+ hbaseAdmin.disableTable(TableName.valueOf(sepNameList[0]));
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
- hbaseAdmin.modifyTable(sepNameList[0], desc);
- hbaseAdmin.enableTable(sepNameList[0]);
+ hbaseAdmin.modifyTable(TableName.valueOf(sepNameList[0]), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(sepNameList[0]));
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
[18/50] [abbrv] kylin git commit: KYLIN 1875 minor,
data model designer
Posted by li...@apache.org.
KYLIN 1875 minor,data model designer
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c2c43cb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c2c43cb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c2c43cb
Branch: refs/heads/master-cdh5.7
Commit: 1c2c43cb88d2abc5a1ea1cbf18cbede02fe8c4f5
Parents: f55cc5c
Author: chenzhx <34...@qq.com>
Authored: Thu Dec 15 17:45:13 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 15 18:57:36 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 2 -
webapp/app/js/controllers/cubeEdit.js | 12 +++++
webapp/app/js/controllers/modelDataModel.js | 47 ++++++--------------
webapp/app/js/controllers/modelEdit.js | 22 ++++++++-
.../modelDesigner/conditions_settings.html | 4 +-
.../app/partials/modelDesigner/data_model.html | 24 +++++-----
6 files changed, 63 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c2c43cb/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index e557afa..20af908 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -160,7 +160,6 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
if (aggregation_group) {
list[index] = aggregation_group;
}
- console.log($scope.cubeMetaFrame.aggregation_groups);
};
$scope.refreshAggregationJoint = function (list, index, aggregation_group,joinIndex,jointDim){
@@ -170,7 +169,6 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
if (aggregation_group) {
list[index] = aggregation_group;
}
- console.log($scope.cubeMetaFrame.aggregation_groups);
};
$scope.refreshIncludes = function (list, index, aggregation_groups) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c2c43cb/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index d58f08f..a2b38f9 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -328,6 +328,18 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
if (!modelsManager.getModels().length) {
ModelDescService.query({model_name: $scope.cubeMetaFrame.model_name}, function (_model) {
$scope.metaModel.model = _model;
+ var rootFactTable = VdmUtil.removeNameSpace($scope.metaModel.model.fact_table);
+ $scope.aliasName.push(rootFactTable);
+ $scope.aliasTableMap[rootFactTable]=$scope.metaModel.model.fact_table;
+ $scope.tableAliasMap[$scope.metaModel.model.fact_table]=rootFactTable;
+ angular.forEach($scope.metaModel.model.lookups,function(joinTable){
+ if(!joinTable.alias){
+ joinTable.alias=VdmUtil.removeNameSpace(joinTable.table);
+ }
+ $scope.aliasTableMap[joinTable.alias]=joinTable.table;
+ $scope.tableAliasMap[joinTable.table]=joinTable.alias;
+ $scope.aliasName.push(joinTable.alias);
+ });
});
}
$scope.state.cubeSchema = angular.toJson($scope.cubeMetaFrame, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c2c43cb/webapp/app/js/controllers/modelDataModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelDataModel.js b/webapp/app/js/controllers/modelDataModel.js
index b6edd43..3834e06 100644
--- a/webapp/app/js/controllers/modelDataModel.js
+++ b/webapp/app/js/controllers/modelDataModel.js
@@ -20,26 +20,13 @@
KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cubeConfig,MetaModel,SweetAlert,ModelGraphService,$log,TableModel,ModelService,loadingRequest,modelsManager,VdmUtil) {
$scope.modelsManager = modelsManager;
+ $scope.VdmUtil = VdmUtil;
angular.forEach($scope.modelsManager.selectedModel.lookups,function(joinTable){
if(!joinTable.alias){
joinTable.alias=VdmUtil.removeNameSpace(joinTable.table);
}
});
- $scope.init = function (){
- $scope.FactTable={root:$scope.modelsManager.selectedModel.fact_table};
- $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)]=$scope.modelsManager.selectedModel.fact_table;
- $scope.tableAliasMap[$scope.modelsManager.selectedModel.fact_table]=VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table);
- $scope.aliasName.push(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table));
- angular.forEach($scope.modelsManager.selectedModel.lookups,function(joinTable){
- $scope.aliasTableMap[joinTable.alias]=joinTable.table;
- $scope.tableAliasMap[joinTable.table]=joinTable.alias;
- $scope.aliasName.push(joinTable.alias);
- });
- }
- if($scope.state.mode=='edit'){
- $scope.init();
- }
-
+ $scope.FactTable={root:$scope.modelsManager.selectedModel.fact_table};
$scope.cubeConfig = cubeConfig;
var DataModel = function () {
return {
@@ -134,13 +121,11 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
$scope.tableAliasMap[$scope.newLookup.table]=$scope.newLookup.alias;
$scope.aliasName.push($scope.newLookup.alias);
lookupList.push(angular.copy($scope.newLookup));
-
$scope.resetParams();
};
$scope.doneEditLookup = function () {
// Copy edited model to destination model.
- angular.copy($scope.newLookup, lookupList[$scope.lookupState.editingIndex]);
var oldAlias=$scope.aliasName[$scope.lookupState.editingIndex+1];
var newAlias=$scope.newLookup.alias;
if(oldAlias!=newAlias){
@@ -168,7 +153,9 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
for(var i=0;i< modelsManager.selectedModel.metrics.length;i++){
modelsManager.selectedModel.metrics[i]= modelsManager.selectedModel.metrics[i].replace(oldAlias+'.',newAlias+'.');
}
- modelsManager.selectedModel.partition_desc.partition_date_column = modelsManager.selectedModel.partition_desc.partition_date_column.replace(oldAlias+'.',newAlias+'.');
+ if(modelsManager.selectedModel.partition_desc.partition_date_column){
+ modelsManager.selectedModel.partition_desc.partition_date_column = modelsManager.selectedModel.partition_desc.partition_date_column.replace(oldAlias+'.',newAlias+'.');
+ }
}
}
angular.copy($scope.newLookup,lookupList[$scope.lookupState.editingIndex]);
@@ -176,19 +163,21 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
$scope.resetParams();
};
$scope.changeFactTable = function () {
- $scope.aliasTableMap={};
+ if(!$scope.FactTable){
+ return;
+ }
$scope.aliasTableMap[VdmUtil.removeNameSpace($scope.FactTable.root)]=$scope.FactTable.root;
- $scope.tableAliasMap={};
$scope.tableAliasMap[$scope.FactTable.root]=VdmUtil.removeNameSpace($scope.FactTable.root);
- $scope.aliasName=[VdmUtil.removeNameSpace($scope.FactTable.root)];
- modelsManager.selectedModel.lookups = [];
- modelsManager.selectedModel.dimensions = [];
- modelsManager.selectedModel.metrics= [];
+ $scope.aliasName.splice(0,$scope.aliasName.length);
+ $scope.aliasName.push(VdmUtil.removeNameSpace($scope.FactTable.root));
+ modelsManager.selectedModel.lookups.splice(0,modelsManager.selectedModel.lookups.length);
+ modelsManager.selectedModel.dimensions.splice(0,modelsManager.selectedModel.dimensions.length);
+ modelsManager.selectedModel.metrics.splice(0,modelsManager.selectedModel.metrics.length);
modelsManager.selectedModel.partition_desc.partition_date_column = null;
$scope.modelsManager.selectedModel.fact_table=$scope.FactTable.root;
}
$scope.changeJoinTable = function () {
- $scope.newLookup.alias=$scope.newLookup.table;
+ $scope.newLookup.alias=VdmUtil.removeNameSpace($scope.newLookup.table);
}
$scope.cancelLookup = function () {
$scope.resetParams();
@@ -221,8 +210,6 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
lookupList.splice(lookupList.indexOf(lookup), 1);
}
};
- $scope.changeAlias = function (){
- }
$scope.changeKey = function(index){
var join_table = $scope.newLookup.joinTable;
@@ -297,10 +284,4 @@ KylinApp.controller('ModelDataModelCtrl', function ($location,$scope, $modal,cub
$scope.filterNotRoot = function (item) {
return item.name!==modelsManager.selectedModel.fact_table;
};
-/* $scope.$watch('$scope.newLookup.alias', function (newValue, oldValue) {
- if (newValue&&$scope.lookupState.editing ) {
- console.log(newValue);
- console.log(oldValue);
- }
- });*/
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c2c43cb/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index fd78d03..c504b52 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -25,6 +25,7 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
$scope.tableAliasMap={};
$scope.aliasTableMap={};
$scope.aliasName=[];
+ $scope.selectedAliasCubeMap={};
$scope.route={params:$routeParams.modelName};
$scope.modelMode = absUrl.indexOf("/models/add")!=-1?'addNewModel':absUrl.indexOf("/models/edit")!=-1?'editExistModel':'default';
@@ -93,8 +94,27 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
ModelDescService.query({model_name: modelName}, function (model) {
if (model) {
modelsManager.selectedModel = model;
+ $scope.FactTable={root:$scope.modelsManager.selectedModel.fact_table};
+ $scope.aliasTableMap[VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table)]=$scope.modelsManager.selectedModel.fact_table;
+ $scope.tableAliasMap[$scope.modelsManager.selectedModel.fact_table]=VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table);
+ $scope.aliasName.push(VdmUtil.removeNameSpace($scope.modelsManager.selectedModel.fact_table));
+ angular.forEach($scope.modelsManager.selectedModel.lookups,function(joinTable){
+ $scope.aliasTableMap[joinTable.alias]=joinTable.table;
+ $scope.tableAliasMap[joinTable.table]=joinTable.alias;
+ $scope.aliasName.push(joinTable.alias);
+ });
CubeService.list({modelName:model.name}, function (_cubes) {
- $scope.cubesLength = _cubes.length;
+ $scope.cubesLength = _cubes.length;
+ angular.forEach(_cubes,function(cube){
+ CubeDescService.query({cube_name:cube.name},{},function(each){
+ angular.forEach(each[0].dimensions,function(dimension){
+ $scope.selectedAliasCubeMap[dimension.table]=true;
+ });
+ angular.forEach(each[0].measures,function(measure){
+ $scope.selectedAliasCubeMap[VdmUtil.getNameSpaceAliasName(measure.function.parameter.value)]=true;
+ });
+ })
+ });
});
modelsManager.selectedModel.project = ProjectModel.getProjectByCubeModel(modelName);
if(!ProjectModel.getSelectedProject()){
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c2c43cb/webapp/app/partials/modelDesigner/conditions_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index 6820b60..30fcb67 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -27,14 +27,14 @@
<div class="row middle-popover">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Date Column</b> <i kylinpopover placement="right" title="Partition Date Column" template="partitionTip.html" class="fa fa-info-circle"></i></label>
<div class="col-xs-12 col-sm-6" ng-if="state.mode=='edit'">
- <select style="width: 45%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
+ <select style="width: 49%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
ng-model="selectedTables.fact"
data-placement=""
ng-options="alias as alias for alias in availableFactTables" >
<option value="">--Select Partition Table--</option>
</select>
- <select style="width: 45%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
+ <select style="width: 49%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
ng-model="modelsManager.selectedModel.partition_desc.partition_date_column"
ng-change="partitionChange(modelsManager.selectedModel.partition_desc.partition_date_column)"
data-placement=""
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c2c43cb/webapp/app/partials/modelDesigner/data_model.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/modelDesigner/data_model.html b/webapp/app/partials/modelDesigner/data_model.html
index e032cf9..97d3c51 100644
--- a/webapp/app/partials/modelDesigner/data_model.html
+++ b/webapp/app/partials/modelDesigner/data_model.html
@@ -27,7 +27,7 @@
</label>
<div class="col-xs-12 col-sm-6" ng-class="{'has-error':forms.data_model_form.table_name.$invalid && (forms.data_model_form.table_name.$dirty||forms.data_model_form.$submitted)}">
<select chosen ng-model="FactTable.root" ng-if="state.mode=='edit'"
- ng-options="table.name as table.name for table in tableModel.selectProjectTables"
+ ng-options="table.name as VdmUtil.removeNameSpace(table.name) for table in tableModel.selectProjectTables"
style="width:100%;" ng-change="changeFactTable()"
name="table_name" ng-disabled="cubesLength>0"
ng-required="true"
@@ -35,7 +35,7 @@
class="chosen-select">
<option value=""> -- Select Fact Table -- </option>
</select>
- <span ng-if="state.mode=='view'">{{modelsManager.selectedModel.fact_table}}</span>
+ <span ng-if="state.mode=='view'">{{VdmUtil.removeNameSpace(modelsManager.selectedModel.fact_table)}}</span>
</div>
</div>
</div>
@@ -67,7 +67,7 @@
<th>Table Kind</th>
<th>Join Type</th>
<th>Join Condition</th>
- <th ng-if="state.mode=='edit'">Actions</th>
+ <th ng-if="state.mode=='edit'" class="col-xs-1">Actions</th>
</tr>
</thead>
<tbody>
@@ -98,13 +98,14 @@
</td>
<td ng-if="state.mode=='edit'">
<!-- edit button -->
- <button class="btn btn-xs btn-info" ng-disabled="lookupState.editing||cubesLength>0"
+ <button class="btn btn-xs btn-info" ng-disabled="lookupState.editing||selectedAliasCubeMap[lookup.alias]"
ng-click="editLookup(lookup)" ><i class="fa fa-pencil"></i>
</button>
<!-- remove button -->
- <button class="btn btn-xs btn-danger" ng-disabled="lookupState.editing||cubesLength>0"
+ <button class="btn btn-xs btn-danger" ng-disabled="lookupState.editing||selectedAliasCubeMap[lookup.alias]"
ng-click="removeLookup(lookup)" ><i class="fa fa-trash-o"></i>
</button>
+ <i class="fa fa-info-circle" ng-if="selectedAliasCubeMap[lookup.alias]" kylinpopover placement="left" title="Tip" template="tableInUse.html"></i>
</td>
</tr>
</tbody>
@@ -115,7 +116,7 @@
<!-- Add Join Table Form -->
<script type="text/ng-template" id="dataModelLookupTable.html">
<div class="modal-header">
- <h4 class="box-title lighter">{{lookupState.editing ? 'Edit' : 'Add'}} Lookup</h4>
+ <h4 class="box-title lighter">{{lookupState.editing ? 'Edit' : 'Add'}} Join Table</h4>
</div>
<div class="modal-body">
<div class="row">
@@ -136,7 +137,7 @@
<small class="help-block" ng-show="lookup_form.table_name.$invalid && (lookup_form.table_name.$dirty||forms.model_info_form.$submitted)">Table name required</small>
<b> </b>
<select chosen ng-model="newLookup.table" style="width: 45%"
- ng-options="table.name as table.name for table in tableModel.selectProjectTables|filter:filterNotRoot"
+ ng-options="table.name as VdmUtil.removeNameSpace(table.name) for table in tableModel.selectProjectTables|filter:filterNotRoot"
name="table_name" ng-disabled="lookupState.editing"
ng-required="true" ng-change="changeJoinTable()"
data-placeholder="Join Table Name"
@@ -155,7 +156,7 @@
<label class="col-sm-3 control-label font-color-default"><b>Alias</b></label>
<div class="col-sm-6">
<input type="text" class="form-control " name="joinTable_alias" placeholder="Input Table Alias" ng-required="true"
- ng-model="newLookup.alias" ng-change="changeAlias()" ng-pattern="/^\w+$/">
+ ng-model="newLookup.alias" ng-pattern="/^\w+$/">
<small class="help-block red" ng-show="!lookup_form.joinTable_alias.$error.required&&lookup_form.joinTable_alias.$invalid && (lookup_form.joinTable_alias.$dirty||lookup_form.$submitted)"><i class="fa fa-exclamation-triangle"></i> Alias is invalid.</small>
</div>
</div>
@@ -186,13 +187,13 @@
<div class="col-xs-9">
<div ng-repeat="joinIndex in [] | range: newLookup.join.primary_key.length">
<div>
- <select style="width: 45%" chosen data-placeholder="Fact Table Column"
+ <select style="width: 45%" chosen data-placeholder="JoinTable Column"
ng-model="newLookup.join.foreign_key[$index]" ng-change="changeKey($index)"
ng-options="newLookup.joinTable+'.'+columns.name as columns.name for columns in getColumnsByAlias(newLookup.joinTable)" >
<option value=""></option>
</select>
<b>=</b>
- <select style="width: 45%" chosen data-placeholder="Lookup Table Column"
+ <select style="width: 45%" chosen data-placeholder=" JoinTable Column"
ng-model="newLookup.join.primary_key[$index]" ng-change="changeKey($index)"
ng-options="newLookup.alias+'.'+columns.name as columns.name for columns in getColumnsByTable(newLookup.table)" >
<option value=""></option>
@@ -249,3 +250,6 @@
</div>
</script>
</div>
+<script type="text/ng-template" id="tableInUse.html">
+ <p>This table is in use in Cube.</p>
+</script>
[06/50] [abbrv] kylin git commit: KYLIN-2243 use at least 1Mb for
TopNCounterSerializer.maxLength()
Posted by li...@apache.org.
KYLIN-2243 use at least 1Mb for TopNCounterSerializer.maxLength()
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a4ddbbdd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a4ddbbdd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a4ddbbdd
Branch: refs/heads/master-cdh5.7
Commit: a4ddbbdde876d0690dd65ea8549e8d40cfb5fdd1
Parents: 8ffb0e7
Author: shaofengshi <sh...@apache.org>
Authored: Mon Dec 12 14:34:19 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 12 14:34:19 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/measure/topn/TopNCounterSerializer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a4ddbbdd/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java
index 071e2a2..cef9177 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNCounterSerializer.java
@@ -54,7 +54,7 @@ public class TopNCounterSerializer extends DataTypeSerializer<TopNCounter<ByteAr
@Override
public int maxLength() {
- return precision * TopNCounter.EXTRA_SPACE_RATE * (4 + 8);
+ return Math.max(precision * TopNCounter.EXTRA_SPACE_RATE * (4 + 8), 1024 * 1024); // use at least 1M
}
@Override
[05/50] [abbrv] kylin git commit: KYLIN-2244
"kylin.job.cuboid.size.memhungry.ratio" shouldn't be applied on measures like
TopN
Posted by li...@apache.org.
KYLIN-2244 "kylin.job.cuboid.size.memhungry.ratio" shouldn't be applied on measures like TopN
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8ffb0e71
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8ffb0e71
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8ffb0e71
Branch: refs/heads/master-cdh5.7
Commit: 8ffb0e7103d63d2c0f5d093f3afde1a0490eb8a0
Parents: 4408579
Author: shaofengshi <sh...@apache.org>
Authored: Mon Dec 12 14:19:55 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 12 14:19:55 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 5 +++
.../kylin/engine/mr/common/CubeStatsReader.java | 36 ++++++++------------
2 files changed, 20 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ffb0e71/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2b35c70..610c2af 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -261,10 +261,15 @@ abstract public class KylinConfigBase implements Serializable {
return Double.parseDouble(getOptional("kylin.cube.size-estimate-ratio", "0.25"));
}
+ @Deprecated
public double getJobCuboidSizeMemHungryRatio() {
return Double.parseDouble(getOptional("kylin.cube.size-estimate-memhungry-ratio", "0.05"));
}
+ public double getJobCuboidSizeCountDistinctRatio() {
+ return Double.parseDouble(getOptional("kylin.cube.size-estimate-countdistinct-ratio", "0.05"));
+ }
+
public String getCubeAlgorithm() {
return getOptional("kylin.cube.algorithm", "auto");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8ffb0e71/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 1cf5da6..21af1e6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -55,6 +55,7 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter;
import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -196,41 +197,34 @@ public class CubeStatsReader {
*/
private static double estimateCuboidStorageSize(CubeSegment cubeSegment, long cuboidId, long rowCount, long baseCuboidId, List<Integer> rowKeyColumnLength) {
- int bytesLength = cubeSegment.getRowKeyPreambleSize();
+ int rowkeyLength = cubeSegment.getRowKeyPreambleSize();
KylinConfig kylinConf = cubeSegment.getConfig();
long mask = Long.highestOneBit(baseCuboidId);
long parentCuboidIdActualLength = Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId);
for (int i = 0; i < parentCuboidIdActualLength; i++) {
if ((mask & cuboidId) > 0) {
- bytesLength += rowKeyColumnLength.get(i); //colIO.getColumnLength(columnList.get(i));
+ rowkeyLength += rowKeyColumnLength.get(i); //colIO.getColumnLength(columnList.get(i));
}
mask = mask >> 1;
}
// add the measure length
- int space = 0;
- boolean isMemoryHungry = false;
+ int normalSpace = rowkeyLength;
+ int countDistinctSpace = 0;
for (MeasureDesc measureDesc : cubeSegment.getCubeDesc().getMeasures()) {
- if (measureDesc.getFunction().getMeasureType().isMemoryHungry()) {
- isMemoryHungry = true;
- }
DataType returnType = measureDesc.getFunction().getReturnDataType();
- space += returnType.getStorageBytesEstimate();
- }
- bytesLength += space;
-
- double ret = 1.0 * bytesLength * rowCount / (1024L * 1024L);
- if (isMemoryHungry) {
- double cuboidSizeMemHungryRatio = kylinConf.getJobCuboidSizeMemHungryRatio();
- logger.info("Cube is memory hungry, storage size estimation multiply " + cuboidSizeMemHungryRatio);
- ret *= cuboidSizeMemHungryRatio;
- } else {
- double cuboidSizeRatio = kylinConf.getJobCuboidSizeRatio();
- logger.info("Cube is not memory hungry, storage size estimation multiply " + cuboidSizeRatio);
- ret *= cuboidSizeRatio;
+ if (measureDesc.getFunction().getExpression().equals(FunctionDesc.FUNC_COUNT_DISTINCT)) {
+ countDistinctSpace += returnType.getStorageBytesEstimate();
+ } else {
+ normalSpace += returnType.getStorageBytesEstimate();
+ }
}
- logger.info("Cuboid " + cuboidId + " has " + rowCount + " rows, each row size is " + bytesLength + " bytes." + " Total size is " + ret + "M.");
+
+ double cuboidSizeRatio = kylinConf.getJobCuboidSizeRatio();
+ double cuboidSizeMemHungryRatio = kylinConf.getJobCuboidSizeCountDistinctRatio();
+ double ret = (1.0 * normalSpace * rowCount * cuboidSizeRatio + 1.0 * countDistinctSpace * rowCount * cuboidSizeMemHungryRatio) / (1024L * 1024L);
+ logger.info("Cuboid " + cuboidId + " has " + rowCount + " rows, each row size is " + (normalSpace + countDistinctSpace) + " bytes." + " Total size is " + ret + "M.");
return ret;
}
[48/50] [abbrv] kylin git commit: KYLIN-1528 Create a branch for v1.5
with HBase 1.x API
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c8410f9..e72859d 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
@@ -81,7 +82,8 @@ public class DeployCoprocessorCLI {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
- HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
+ Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String localCoprocessorJar;
if ("default".equals(args[0])) {
@@ -165,10 +167,10 @@ public class DeployCoprocessorCLI {
public static void deployCoprocessor(HTableDescriptor tableDesc) {
try {
initHTableCoprocessor(tableDesc);
- logger.info("hbase table " + tableDesc.getName() + " deployed with coprocessor.");
+ logger.info("hbase table " + tableDesc.getTableName() + " deployed with coprocessor.");
} catch (Exception ex) {
- logger.error("Error deploying coprocessor on " + tableDesc.getName(), ex);
+ logger.error("Error deploying coprocessor on " + tableDesc.getTableName(), ex);
logger.error("Will try creating the table without coprocessor.");
}
}
@@ -189,7 +191,7 @@ public class DeployCoprocessorCLI {
desc.addCoprocessor(CubeEndpointClass, hdfsCoprocessorJar, 1001, null);
}
- public static boolean resetCoprocessor(String tableName, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
+ public static boolean resetCoprocessor(String tableName, Admin hbaseAdmin, Path hdfsCoprocessorJar) throws IOException {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
@@ -204,7 +206,7 @@ public class DeployCoprocessorCLI {
logger.info("reset coprocessor on " + tableName);
logger.info("Disable " + tableName);
- hbaseAdmin.disableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
while (desc.hasCoprocessor(CubeObserverClassOld2)) {
desc.removeCoprocessor(CubeObserverClassOld2);
@@ -230,16 +232,15 @@ public class DeployCoprocessorCLI {
desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
}
- hbaseAdmin.modifyTable(tableName, desc);
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
logger.info("Enable " + tableName);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
return true;
}
-
- private static List<String> resetCoprocessorOnHTables(final HBaseAdmin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static List<String> resetCoprocessorOnHTables(final Admin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processedTables = Collections.synchronizedList(new ArrayList<String>());
ExecutorService coprocessorPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
@@ -260,12 +261,12 @@ public class DeployCoprocessorCLI {
private static class ResetCoprocessorWorker implements Runnable {
private final CountDownLatch countDownLatch;
- private final HBaseAdmin hbaseAdmin;
+ private final Admin hbaseAdmin;
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List<String> processedTables;
- public ResetCoprocessorWorker(CountDownLatch countDownLatch, HBaseAdmin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
+ public ResetCoprocessorWorker(CountDownLatch countDownLatch, Admin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
this.countDownLatch = countDownLatch;
this.hbaseAdmin = hbaseAdmin;
this.hdfsCoprocessorJar = hdfsCoprocessorJar;
@@ -386,7 +387,7 @@ public class DeployCoprocessorCLI {
return coprocessorDir;
}
- private static Set<String> getCoprocessorJarPaths(HBaseAdmin hbaseAdmin, List<String> tableNames) throws IOException {
+ private static Set<String> getCoprocessorJarPaths(Admin hbaseAdmin, List<String> tableNames) throws IOException {
HashSet<String> result = new HashSet<String>();
for (String tableName : tableNames) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index 61c73d5..1cdb2f8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -25,10 +25,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -235,9 +236,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -257,7 +258,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
index 86ba22f..dd5f8fa 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
@@ -75,7 +75,7 @@ public class GridTableHBaseBenchmark {
System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio);
String hbaseUrl = "hbase"; // use hbase-site.xml on classpath
- HConnection conn = HBaseConnection.get(hbaseUrl);
+ Connection conn = HBaseConnection.get(hbaseUrl);
createHTableIfNeeded(conn, TEST_TABLE);
prepareData(conn);
@@ -91,10 +91,10 @@ public class GridTableHBaseBenchmark {
}
- private static void testColumnScan(HConnection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
+ private static void testColumnScan(Connection conn, List<Pair<Integer, Integer>> colScans) throws IOException {
Stats stats = new Stats("COLUMN_SCAN");
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -122,20 +122,20 @@ public class GridTableHBaseBenchmark {
}
}
- private static void testRowScanNoIndexFullScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexFullScan(Connection conn, boolean[] hits) throws IOException {
fullScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_FULL"));
}
- private static void testRowScanNoIndexSkipScan(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanNoIndexSkipScan(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_NO_IDX_SKIP"));
}
- private static void testRowScanWithIndex(HConnection conn, boolean[] hits) throws IOException {
+ private static void testRowScanWithIndex(Connection conn, boolean[] hits) throws IOException {
jumpScan(conn, hits, new Stats("ROW_SCAN_IDX"));
}
- private static void fullScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -156,11 +156,11 @@ public class GridTableHBaseBenchmark {
}
}
- private static void jumpScan(HConnection conn, boolean[] hits, Stats stats) throws IOException {
+ private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience
- HTableInterface table = conn.getTable(TEST_TABLE);
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
stats.markStart();
@@ -204,8 +204,8 @@ public class GridTableHBaseBenchmark {
}
}
- private static void prepareData(HConnection conn) throws IOException {
- HTableInterface table = conn.getTable(TEST_TABLE);
+ private static void prepareData(Connection conn) throws IOException {
+ Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
try {
// check how many rows existing
@@ -258,8 +258,8 @@ public class GridTableHBaseBenchmark {
return bytes;
}
- private static void createHTableIfNeeded(HConnection conn, String tableName) throws IOException {
- HBaseAdmin hbase = new HBaseAdmin(conn);
+ private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
+ Admin hbase = conn.getAdmin();
try {
boolean tableExist = false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
index 6749d6c..940d64a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -24,9 +24,11 @@ import java.util.List;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
@@ -55,8 +57,8 @@ public class HBaseClean extends AbstractApplication {
private void cleanUp() {
try {
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = Lists.newArrayList();
@@ -71,12 +73,12 @@ public class HBaseClean extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
index 937b65f..1daca0a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseRegionSizeCalculator.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -31,12 +32,15 @@ import java.util.TreeSet;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerLoad;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
@@ -58,30 +62,31 @@ public class HBaseRegionSizeCalculator {
/**
* Computes size of each region for table and given column families.
* */
- public HBaseRegionSizeCalculator(HTable table) throws IOException {
- this(table, new HBaseAdmin(table.getConfiguration()));
- }
-
- /** Constructor for unit testing */
- HBaseRegionSizeCalculator(HTable table, HBaseAdmin hBaseAdmin) throws IOException {
+ public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {
+ Table table = null;
+ Admin admin = null;
try {
+ table = hbaseConnection.getTable(TableName.valueOf(tableName));
+ admin = hbaseConnection.getAdmin();
+
if (!enabled(table.getConfiguration())) {
logger.info("Region size calculation disabled.");
return;
}
- logger.info("Calculating region sizes for table \"" + new String(table.getTableName()) + "\".");
+ logger.info("Calculating region sizes for table \"" + table.getName() + "\".");
// Get regions for table.
- Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet();
+ RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
+ List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- for (HRegionInfo regionInfo : tableRegionInfos) {
- tableRegions.add(regionInfo.getRegionName());
+ for (HRegionLocation hRegionLocation : regionLocationList) {
+ tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
}
- ClusterStatus clusterStatus = hBaseAdmin.getClusterStatus();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
@@ -105,7 +110,7 @@ public class HBaseRegionSizeCalculator {
}
}
} finally {
- IOUtils.closeQuietly(hBaseAdmin);
+ IOUtils.closeQuietly(admin);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
index 266f7e7..a2f60d4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -23,9 +23,10 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -42,8 +43,8 @@ public class HBaseUsage {
Map<String, List<String>> envs = Maps.newHashMap();
// get all kylin hbase tables
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
for (HTableDescriptor desc : tableDescriptors) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
index e26c8e8..da13fa4 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HbaseStreamingInput.java
@@ -32,15 +32,15 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -58,11 +58,11 @@ public class HbaseStreamingInput {
private static final byte[] QN = "C".getBytes();
public static void createTable(String tableName) throws IOException {
- HConnection conn = getConnection();
- HBaseAdmin hadmin = new HBaseAdmin(conn);
+ Connection conn = getConnection();
+ Admin hadmin = conn.getAdmin();
try {
- boolean tableExist = hadmin.tableExists(tableName);
+ boolean tableExist = hadmin.tableExists(TableName.valueOf(tableName));
if (tableExist) {
logger.info("HTable '" + tableName + "' already exists");
return;
@@ -119,8 +119,8 @@ public class HbaseStreamingInput {
e.printStackTrace();
}
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
byte[] key = new byte[8 + 4];//time + id
@@ -135,7 +135,7 @@ public class HbaseStreamingInput {
Bytes.putInt(key, 8, i);
Put put = new Put(key);
byte[] cell = randomBytes(CELL_SIZE);
- put.add(CF, QN, cell);
+ put.addColumn(CF, QN, cell);
buffer.add(put);
}
table.put(buffer);
@@ -170,8 +170,8 @@ public class HbaseStreamingInput {
}
Random r = new Random();
- HConnection conn = getConnection();
- HTableInterface table = conn.getTable(tableName);
+ Connection conn = getConnection();
+ Table table = conn.getTable(TableName.valueOf(tableName));
long leftBound = getFirstKeyTime(table);
long rightBound = System.currentTimeMillis();
@@ -206,7 +206,7 @@ public class HbaseStreamingInput {
}
}
- private static long getFirstKeyTime(HTableInterface table) throws IOException {
+ private static long getFirstKeyTime(Table table) throws IOException {
long startTime = 0;
Scan scan = new Scan();
@@ -224,8 +224,8 @@ public class HbaseStreamingInput {
}
- private static HConnection getConnection() throws IOException {
- return HConnectionManager.createConnection(HBaseConnection.getCurrentHBaseConfiguration());
+ private static Connection getConnection() throws IOException {
+ return HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
}
private static String formatTime(long time) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
index ca1a060..ea05ab2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HtableAlterMetadataCLI.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.engine.mr.common.BatchConstants;
@@ -50,8 +51,8 @@ public class HtableAlterMetadataCLI extends AbstractApplication {
String metadataValue;
private void alter() throws IOException {
- Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
hbaseAdmin.disableTable(table.getTableName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
index 8ff5b0f..df4e912 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/OrphanHBaseCleanJob.java
@@ -30,10 +30,14 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,9 +56,9 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
Set<String> metastoreWhitelistSet = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
-
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -73,12 +77,13 @@ public class OrphanHBaseCleanJob extends AbstractApplication {
// drop tables
for (String htableName : allTablesNeedToBeDropped) {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ TableName tableName = TableName.valueOf(htableName);
+ if (hbaseAdmin.tableExists(tableName)) {
+ if (hbaseAdmin.isTableEnabled(tableName)) {
+ hbaseAdmin.disableTable(tableName);
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(tableName);
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
index e219c5a..8a93160 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/PingHBaseCLI.java
@@ -22,12 +22,13 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -58,12 +59,12 @@ public class PingHBaseCLI {
Scan scan = new Scan();
int limit = 20;
- HConnection conn = null;
- HTableInterface table = null;
+ Connection conn = null;
+ Table table = null;
ResultScanner scanner = null;
try {
- conn = HConnectionManager.createConnection(hconf);
- table = conn.getTable(hbaseTable);
+ conn = ConnectionFactory.createConnection(hconf);
+ table = conn.getTable(TableName.valueOf(hbaseTable));
scanner = table.getScanner(scan);
int count = 0;
for (Result r : scanner) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
index 01edb1f..db516bb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/RowCounterCLI.java
@@ -22,11 +22,12 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -70,8 +71,8 @@ public class RowCounterCLI {
logger.info("My Scan " + scan.toString());
- HConnection conn = HConnectionManager.createConnection(conf);
- HTableInterface tableInterface = conn.getTable(htableName);
+ Connection conn = ConnectionFactory.createConnection(conf);
+ Table tableInterface = conn.getTable(TableName.valueOf(htableName));
Iterator<Result> iterator = tableInterface.getScanner(scan).iterator();
int counter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 9fe5a23..74a4718 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -40,7 +40,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -56,6 +58,7 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,7 +79,8 @@ public class StorageCleanupJob extends AbstractApplication {
private void cleanUnusedHBaseTables(Configuration conf) throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
// get all kylin hbase tables
- HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
List<String> allTablesNeedToBeDropped = new ArrayList<String>();
@@ -152,22 +156,22 @@ public class StorageCleanupJob extends AbstractApplication {
}
class DeleteHTableRunnable implements Callable {
- HBaseAdmin hbaseAdmin;
+ Admin hbaseAdmin;
String htableName;
- DeleteHTableRunnable(HBaseAdmin hbaseAdmin, String htableName) {
+ DeleteHTableRunnable(Admin hbaseAdmin, String htableName) {
this.hbaseAdmin = hbaseAdmin;
this.htableName = htableName;
}
public Object call() throws Exception {
logger.info("Deleting HBase table " + htableName);
- if (hbaseAdmin.tableExists(htableName)) {
- if (hbaseAdmin.isTableEnabled(htableName)) {
- hbaseAdmin.disableTable(htableName);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htableName))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htableName))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htableName));
}
- hbaseAdmin.deleteTable(htableName);
+ hbaseAdmin.deleteTable(TableName.valueOf(htableName));
logger.info("Deleted HBase table " + htableName);
} else {
logger.info("HBase table" + htableName + " does not exist");
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
index e36f662..42a54c8 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java
@@ -24,16 +24,18 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,14 +51,15 @@ public class UpdateHTableHostCLI {
private List<String> errorMsgs = Lists.newArrayList();
private List<String> htables;
- private HBaseAdmin hbaseAdmin;
+ private Admin hbaseAdmin;
private KylinConfig kylinConfig;
private String oldHostValue;
public UpdateHTableHostCLI(List<String> htables, String oldHostValue) throws IOException {
this.htables = htables;
this.oldHostValue = oldHostValue;
- this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration());
+ Connection conn = ConnectionFactory.createConnection(HBaseConfiguration.create());
+ hbaseAdmin = conn.getAdmin();
this.kylinConfig = KylinConfig.getInstanceFromEnv();
}
@@ -166,9 +169,9 @@ public class UpdateHTableHostCLI {
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
if (oldHostValue.equals(desc.getValue(IRealizationConstants.HTableTag))) {
desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
- hbaseAdmin.disableTable(tableName);
- hbaseAdmin.modifyTable(tableName, desc);
- hbaseAdmin.enableTable(tableName);
+ hbaseAdmin.disableTable(TableName.valueOf(tableName));
+ hbaseAdmin.modifyTable(TableName.valueOf(tableName), desc);
+ hbaseAdmin.enableTable(TableName.valueOf(tableName));
updatedResources.add(tableName);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index 40306c9..fa551f8 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -36,9 +36,9 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
@@ -230,6 +230,7 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
}
}
+
private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName) throws IOException {
String projectResPath = ProjectInstance.concatResourcePath(projectName);
if (!dstStore.exists(projectResPath))
@@ -446,11 +447,11 @@ public class CubeMigrationCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface srcAclHtable = null;
- HTableInterface destAclHtable = null;
+ Table srcAclHtable = null;
+ Table destAclHtable = null;
try {
- srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ srcAclHtable = HBaseConnection.get(srcConfig.getStorageUrl()).getTable(TableName.valueOf(srcConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
// cube acl
Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
@@ -470,7 +471,6 @@ public class CubeMigrationCLI {
destAclHtable.put(put);
}
}
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(srcAclHtable);
IOUtils.closeQuietly(destAclHtable);
@@ -536,13 +536,12 @@ public class CubeMigrationCLI {
case COPY_ACL: {
String cubeId = (String) opt.params[0];
String modelId = (String) opt.params[1];
- HTableInterface destAclHtable = null;
+ Table destAclHtable = null;
try {
- destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME);
+ destAclHtable = HBaseConnection.get(dstConfig.getStorageUrl()).getTable(TableName.valueOf(dstConfig.getMetadataUrlPrefix() + ACL_TABLE_NAME));
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
- destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);
}
@@ -559,7 +558,7 @@ public class CubeMigrationCLI {
}
}
- private static void updateMeta(KylinConfig config){
+ private static void updateMeta(KylinConfig config) {
String[] nodes = config.getRestServers();
for (String node : nodes) {
RestClient restClient = new RestClient(node);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9690649b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index 19e5db0..f52fc3e 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -25,10 +25,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -231,9 +232,9 @@ public class ExtendCubeToHybridCLI {
Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
ProjectInstance project = store.getResource(projectResPath, ProjectInstance.class, projectSerializer);
String projUUID = project.getUuid();
- HTableInterface aclHtable = null;
+ Table aclHtable = null;
try {
- aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(kylinConfig.getMetadataUrlPrefix() + "_acl");
+ aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));
// cube acl
Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
@@ -253,7 +254,6 @@ public class ExtendCubeToHybridCLI {
aclHtable.put(put);
}
}
- aclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(aclHtable);
}
[09/50] [abbrv] kylin git commit: minor,
more extensible of CreateFlatHiveTableStep class
Posted by li...@apache.org.
minor, more extensible of CreateFlatHiveTableStep class
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/05f35f0e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/05f35f0e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/05f35f0e
Branch: refs/heads/master-cdh5.7
Commit: 05f35f0e41269f21c228bbf638a927d3bdf93887
Parents: e393338
Author: lidongsjtu <li...@apache.org>
Authored: Tue Dec 13 10:38:26 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Dec 13 10:38:34 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/05f35f0e/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
index f0e5703..b197f0e 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CreateFlatHiveTableStep.java
@@ -40,7 +40,7 @@ public class CreateFlatHiveTableStep extends AbstractExecutable {
private static final Logger logger = LoggerFactory.getLogger(CreateFlatHiveTableStep.class);
private final BufferedLogger stepLogger = new BufferedLogger(logger);
- private void createFlatHiveTable(KylinConfig config) throws IOException {
+ protected void createFlatHiveTable(KylinConfig config) throws IOException {
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement(getInitStatement());
hiveCmdBuilder.addStatement(getCreateTableStatement());
[50/50] [abbrv] kylin git commit: KYLIN-1672 support kylin on cdh 5.7
Posted by li...@apache.org.
KYLIN-1672 support kylin on cdh 5.7
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/980b8cef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/980b8cef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/980b8cef
Branch: refs/heads/master-cdh5.7
Commit: 980b8ceff60de442b4cb74f250ba018fdbccfba5
Parents: 9690649
Author: Lynne Jiang <ly...@hotmail.com>
Authored: Mon May 16 03:33:27 2016 -0700
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Dec 20 19:26:03 2016 +0800
----------------------------------------------------------------------
dev-support/test_all_against_hdp_2_2_4_2_2.sh | 0
.../kylin/engine/mr/steps/MockupMapContext.java | 15 +-
examples/test_case_data/sandbox/core-site.xml | 146 +++---
examples/test_case_data/sandbox/hbase-site.xml | 162 ++----
examples/test_case_data/sandbox/hdfs-site.xml | 259 ++--------
examples/test_case_data/sandbox/mapred-site.xml | 398 ++++++---------
examples/test_case_data/sandbox/yarn-site.xml | 496 ++-----------------
pom.xml | 16 +-
server/pom.xml | 36 ++
.../storage/hbase/steps/MockupMapContext.java | 19 +-
tool/pom.xml | 12 +
11 files changed, 428 insertions(+), 1131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/dev-support/test_all_against_hdp_2_2_4_2_2.sh
----------------------------------------------------------------------
diff --git a/dev-support/test_all_against_hdp_2_2_4_2_2.sh b/dev-support/test_all_against_hdp_2_2_4_2_2.sh
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
index 847071d..9900465 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MockupMapContext.java
@@ -77,6 +77,7 @@ public class MockupMapContext {
outKV[0] = key;
outKV[1] = value;
}
+
}
@Override
@@ -99,6 +100,7 @@ public class MockupMapContext {
throw new NotImplementedException();
}
+
@Override
public float getProgress() {
throw new NotImplementedException();
@@ -195,17 +197,17 @@ public class MockupMapContext {
}
@Override
- public RawComparator<?> getSortComparator() {
+ public boolean userClassesTakesPrecedence() {
throw new NotImplementedException();
}
@Override
- public String getJar() {
+ public RawComparator<?> getSortComparator() {
throw new NotImplementedException();
}
@Override
- public RawComparator<?> getGroupingComparator() {
+ public String getJar() {
throw new NotImplementedException();
}
@@ -221,7 +223,7 @@ public class MockupMapContext {
@Override
public boolean getProfileEnabled() {
- throw new NotImplementedException();
+ return false;
}
@Override
@@ -308,6 +310,11 @@ public class MockupMapContext {
public RawComparator<?> getCombinerKeyGroupingComparator() {
throw new NotImplementedException();
}
+
+ @Override
+ public RawComparator<?> getGroupingComparator() {
+ return null;
+ }
});
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index 9aa588c..6162406 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -14,152 +14,146 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
+<!--Autogenerated by Cloudera Manager-->
<configuration>
-
<property>
<name>fs.defaultFS</name>
- <value>hdfs://sandbox.hortonworks.com:8020</value>
- <final>true</final>
+ <value>hdfs://quickstart.cloudera:8020</value>
</property>
-
<property>
<name>fs.trash.interval</name>
- <value>360</value>
+ <value>1</value>
</property>
-
<property>
- <name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
- <value>120</value>
+ <name>io.compression.codecs</name>
+ <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value>
</property>
-
<property>
- <name>hadoop.http.authentication.simple.anonymous.allowed</name>
- <value>true</value>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
</property>
-
<property>
- <name>hadoop.proxyuser.falcon.groups</name>
- <value>users</value>
+ <name>hadoop.security.authorization</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.rpc.protection</name>
+ <value>authentication</value>
+ </property>
+ <property>
+ <name>hadoop.security.auth_to_local</name>
+ <value>DEFAULT</value>
</property>
-
<property>
- <name>hadoop.proxyuser.falcon.hosts</name>
+ <name>hadoop.proxyuser.oozie.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hbase.groups</name>
- <value>users</value>
+ <name>hadoop.proxyuser.oozie.groups</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hbase.hosts</name>
+ <name>hadoop.proxyuser.mapred.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hcat.groups</name>
+ <name>hadoop.proxyuser.mapred.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hcat.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <name>hadoop.proxyuser.flume.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hive.groups</name>
- <value>users</value>
+ <name>hadoop.proxyuser.flume.groups</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.HTTP.hosts</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.HTTP.groups</name>
+ <value>*</value>
</property>
-
<property>
<name>hadoop.proxyuser.hive.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.hue.groups</name>
+ <name>hadoop.proxyuser.hive.groups</name>
<value>*</value>
</property>
-
<property>
<name>hadoop.proxyuser.hue.hosts</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.oozie.groups</name>
+ <name>hadoop.proxyuser.hue.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.oozie.hosts</name>
- <value>sandbox.hortonworks.com</value>
+ <name>hadoop.proxyuser.httpfs.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.root.groups</name>
+ <name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.proxyuser.root.hosts</name>
+ <name>hadoop.proxyuser.hdfs.groups</name>
<value>*</value>
</property>
-
<property>
- <name>hadoop.security.auth_to_local</name>
- <value>DEFAULT</value>
+ <name>hadoop.proxyuser.hdfs.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.security.authentication</name>
- <value>simple</value>
+ <name>hadoop.proxyuser.yarn.hosts</name>
+ <value>*</value>
</property>
-
<property>
- <name>hadoop.security.authorization</name>
- <value>false</value>
+ <name>hadoop.proxyuser.yarn.groups</name>
+ <value>*</value>
</property>
-
<property>
- <name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value>
+ <name>hadoop.security.group.mapping</name>
+ <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
</property>
-
<property>
- <name>io.file.buffer.size</name>
- <value>131072</value>
+ <name>hadoop.security.instrumentation.requires.admin</name>
+ <value>false</value>
</property>
-
<property>
- <name>io.serializations</name>
- <value>org.apache.hadoop.io.serializer.WritableSerialization</value>
+ <name>net.topology.script.file.name</name>
+ <value>/etc/hadoop/conf.cloudera.yarn/topology.py</value>
</property>
-
<property>
- <name>ipc.client.connect.max.retries</name>
- <value>50</value>
+ <name>io.file.buffer.size</name>
+ <value>65536</value>
</property>
-
<property>
- <name>ipc.client.connection.maxidletime</name>
- <value>30000</value>
+ <name>hadoop.ssl.enabled</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>hadoop.ssl.require.client.cert</name>
+ <value>false</value>
+ <final>true</final>
</property>
-
<property>
- <name>ipc.client.idlethreshold</name>
- <value>8000</value>
+ <name>hadoop.ssl.keystores.factory.class</name>
+ <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
+ <final>true</final>
</property>
-
<property>
- <name>ipc.server.tcpnodelay</name>
- <value>true</value>
+ <name>hadoop.ssl.server.conf</name>
+ <value>ssl-server.xml</value>
+ <final>true</final>
</property>
-
<property>
- <name>mapreduce.jobtracker.webinterface.trusted</name>
- <value>false</value>
+ <name>hadoop.ssl.client.conf</name>
+ <value>ssl-client.xml</value>
+ <final>true</final>
</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index 734908e..58c6223 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -15,180 +15,104 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
<property>
- <name>hbase.client.keyvalue.maxsize</name>
- <value>10485760</value>
- </property>
-
- <property>
- <name>hbase.client.scanner.caching</name>
- <value>100</value>
+ <name>hbase.rootdir</name>
+ <value>hdfs://quickstart.cloudera:8020/hbase</value>
</property>
-
<property>
- <name>hbase.cluster.distributed</name>
+ <name>hbase.replication</name>
<value>true</value>
</property>
-
- <property>
- <name>hbase.coprocessor.master.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
- </property>
-
<property>
- <name>hbase.coprocessor.region.classes</name>
- <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
+ <name>hbase.client.write.buffer</name>
+ <value>2097152</value>
</property>
-
- <property>
- <name>hbase.defaults.for.version.skip</name>
- <value>true</value>
- </property>
-
<property>
- <name>hbase.hregion.majorcompaction</name>
- <value>604800000</value>
+ <name>hbase.client.pause</name>
+ <value>100</value>
</property>
-
<property>
- <name>hbase.hregion.majorcompaction.jitter</name>
- <value>0.50</value>
+ <name>hbase.client.retries.number</name>
+ <value>35</value>
</property>
-
<property>
- <name>hbase.hregion.max.filesize</name>
- <value>10737418240</value>
+ <name>hbase.client.scanner.caching</name>
+ <value>100</value>
</property>
-
<property>
- <name>hbase.hregion.memstore.block.multiplier</name>
- <value>4</value>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value>10485760</value>
</property>
-
<property>
- <name>hbase.hregion.memstore.flush.size</name>
- <value>134217728</value>
- </property>
-
- <property>
- <name>hbase.hregion.memstore.mslab.enabled</name>
+ <name>hbase.ipc.client.allowsInterrupt</name>
<value>true</value>
</property>
-
<property>
- <name>hbase.hstore.blockingStoreFiles</name>
+ <name>hbase.client.primaryCallTimeout.get</name>
<value>10</value>
</property>
-
- <property>
- <name>hbase.hstore.compactionThreshold</name>
- <value>3</value>
- </property>
-
<property>
- <name>hbase.local.dir</name>
- <value>${hbase.tmp.dir}/local</value>
+ <name>hbase.client.primaryCallTimeout.multiget</name>
+ <value>10</value>
</property>
-
<property>
- <name>hbase.master.info.bindAddress</name>
- <value>0.0.0.0</value>
+ <name>hbase.regionserver.thrift.http</name>
+ <value>false</value>
</property>
-
<property>
- <name>hbase.master.info.port</name>
- <value>60010</value>
+ <name>hbase.thrift.support.proxyuser</name>
+ <value>false</value>
</property>
-
<property>
- <name>hbase.master.port</name>
+ <name>hbase.rpc.timeout</name>
<value>60000</value>
</property>
-
- <property>
- <name>hbase.regionserver.global.memstore.lowerLimit</name>
- <value>0.38</value>
- </property>
-
<property>
- <name>hbase.regionserver.global.memstore.upperLimit</name>
- <value>0.4</value>
- </property>
-
- <property>
- <name>hbase.regionserver.handler.count</name>
- <value>60</value>
+ <name>hbase.snapshot.enabled</name>
+ <value>true</value>
</property>
-
<property>
- <name>hbase.regionserver.info.port</name>
- <value>60030</value>
+ <name>hbase.snapshot.master.timeoutMillis</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.rootdir</name>
- <value>hdfs://sandbox.hortonworks.com:8020/apps/hbase/data</value>
+ <name>hbase.snapshot.region.timeout</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.rpc.protection</name>
- <value>PRIVACY</value>
+ <name>hbase.snapshot.master.timeout.millis</name>
+ <value>60000</value>
</property>
-
<property>
<name>hbase.security.authentication</name>
<value>simple</value>
</property>
-
<property>
- <name>hbase.security.authorization</name>
- <value>true</value>
+ <name>hbase.rpc.protection</name>
+ <value>authentication</value>
</property>
-
<property>
- <name>hbase.superuser</name>
- <value>hbase</value>
+ <name>zookeeper.session.timeout</name>
+ <value>60000</value>
</property>
-
<property>
- <name>hbase.tmp.dir</name>
- <value>/hadoop/hbase</value>
+ <name>zookeeper.znode.parent</name>
+ <value>/hbase</value>
</property>
-
<property>
- <name>hbase.zookeeper.property.clientPort</name>
- <value>2181</value>
+ <name>zookeeper.znode.rootserver</name>
+ <value>root-region-server</value>
</property>
-
<property>
<name>hbase.zookeeper.quorum</name>
- <value>sandbox.hortonworks.com</value>
+ <value>quickstart.cloudera</value>
</property>
-
<property>
- <name>hbase.zookeeper.useMulti</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hfile.block.cache.size</name>
- <value>0.40</value>
- </property>
-
- <property>
- <name>zookeeper.session.timeout</name>
- <value>30000</value>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>2181</value>
</property>
-
<property>
- <name>zookeeper.znode.parent</name>
- <value>/hbase-unsecure</value>
+ <name>hbase.rest.ssl.enabled</name>
+ <value>false</value>
</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index 1175fff..05854bd 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -15,271 +15,68 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>dfs.block.access.token.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>dfs.block.size</name>
- <value>34217472</value>
- </property>
-
- <property>
- <name>dfs.blockreport.initialDelay</name>
- <value>120</value>
- </property>
-
- <property>
- <name>dfs.blocksize</name>
- <value>134217728</value>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.client.read.shortcircuit.streams.cache.size</name>
- <value>4096</value>
- </property>
-
- <property>
- <name>dfs.cluster.administrators</name>
- <value>hdfs</value>
- </property>
-
- <property>
- <name>dfs.datanode.address</name>
- <value>0.0.0.0:50010</value>
- </property>
-
- <property>
- <name>dfs.datanode.balance.bandwidthPerSec</name>
- <value>6250000</value>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir</name>
- <value>/hadoop/hdfs/data</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.data.dir.perm</name>
- <value>750</value>
- </property>
-
<property>
- <name>dfs.datanode.du.reserved</name>
- <value>1073741824</value>
- </property>
-
- <property>
- <name>dfs.datanode.failed.volumes.tolerated</name>
- <value>0</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:50075</value>
- </property>
-
- <property>
- <name>dfs.datanode.https.address</name>
- <value>0.0.0.0:50475</value>
- </property>
-
- <property>
- <name>dfs.datanode.ipc.address</name>
- <value>0.0.0.0:8010</value>
- </property>
-
- <property>
- <name>dfs.datanode.max.transfer.threads</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>dfs.datanode.max.xcievers</name>
- <value>1024</value>
- </property>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- </property>
-
- <property>
- <name>dfs.heartbeat.interval</name>
- <value>3</value>
+ <name>dfs.namenode.name.dir</name>
+ <value>file:///var/lib/hadoop-hdfs/cache/hdfs/dfs/name</value>
</property>
-
<property>
- <name>dfs.hosts.exclude</name>
- <value>/etc/hadoop/conf/dfs.exclude</value>
+ <name>dfs.namenode.servicerpc-address</name>
+ <value>quickstart.cloudera:8022</value>
</property>
-
<property>
- <name>dfs.http.policy</name>
- <value>HTTP_ONLY</value>
+ <name>dfs.https.address</name>
+ <value>quickstart.cloudera:50470</value>
</property>
-
<property>
<name>dfs.https.port</name>
<value>50470</value>
</property>
-
- <property>
- <name>dfs.journalnode.edits.dir</name>
- <value>/hadoop/hdfs/journalnode</value>
- </property>
-
- <property>
- <name>dfs.journalnode.http-address</name>
- <value>0.0.0.0:8480</value>
- </property>
-
- <property>
- <name>dfs.journalnode.https-address</name>
- <value>0.0.0.0:8481</value>
- </property>
-
- <property>
- <name>dfs.namenode.accesstime.precision</name>
- <value>3600000</value>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.read.stale.datanode</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.avoid.write.stale.datanode</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.dir</name>
- <value>/hadoop/hdfs/namesecondary</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.edits.dir</name>
- <value>${dfs.namenode.checkpoint.dir}</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.period</name>
- <value>21600</value>
- </property>
-
- <property>
- <name>dfs.namenode.checkpoint.txns</name>
- <value>1000000</value>
- </property>
-
- <property>
- <name>dfs.namenode.handler.count</name>
- <value>100</value>
- </property>
-
<property>
<name>dfs.namenode.http-address</name>
- <value>sandbox.hortonworks.com:50070</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.namenode.https-address</name>
- <value>sandbox.hortonworks.com:50470</value>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir</name>
- <value>/hadoop/hdfs/namenode</value>
- <final>true</final>
- </property>
-
- <property>
- <name>dfs.namenode.name.dir.restore</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.safemode.threshold-pct</name>
- <value>1.0f</value>
+ <value>quickstart.cloudera:50070</value>
</property>
-
<property>
- <name>dfs.namenode.secondary.http-address</name>
- <value>sandbox.hortonworks.com:50090</value>
+ <name>dfs.replication</name>
+ <value>1</value>
</property>
-
<property>
- <name>dfs.namenode.stale.datanode.interval</name>
- <value>30000</value>
+ <name>dfs.blocksize</name>
+ <value>134217728</value>
</property>
-
<property>
- <name>dfs.namenode.startup.delay.block.deletion.sec</name>
- <value>3600</value>
+ <name>dfs.client.use.datanode.hostname</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.namenode.write.stale.datanode.ratio</name>
- <value>1.0f</value>
+ <name>fs.permissions.umask-mode</name>
+ <value>022</value>
</property>
-
<property>
- <name>dfs.nfs.exports.allowed.hosts</name>
- <value>* rw</value>
+ <name>dfs.namenode.acls.enabled</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.nfs3.dump.dir</name>
- <value>/tmp/.hdfs-nfs</value>
+ <name>dfs.client.use.legacy.blockreader</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.permissions.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.permissions.superusergroup</name>
- <value>hdfs</value>
+ <name>dfs.client.read.shortcircuit</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.replication</name>
- <value>1</value>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/run/hdfs-sockets/dn</value>
</property>
-
<property>
- <name>dfs.replication.max</name>
- <value>50</value>
+ <name>dfs.client.read.shortcircuit.skip.checksum</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.support.append</name>
- <value>true</value>
- <final>true</final>
+ <name>dfs.client.domain.socket.data.traffic</name>
+ <value>false</value>
</property>
-
<property>
- <name>dfs.webhdfs.enabled</name>
+ <name>dfs.datanode.hdfs-blocks-metadata.enabled</name>
<value>true</value>
- <final>true</final>
</property>
-
- <property>
- <name>fs.permissions.umask-mode</name>
- <value>022</value>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index e90f594..c9b1ca4 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -15,241 +15,165 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>io.sort.mb</name>
- <value>128</value>
- </property>
-
- <property>
- <name>mapred.child.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.admin.map.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>mapreduce.admin.reduce.child.java.opts</name>
- <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>mapreduce.admin.user.env</name>
- <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
- </property>
-
- <property>
- <name>mapreduce.am.max-attempts</name>
- <value>2</value>
- </property>
-
- <property>
- <name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/usr/hdp/${hdp.version}/hadoop/lib/snappy-java-1.0.4.1.jar:/etc/hadoop/conf/secure</value>
- </property>
-
- <property>
- <name>mapreduce.application.framework.path</name>
- <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value>
- </property>
-
- <property>
- <name>mapreduce.cluster.administrators</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>mapreduce.framework.name</name>
- <value>yarn</value>
- </property>
-
- <property>
- <name>mapreduce.job.emit-timeline-data</name>
- <value>false</value>
- </property>
-
- <!--the default value on hdp is 0.05, however for test environments we need to be conservative on resource -->
- <property>
- <name>mapreduce.job.reduce.slowstart.completedmaps</name>
- <value>1</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.address</name>
- <value>sandbox.hortonworks.com:10020</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.done-dir</name>
- <value>/mr-history/done</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.intermediate-done-dir</name>
- <value>/mr-history/tmp</value>
- </property>
-
- <property>
- <name>mapreduce.jobhistory.webapp.address</name>
- <value>sandbox.hortonworks.com:19888</value>
- </property>
-
- <property>
- <name>mapreduce.map.java.opts</name>
- <value>-Xmx512m</value>
- </property>
-
- <property>
- <name>mapreduce.map.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>mapreduce.map.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.map.output.compress</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.map.sort.spill.percent</name>
- <value>0.7</value>
- </property>
-
- <property>
- <name>mapreduce.map.speculative</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.output.fileoutputformat.compress.type</name>
- <value>BLOCK</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.input.buffer.percent</name>
- <value>0.0</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.java.opts</name>
- <value>-Xmx200m</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.memory.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name>
- <value>1</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.input.buffer.percent</name>
- <value>0.7</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.merge.percent</name>
- <value>0.66</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.shuffle.parallelcopies</name>
- <value>30</value>
- </property>
-
- <property>
- <name>mapreduce.reduce.speculative</name>
- <value>false</value>
- </property>
-
- <property>
- <name>mapreduce.shuffle.port</name>
- <value>13562</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.factor</name>
- <value>100</value>
- </property>
-
- <property>
- <name>mapreduce.task.io.sort.mb</name>
- <value>128</value>
- </property>
-
- <property>
- <name>mapreduce.task.timeout</name>
- <value>300000</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.admin-command-opts</name>
- <value>-Dhdp.version=${hdp.version}</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx512m</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.log.level</name>
- <value>INFO</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.resource.mb</name>
- <value>512</value>
- </property>
-
- <property>
- <name>yarn.app.mapreduce.am.staging-dir</name>
- <value>/user</value>
- </property>
-
+<property>
+ <name>mapreduce.job.split.metainfo.maxsize</name>
+ <value>10000000</value>
+</property>
+<property>
+ <name>mapreduce.job.counters.max</name>
+ <value>120</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress.type</name>
+ <value>BLOCK</value>
+</property>
+<property>
+ <name>mapreduce.output.fileoutputformat.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.DefaultCodec</value>
+</property>
+<property>
+ <name>mapreduce.map.output.compress.codec</name>
+ <value>org.apache.hadoop.io.compress.SnappyCodec</value>
+</property>
+<property>
+ <name>mapreduce.map.output.compress</name>
+ <value>true</value>
+</property>
+<property>
+ <name>zlib.compress.level</name>
+ <value>DEFAULT_COMPRESSION</value>
+</property>
+<property>
+ <name>mapreduce.task.io.sort.factor</name>
+ <value>64</value>
+</property>
+<property>
+ <name>mapreduce.map.sort.spill.percent</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.reduce.shuffle.parallelcopies</name>
+ <value>10</value>
+</property>
+<property>
+ <name>mapreduce.task.timeout</name>
+ <value>600000</value>
+</property>
+<property>
+ <name>mapreduce.client.submit.file.replication</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.reduces</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.task.io.sort.mb</name>
+ <value>16</value>
+</property>
+<property>
+ <name>mapreduce.map.speculative</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.reduce.speculative</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapreduce.job.reduce.slowstart.completedmaps</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.address</name>
+ <value>quickstart.cloudera:10020</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value>quickstart.cloudera:19888</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.webapp.https.address</name>
+ <value>quickstart.cloudera:19890</value>
+</property>
+<property>
+ <name>mapreduce.jobhistory.admin.address</name>
+ <value>quickstart.cloudera:10033</value>
+</property>
+<property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <value>/user</value>
+</property>
+<property>
+ <name>mapreduce.am.max-attempts</name>
+ <value>2</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.resource.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.resource.cpu-vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.ubertask.enable</name>
+ <value>false</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.command-opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>mapreduce.map.java.opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>mapreduce.reduce.java.opts</name>
+ <value>-Djava.net.preferIPv4Stack=true -Xmx52428800</value>
+</property>
+<property>
+ <name>yarn.app.mapreduce.am.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
+</property>
+<property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>mapreduce.map.cpu.vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>128</value>
+</property>
+<property>
+ <name>mapreduce.reduce.cpu.vcores</name>
+ <value>1</value>
+</property>
+<property>
+ <name>mapreduce.job.heap.memory-mb.ratio</name>
+ <value>0.8</value>
+</property>
+<property>
+ <name>mapreduce.application.classpath</name>
+ <value>/tmp/kylin/*,/usr/lib/hadoop-mapreduce/lib/*,/etc/hadoop/conf:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-mapreduce/.//*,/usr/lib/hbase/hbase-common.jar,/etc/hive/conf</value>
+</property>
+<property>
+ <name>mapreduce.admin.user.env</name>
+ <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH</value>
+</property>
+<property>
+ <name>mapreduce.shuffle.max.connections</name>
+ <value>80</value>
+</property>
</configuration>
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index 8256158..8988d4a 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -15,520 +15,128 @@
limitations under the License.
-->
<configuration>
-
- <property>
- <name>hadoop.registry.rm.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>hadoop.registry.zk.quorum</name>
- <value>sandbox.hortonworks.com:2181</value>
- </property>
-
<property>
<name>yarn.acl.enable</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.admin.acl</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.application.classpath</name>
- <value>$HADOOP_CONF_DIR,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value>
- </property>
-
- <property>
- <name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.http.policy</name>
- <value>HTTP_ONLY</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation.retain-seconds</name>
- <value>2592000</value>
- </property>
-
- <property>
- <name>yarn.log.server.url</name>
- <value>http://sandbox.hortonworks.com:19888/jobhistory/logs</value>
- </property>
-
- <property>
- <name>yarn.node-labels.fs-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <name>yarn.node-labels.fs-store.root-dir</name>
- <value>/system/yarn/node-labels</value>
- </property>
-
- <property>
- <name>yarn.node-labels.manager-class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.address</name>
- <value>0.0.0.0:45454</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.admin-env</name>
- <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
- <value>org.apache.hadoop.mapred.ShuffleHandler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-executor.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.container-monitor.interval-ms</name>
- <value>3000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>0</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
- <value>90</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
- <value>1000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name>
- <value>0.25</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.interval-ms</name>
- <value>135000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
- <value>hadoop-yarn</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.group</name>
- <value>hadoop</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
- <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.local-dirs</name>
- <value>/hadoop/yarn/local</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.compression-type</name>
- <value>gz</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
- <value>30</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
- <value>-1</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-dirs</name>
- <value>/hadoop/yarn/log</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log.retain-second</name>
- <value>604800</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.pmem-check-enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.recovery.dir</name>
- <value>/var/log/hadoop-yarn/nodemanager/recovery-state</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.recovery.enabled</name>
<value>true</value>
</property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>/app-logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
- <value>logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.cpu-vcores</name>
- <value>8</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>9216</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- </property>
-
<property>
- <name>yarn.nodemanager.vmem-pmem-ratio</name>
- <value>10</value>
+ <name>yarn.admin.acl</name>
+ <value>*</value>
</property>
-
<property>
<name>yarn.resourcemanager.address</name>
- <value>sandbox.hortonworks.com:8050</value>
+ <value>quickstart.cloudera:8032</value>
</property>
-
<property>
<name>yarn.resourcemanager.admin.address</name>
- <value>sandbox.hortonworks.com:8141</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.am.max-attempts</name>
- <value>2</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.bind-host</name>
- <value>0.0.0.0</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.connect.max-wait.ms</name>
- <value>900000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.connect.retry-interval.ms</name>
- <value>30000</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
- <value>2000, 500</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.fs.state-store.uri</name>
- <value></value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.ha.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.hostname</name>
- <value>sandbox.hortonworks.com</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.nodes.exclude-path</name>
- <value>/etc/hadoop/conf/yarn.exclude</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.recovery.enabled</name>
- <value>true</value>
+ <value>quickstart.cloudera:8033</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.resource-tracker.address</name>
- <value>sandbox.hortonworks.com:8025</value>
- </property>
-
<property>
<name>yarn.resourcemanager.scheduler.address</name>
- <value>sandbox.hortonworks.com:8030</value>
+ <value>quickstart.cloudera:8030</value>
</property>
-
<property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.state-store.max-completed-applications</name>
- <value>${yarn.resourcemanager.max-completed-applications}</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.store.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
- <value>10</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
- <value>true</value>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value>quickstart.cloudera:8031</value>
</property>
-
<property>
<name>yarn.resourcemanager.webapp.address</name>
- <value>sandbox.hortonworks.com:8088</value>
+ <value>quickstart.cloudera:8088</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
- <value>false</value>
- </property>
-
<property>
<name>yarn.resourcemanager.webapp.https.address</name>
- <value>localhost:8090</value>
+ <value>quickstart.cloudera:8090</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.groups</name>
- <value>*</value>
+ <name>yarn.resourcemanager.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.hcat.hosts</name>
- <value>*</value>
+ <name>yarn.resourcemanager.scheduler.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.groups</name>
- <value>*</value>
+ <name>yarn.resourcemanager.admin.client.thread-count</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.webapp.proxyuser.oozie.hosts</name>
- <value>*</value>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
- <value>true</value>
+ <name>yarn.scheduler.increment-allocation-mb</name>
+ <value>512</value>
</property>
-
<property>
- <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
- <value>10000</value>
+ <name>yarn.scheduler.maximum-allocation-mb</name>
+ <value>2816</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-acl</name>
- <value>world:anyone:rwcda</value>
+ <name>yarn.scheduler.minimum-allocation-vcores</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-address</name>
- <value>localhost:2181</value>
+ <name>yarn.scheduler.increment-allocation-vcores</name>
+ <value>1</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-num-retries</name>
- <value>1000</value>
+ <name>yarn.scheduler.maximum-allocation-vcores</name>
+ <value>2</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-retry-interval-ms</name>
+ <name>yarn.resourcemanager.amliveliness-monitor.interval-ms</name>
<value>1000</value>
</property>
-
<property>
- <name>yarn.resourcemanager.zk-state-store.parent-path</name>
- <value>/rmstore</value>
+ <name>yarn.am.liveness-monitor.expiry-interval-ms</name>
+ <value>600000</value>
</property>
-
- <property>
- <name>yarn.resourcemanager.zk-timeout-ms</name>
- <value>10000</value>
- </property>
-
- <property>
- <name>yarn.scheduler.maximum-allocation-mb</name>
- <value>9216</value>
- </property>
-
<property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1536</value>
+ <name>yarn.resourcemanager.am.max-attempts</name>
+ <value>2</value>
</property>
-
<property>
- <name>yarn.timeline-service.address</name>
- <value>sandbox.hortonworks.com:10200</value>
+ <name>yarn.resourcemanager.container.liveness-monitor.interval-ms</name>
+ <value>600000</value>
</property>
-
<property>
- <name>yarn.timeline-service.bind-host</name>
- <value>0.0.0.0</value>
+ <name>yarn.resourcemanager.nm.liveness-monitor.interval-ms</name>
+ <value>1000</value>
</property>
-
<property>
- <name>yarn.timeline-service.client.max-retries</name>
- <value>30</value>
+ <name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
+ <value>600000</value>
</property>
-
<property>
- <name>yarn.timeline-service.client.retry-interval-ms</name>
- <value>1000</value>
+ <name>yarn.resourcemanager.resource-tracker.client.thread-count</name>
+ <value>50</value>
</property>
-
<property>
- <name>yarn.timeline-service.enabled</name>
- <value>true</value>
+ <name>yarn.application.classpath</name>
+ <value>$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*</value>
</property>
-
<property>
- <name>yarn.timeline-service.generic-application-history.store-class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
+ <name>yarn.resourcemanager.scheduler.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
</property>
-
<property>
- <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+ <name>yarn.scheduler.fair.user-as-default-queue</name>
<value>true</value>
</property>
-
- <property>
- <name>yarn.timeline-service.http-authentication.type</name>
- <value>simple</value>
- </property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.path</name>
- <value>/hadoop/yarn/timeline</value>
+ <name>yarn.scheduler.fair.preemption</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
- <value>104857600</value>
+ <name>yarn.scheduler.fair.sizebasedweight</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
- <value>10000</value>
+ <name>yarn.scheduler.fair.assignmultiple</name>
+ <value>false</value>
</property>
-
<property>
- <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
+ <name>yarn.resourcemanager.max-completed-applications</name>
<value>10000</value>
</property>
-
- <property>
- <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
- <value>300000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.ttl-ms</name>
- <value>2678400000</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.address</name>
- <value>sandbox.hortonworks.com:8188</value>
- </property>
-
- <property>
- <name>yarn.timeline-service.webapp.https.address</name>
- <value>sandbox.hortonworks.com:8190</value>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6d3425e..9b857b5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,19 +46,19 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Hadoop versions -->
- <hadoop2.version>2.7.1</hadoop2.version>
- <yarn.version>2.7.1</yarn.version>
+ <hadoop2.version>2.6.0-cdh5.7.0</hadoop2.version>
+ <yarn.version>2.6.0-cdh5.7.0</yarn.version>
<!-- Hive versions -->
- <hive.version>1.2.1</hive.version>
- <hive-hcatalog.version>1.2.1</hive-hcatalog.version>
+ <hive.version>1.1.0-cdh5.7.0</hive.version>
+ <hive-hcatalog.version>1.1.0-cdh5.7.0</hive-hcatalog.version>
<!-- HBase versions -->
- <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+ <hbase-hadoop2.version>1.2.0-cdh5.7.0</hbase-hadoop2.version>
<kafka.version>0.10.0.0</kafka.version>
<!-- Hadoop deps, keep compatible with hadoop2.version -->
- <zookeeper.version>3.4.6</zookeeper.version>
+ <zookeeper.version>3.4.5-cdh5.7.0</zookeeper.version>
<curator.version>2.7.1</curator.version>
<jackson.version>2.2.4</jackson.version>
<jsr305.version>3.0.1</jsr305.version>
@@ -814,6 +814,10 @@
<id>conjars</id>
<url>http://conjars.org/repo/</url>
</repository>
+ <repository>
+ <id>cloudera</id>
+ <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+ </repository>
</repositories>
<build>
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index cf92fb1..20f4483 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -112,6 +112,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
@@ -131,6 +135,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -146,6 +154,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -161,6 +173,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -176,6 +192,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -199,6 +219,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -214,6 +238,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -229,6 +257,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -262,6 +294,10 @@
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
index d5c3f60..5adf327 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java
@@ -100,11 +100,6 @@ public class MockupMapContext {
}
@Override
- public float getProgress() {
- throw new NotImplementedException();
- }
-
- @Override
public Counter getCounter(Enum<?> counterName) {
throw new NotImplementedException();
}
@@ -165,6 +160,11 @@ public class MockupMapContext {
}
@Override
+ public boolean userClassesTakesPrecedence() {
+ return false;
+ }
+
+ @Override
public Class<? extends InputFormat<?, ?>> getInputFormatClass() throws ClassNotFoundException {
throw new NotImplementedException();
}
@@ -214,10 +214,6 @@ public class MockupMapContext {
throw new NotImplementedException();
}
- @Override
- public boolean getTaskCleanupNeeded() {
- throw new NotImplementedException();
- }
@Override
public boolean getProfileEnabled() {
@@ -230,11 +226,6 @@ public class MockupMapContext {
}
@Override
- public IntegerRanges getProfileTaskRange(boolean isMap) {
- throw new NotImplementedException();
- }
-
- @Override
public String getUser() {
throw new NotImplementedException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/980b8cef/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 8eddec4..3d466f0 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -49,6 +49,18 @@
<!--Env-->
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${yarn.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${yarn.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<scope>provided</scope>
[14/50] [abbrv] kylin git commit: KYLIN-1832 code review
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterOldTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterOldTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterOldTest.java
deleted file mode 100644
index 5d17fea..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/measure/hll/HyperLogLogCounterOldTest.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.measure.hll;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterOld;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * @author yangli9
- *
- */
-public class HyperLogLogCounterOldTest {
-
- ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
- Random rand1 = new Random(1);
- Random rand2 = new Random(2);
- Random rand3 = new Random(3);
- int errorCount1 = 0;
- int errorCount2 = 0;
- int errorCount3 = 0;
-
- @Test
- public void testOneAdd() throws IOException {
- HyperLogLogPlusCounterOld hllc = new HyperLogLogPlusCounterOld(14);
- HyperLogLogPlusCounterOld one = new HyperLogLogPlusCounterOld(14);
- for (int i = 0; i < 1000000; i++) {
- one.clear();
- one.add(rand1.nextInt());
- hllc.merge(one);
- }
- assertTrue(hllc.getCountEstimate() > 1000000 * 0.9);
- }
-
- @Test
- public void testPeekLength() throws IOException {
- HyperLogLogPlusCounterOld hllc = new HyperLogLogPlusCounterOld(10);
- HyperLogLogPlusCounterOld copy = new HyperLogLogPlusCounterOld(10);
- byte[] value = new byte[10];
- for (int i = 0; i < 200000; i++) {
- rand1.nextBytes(value);
- hllc.add(value);
-
- buf.clear();
- hllc.writeRegisters(buf);
-
- int len = buf.position();
- buf.position(0);
- assertEquals(len, hllc.peekLength(buf));
-
- copy.readRegisters(buf);
- assertEquals(len, buf.position());
- assertEquals(hllc, copy);
- }
- buf.clear();
- }
-
- private Set<String> generateTestData(int n) {
- Set<String> testData = new HashSet<String>();
- for (int i = 0; i < n; i++) {
- String[] samples = generateSampleData();
- for (String sample : samples) {
- testData.add(sample);
- }
- }
- return testData;
- }
-
- // simulate the visit (=visitor+id)
- private String[] generateSampleData() {
-
- StringBuilder buf = new StringBuilder();
- for (int i = 0; i < 19; i++) {
- buf.append(Math.abs(rand1.nextInt()) % 10);
- }
- String header = buf.toString();
-
- int size = Math.abs(rand3.nextInt()) % 9 + 1;
- String[] samples = new String[size];
- for (int k = 0; k < size; k++) {
- buf = new StringBuilder(header);
- buf.append("-");
- for (int i = 0; i < 10; i++) {
- buf.append(Math.abs(rand3.nextInt()) % 10);
- }
- samples[k] = buf.toString();
- }
-
- return samples;
- }
-
- @Test
- public void countTest() throws IOException {
- int n = 10;
- for (int i = 0; i < 5; i++) {
- count(n);
- n *= 10;
- }
- }
-
- private void count(int n) throws IOException {
- Set<String> testSet = generateTestData(n);
-
- HyperLogLogPlusCounterOld hllc = newHLLC();
- for (String testData : testSet) {
- hllc.add(Bytes.toBytes(testData));
- }
- long estimate = hllc.getCountEstimate();
- double errorRate = hllc.getErrorRate();
- double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size();
- System.out.println(estimate);
- System.out.println(testSet.size());
- System.out.println(errorRate);
- System.out.println("=" + actualError);
- Assert.assertTrue(actualError < errorRate * 3.0);
-
- checkSerialize(hllc);
- }
-
- private void checkSerialize(HyperLogLogPlusCounterOld hllc) throws IOException {
- long estimate = hllc.getCountEstimate();
- buf.clear();
- hllc.writeRegisters(buf);
- buf.flip();
- hllc.readRegisters(buf);
- Assert.assertEquals(estimate, hllc.getCountEstimate());
- }
-
- @Test
- public void mergeTest() throws IOException {
- double error = 0;
- int n = 100;
- for (int i = 0; i < n; i++) {
- double e = merge(i);
- error += e;
- }
- System.out.println("Total average error is " + error / n);
-
- System.out.println(" errorRateCount1 is " + errorCount1 + "!");
- System.out.println(" errorRateCount2 is " + errorCount2 + "!");
- System.out.println(" errorRateCount3 is " + errorCount3 + "!");
-
- Assert.assertTrue(errorCount1 <= n * 0.30);
- Assert.assertTrue(errorCount2 <= n * 0.05);
- Assert.assertTrue(errorCount3 <= n * 0.02);
- }
-
- private double merge(int round) throws IOException {
- int ln = 20;
- int dn = 100 * (round + 1);
- Set<String> testSet = new HashSet<String>();
- HyperLogLogPlusCounterOld[] hllcs = new HyperLogLogPlusCounterOld[ln];
- for (int i = 0; i < ln; i++) {
- hllcs[i] = newHLLC();
- for (int k = 0; k < dn; k++) {
- String[] samples = generateSampleData();
- for (String data : samples) {
- testSet.add(data);
- hllcs[i].add(Bytes.toBytes(data));
- }
- }
- }
- HyperLogLogPlusCounterOld mergeHllc = newHLLC();
- for (HyperLogLogPlusCounterOld hllc : hllcs) {
- mergeHllc.merge(serDes(hllc));
- }
-
- double errorRate = mergeHllc.getErrorRate();
- long estimate = mergeHllc.getCountEstimate();
- double actualError = Math.abs((double) (testSet.size() - estimate) / testSet.size());
-
- System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
- Assert.assertTrue(actualError < 0.1);
-
- if (actualError > errorRate) {
- errorCount1++;
- }
- if (actualError > 2 * errorRate) {
- errorCount2++;
- }
- if (actualError > 3 * errorRate) {
- errorCount3++;
- }
-
- return actualError;
- }
-
- private HyperLogLogPlusCounterOld serDes(HyperLogLogPlusCounterOld hllc) throws IOException {
- buf.clear();
- hllc.writeRegisters(buf);
- buf.flip();
- HyperLogLogPlusCounterOld copy = new HyperLogLogPlusCounterOld(hllc.getPrecision());
- copy.readRegisters(buf);
- Assert.assertEquals(copy.getCountEstimate(), hllc.getCountEstimate());
- return copy;
- }
-
- @Test
- public void testPerformance() throws IOException {
- int N = 3; // reduce N HLLC into one
- int M = 1000; // for M times, use 100000 for real perf test
-
- HyperLogLogPlusCounterOld samples[] = new HyperLogLogPlusCounterOld[N];
- for (int i = 0; i < N; i++) {
- samples[i] = newHLLC();
- for (String str : generateTestData(10000))
- samples[i].add(str);
- }
-
- System.out.println("Perf test running ... ");
- long start = System.currentTimeMillis();
- HyperLogLogPlusCounterOld sum = newHLLC();
- for (int i = 0; i < M; i++) {
- sum.clear();
- for (int j = 0; j < N; j++) {
- sum.merge(samples[j]);
- checkSerialize(sum);
- }
- }
- long duration = System.currentTimeMillis() - start;
- System.out.println("Perf test result: " + duration / 1000 + " seconds");
- }
-
- @Test
- public void testEquivalence() {
- byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 };
- byte[] b = new byte[] { 3, 4, 42 };
- HyperLogLogPlusCounterOld ha = new HyperLogLogPlusCounterOld();
- HyperLogLogPlusCounterOld hb = new HyperLogLogPlusCounterOld();
- ha.add(a, 1, 3);
- hb.add(b);
-
- Assert.assertTrue(ha.getCountEstimate() == hb.getCountEstimate());
- }
-
- private HyperLogLogPlusCounterOld newHLLC() {
- return new HyperLogLogPlusCounterOld(16);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/test/java/org/apache/kylin/measure/hll2/HyperLogLogCounterNewTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hll2/HyperLogLogCounterNewTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hll2/HyperLogLogCounterNewTest.java
deleted file mode 100644
index feb8c8e..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/measure/hll2/HyperLogLogCounterNewTest.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.measure.hll2;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterOld;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
-import org.apache.kylin.measure.hllc.RegisterType;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Created by xiefan on 16-12-12.
- */
-public class HyperLogLogCounterNewTest {
- ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
- Random rand1 = new Random(1);
- Random rand2 = new Random(2);
- Random rand3 = new Random(3);
- int errorCount1 = 0;
- int errorCount2 = 0;
- int errorCount3 = 0;
-
- @Test
- public void testOneAdd() throws IOException {
- HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew(14);
- HyperLogLogPlusCounterNew one = new HyperLogLogPlusCounterNew(14);
- for (int i = 0; i < 1000000; i++) {
- one.clear();
- one.add(rand1.nextInt());
- hllc.merge(one);
- }
- System.out.println(hllc.getCountEstimate());
- assertTrue(hllc.getCountEstimate() > 1000000 * 0.9);
- }
-
- @Test
- public void tesSparseEstimate() throws IOException {
- HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew(14);
- for (int i = 0; i < 10; i++) {
- hllc.add(i);
- }
- System.out.println(hllc.getCountEstimate());
- assertTrue(hllc.getCountEstimate() > 10 * 0.9);
- }
-
- @Test
- public void countTest() throws IOException {
- int n = 10;
- for (int i = 0; i < 5; i++) {
- count(n);
- n *= 10;
- }
- }
-
- @Test
- public void mergeTest() throws IOException {
- double error = 0;
- int n = 100;
- for (int i = 0; i < n; i++) {
- double e = merge(i);
- error += e;
- }
- System.out.println("Total average error is " + error / n);
-
- System.out.println(" errorRateCount1 is " + errorCount1 + "!");
- System.out.println(" errorRateCount2 is " + errorCount2 + "!");
- System.out.println(" errorRateCount3 is " + errorCount3 + "!");
-
- Assert.assertTrue(errorCount1 <= n * 0.30);
- Assert.assertTrue(errorCount2 <= n * 0.05);
- Assert.assertTrue(errorCount3 <= n * 0.02);
- }
-
- /*
- compare the result of two different hll counter
- */
- @Test
- public void compareResult() {
- int p = 12; //4096
- int m = 1 << p;
-
- for (int t = 0; t < 5; t++) {
- //compare sparse
- HyperLogLogPlusCounterOld oldCounter = new HyperLogLogPlusCounterOld(p);
- HyperLogLogPlusCounterNew newCounter = new HyperLogLogPlusCounterNew(p);
-
- for (int i = 0; i < 20; i++) {
- //int r = rand1.nextInt();
- oldCounter.add(i);
- newCounter.add(i);
- }
- assertEquals(RegisterType.SPARSE, newCounter.getRegisterType());
- assertEquals(oldCounter.getCountEstimate(), newCounter.getCountEstimate());
- //compare dense
- for (int i = 0; i < m; i++) {
- oldCounter.add(i);
- newCounter.add(i);
- }
- assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
- assertEquals(oldCounter.getCountEstimate(), newCounter.getCountEstimate());
- }
-
- }
-
- @Test
- public void testPeekLength() throws IOException {
- HyperLogLogPlusCounterNew hllc = new HyperLogLogPlusCounterNew(10);
- HyperLogLogPlusCounterNew copy = new HyperLogLogPlusCounterNew(10);
- byte[] value = new byte[10];
- for (int i = 0; i < 200000; i++) {
- rand1.nextBytes(value);
- hllc.add(value);
-
- buf.clear();
- hllc.writeRegisters(buf);
-
- int len = buf.position();
- buf.position(0);
- assertEquals(len, hllc.peekLength(buf));
-
- copy.readRegisters(buf);
- assertEquals(len, buf.position());
- assertEquals(hllc, copy);
- }
- buf.clear();
- }
-
- @Test
- public void testEquivalence() {
- byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 };
- byte[] b = new byte[] { 3, 4, 42 };
- HyperLogLogPlusCounterNew ha = new HyperLogLogPlusCounterNew();
- HyperLogLogPlusCounterNew hb = new HyperLogLogPlusCounterNew();
- ha.add(a, 1, 3);
- hb.add(b);
-
- Assert.assertTrue(ha.getCountEstimate() == hb.getCountEstimate());
- }
-
- @Test
- public void testAutoChangeToSparse() {
- int p = 15;
- int m = 1 << p;
- HyperLogLogPlusCounterNew counter = new HyperLogLogPlusCounterNew(p);
- assertEquals(RegisterType.SPARSE, counter.getRegisterType());
- double over = HyperLogLogPlusCounterNew.overflowFactor * m;
- int overFlow = (int) over + 1000;
- for (int i = 0; i < overFlow; i++)
- counter.add(i);
- assertEquals(RegisterType.DENSE, counter.getRegisterType());
- }
-
- @Test
- public void testSerialilze() throws Exception {
- //test sparse serialize
- int p = 15;
- int m = 1 << p;
- HyperLogLogPlusCounterNew counter = new HyperLogLogPlusCounterNew(p);
- counter.add(123);
- assertEquals(RegisterType.SPARSE, counter.getRegisterType());
- checkSerialize(counter);
- //test dense serialize
- double over = HyperLogLogPlusCounterNew.overflowFactor * m;
- int overFlow = (int) over + 1000;
- for (int i = 0; i < overFlow; i++)
- counter.add(i);
- assertEquals(RegisterType.DENSE, counter.getRegisterType());
- checkSerialize(counter);
- }
-
- private Set<String> generateTestData(int n) {
- Set<String> testData = new HashSet<String>();
- for (int i = 0; i < n; i++) {
- String[] samples = generateSampleData();
- for (String sample : samples) {
- testData.add(sample);
- }
- }
- return testData;
- }
-
- // simulate the visit (=visitor+id)
- private String[] generateSampleData() {
-
- StringBuilder buf = new StringBuilder();
- for (int i = 0; i < 19; i++) {
- buf.append(Math.abs(rand1.nextInt()) % 10);
- }
- String header = buf.toString();
-
- int size = Math.abs(rand3.nextInt()) % 9 + 1;
- String[] samples = new String[size];
- for (int k = 0; k < size; k++) {
- buf = new StringBuilder(header);
- buf.append("-");
- for (int i = 0; i < 10; i++) {
- buf.append(Math.abs(rand3.nextInt()) % 10);
- }
- samples[k] = buf.toString();
- }
-
- return samples;
- }
-
- private double merge(int round) throws IOException {
- int ln = 20;
- int dn = 100 * (round + 1);
- Set<String> testSet = new HashSet<String>();
- HyperLogLogPlusCounterNew[] hllcs = new HyperLogLogPlusCounterNew[ln];
- for (int i = 0; i < ln; i++) {
- hllcs[i] = newHLLC();
- for (int k = 0; k < dn; k++) {
- String[] samples = generateSampleData();
- for (String data : samples) {
- testSet.add(data);
- hllcs[i].add(Bytes.toBytes(data));
- }
- }
- }
- HyperLogLogPlusCounterNew mergeHllc = newHLLC();
- for (HyperLogLogPlusCounterNew hllc : hllcs) {
- mergeHllc.merge(hllc);
- }
-
- double errorRate = mergeHllc.getErrorRate();
- long estimate = mergeHllc.getCountEstimate();
- double actualError = Math.abs((double) (testSet.size() - estimate) / testSet.size());
-
- System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
- Assert.assertTrue(actualError < 0.1);
-
- if (actualError > errorRate) {
- errorCount1++;
- }
- if (actualError > 2 * errorRate) {
- errorCount2++;
- }
- if (actualError > 3 * errorRate) {
- errorCount3++;
- }
-
- return actualError;
- }
-
- private HyperLogLogPlusCounterNew newHLLC() {
- return new HyperLogLogPlusCounterNew(16);
- }
-
- private void count(int n) throws IOException {
- Set<String> testSet = generateTestData(n);
-
- HyperLogLogPlusCounterNew hllc = newHLLC();
- for (String testData : testSet) {
- hllc.add(Bytes.toBytes(testData));
- }
- long estimate = hllc.getCountEstimate();
- double errorRate = hllc.getErrorRate();
- double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size();
- System.out.println(estimate);
- System.out.println(testSet.size());
- System.out.println(errorRate);
- System.out.println("=" + actualError);
- Assert.assertTrue(actualError < errorRate * 3.0);
-
- checkSerialize(hllc);
- }
-
- private void checkSerialize(HyperLogLogPlusCounterNew hllc) throws IOException {
- long estimate = hllc.getCountEstimate();
- buf.clear();
- hllc.writeRegisters(buf);
- buf.flip();
- hllc.readRegisters(buf);
- Assert.assertEquals(estimate, hllc.getCountEstimate());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/test/java/org/apache/kylin/measure/hll2/NewHyperLogLogBenchmarkTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hll2/NewHyperLogLogBenchmarkTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hll2/NewHyperLogLogBenchmarkTest.java
deleted file mode 100644
index bfb87f9..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/measure/hll2/NewHyperLogLogBenchmarkTest.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.measure.hll2;
-
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterOld;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
-import org.apache.kylin.measure.hllc.RegisterType;
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
-import java.util.Random;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Created by xiefan on 16-12-12.
- */
-public class NewHyperLogLogBenchmarkTest {
-
- public static final Random rand = new Random(1);
-
- final int testTimes = 10000;
-
- @Test
- public void denseToDenseRegisterMergeBenchmark() throws Exception {
- final int p = 15;
- int m = 1 << p;
-
- System.out.println("m : " + m);
- double oldFactor = HyperLogLogPlusCounterNew.overflowFactor;
- HyperLogLogPlusCounterNew.overflowFactor = 1.1; //keep sparse
- for (int cardinality : getTestDataDivide(m)) {
- final HyperLogLogPlusCounterOld oldCounter = new HyperLogLogPlusCounterOld(p);
- final HyperLogLogPlusCounterOld oldCounter2 = getRandOldCounter(p, cardinality);
- long oldTime = runTestCase(new TestCase() {
- @Override
- public void run() {
-
- for (int i = 0; i < testTimes; i++) {
- oldCounter.merge(oldCounter2);
- }
- }
- });
- final HyperLogLogPlusCounterNew newCounter = new HyperLogLogPlusCounterNew(p, RegisterType.DENSE);
- final HyperLogLogPlusCounterNew newCounter2 = new HyperLogLogPlusCounterNew(p, RegisterType.DENSE);
- for (int i = 0; i < testTimes; i++)
- newCounter2.add(i);
- long newTime = runTestCase(new TestCase() {
- @Override
- public void run() {
- for (int i = 0; i < testTimes; i++) {
- newCounter.merge(newCounter2);
- }
- }
- });
- assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
- assertEquals(RegisterType.DENSE, newCounter2.getRegisterType());
- System.out.println("----------------------------");
- System.out.println("cardinality : " + cardinality);
- System.out.println("old time : " + oldTime);
- System.out.println("new time : " + newTime);
- }
- HyperLogLogPlusCounterNew.overflowFactor = oldFactor;
- }
-
- @Test
- public void sparseToSparseMergeBenchmark() throws Exception {
- final int p = 15;
- int m = 1 << p;
- System.out.println("m : " + m);
- double oldFactor = HyperLogLogPlusCounterNew.overflowFactor;
- HyperLogLogPlusCounterNew.overflowFactor = 1.1; //keep sparse
- for (int cardinality : getTestDataDivide(m)) {
- final HyperLogLogPlusCounterOld oldCounter = new HyperLogLogPlusCounterOld(p);
- final HyperLogLogPlusCounterOld oldCounter2 = getRandOldCounter(p, cardinality);
- long oldTime = runTestCase(new TestCase() {
- @Override
- public void run() {
-
- for (int i = 0; i < testTimes; i++) {
- oldCounter.merge(oldCounter2);
- }
- }
- });
- final HyperLogLogPlusCounterNew newCounter = new HyperLogLogPlusCounterNew(p);
- final HyperLogLogPlusCounterNew newCounter2 = getRandNewCounter(p, cardinality);
- long newTime = runTestCase(new TestCase() {
- @Override
- public void run() {
- for (int i = 0; i < testTimes; i++) {
- newCounter.merge(newCounter2);
- }
- }
- });
- assertEquals(RegisterType.SPARSE, newCounter.getRegisterType());
- assertEquals(RegisterType.SPARSE, newCounter2.getRegisterType());
- System.out.println("----------------------------");
- System.out.println("cardinality : " + cardinality);
- System.out.println("old time : " + oldTime);
- System.out.println("new time : " + newTime);
- }
- HyperLogLogPlusCounterNew.overflowFactor = oldFactor;
- }
-
- @Test
- public void sparseToDenseRegisterMergeBenchmark() throws Exception {
- final int p = 15;
- int m = 1 << p;
- System.out.println("m : " + m);
- double oldFactor = HyperLogLogPlusCounterNew.overflowFactor;
- HyperLogLogPlusCounterNew.overflowFactor = 1.1; //keep sparse
- for (int cardinality : getTestDataDivide(m)) {
- System.out.println("----------------------------");
- System.out.println("cardinality : " + cardinality);
- final HyperLogLogPlusCounterOld oldCounter = new HyperLogLogPlusCounterOld(p);
- final HyperLogLogPlusCounterOld oldCounter2 = getRandOldCounter(p, cardinality);
- long oldTime = runTestCase(new TestCase() {
- @Override
- public void run() {
- for (int i = 0; i < testTimes; i++) {
- oldCounter.merge(oldCounter2);
- }
- }
- });
- final HyperLogLogPlusCounterNew newCounter = new HyperLogLogPlusCounterNew(p, RegisterType.DENSE);
- final HyperLogLogPlusCounterNew newCounter2 = getRandNewCounter(p, cardinality);
- long newTime = runTestCase(new TestCase() {
- @Override
- public void run() {
- for (int i = 0; i < testTimes; i++) {
- newCounter.merge(newCounter2);
- }
- }
- });
- assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
- assertEquals(RegisterType.SPARSE, newCounter2.getRegisterType());
- System.out.println("old time : " + oldTime);
- System.out.println("new time : " + newTime);
- }
- HyperLogLogPlusCounterNew.overflowFactor = oldFactor;
- }
-
- @Test
- public void sparseSerializeBenchmark() throws Exception {
- final int p = 15;
- int m = 1 << p;
- double oldFactor = HyperLogLogPlusCounterNew.overflowFactor;
- HyperLogLogPlusCounterNew.overflowFactor = 1.1; //keep sparse
- for (int cardinality : getTestDataDivide(m)) {
- System.out.println("----------------------------");
- System.out.println("cardinality : " + cardinality);
- final HyperLogLogPlusCounterOld oldCounter = getRandOldCounter(p, cardinality);
- long oldTime = runTestCase(new TestCase() {
- @Override
- public void run() throws Exception {
- ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
- long totalBytes = 0;
- for (int i = 0; i < testTimes; i++) {
- buf.clear();
- oldCounter.writeRegisters(buf);
- totalBytes += buf.position();
- buf.flip();
- oldCounter.readRegisters(buf);
- }
- System.out.println("old serialize bytes : " + totalBytes / testTimes + "B");
- }
- });
- final HyperLogLogPlusCounterNew newCounter = getRandNewCounter(p, cardinality);
- long newTime = runTestCase(new TestCase() {
- @Override
- public void run() throws Exception {
- ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
- long totalBytes = 0;
- for (int i = 0; i < testTimes; i++) {
- buf.clear();
- newCounter.writeRegisters(buf);
- totalBytes += buf.position();
- buf.flip();
- newCounter.readRegisters(buf);
- }
- System.out.println("new serialize bytes : " + totalBytes / testTimes + "B");
- }
- });
- assertEquals(RegisterType.SPARSE, newCounter.getRegisterType());
- System.out.println("old serialize time : " + oldTime);
- System.out.println("new serialize time : " + newTime);
- }
- HyperLogLogPlusCounterNew.overflowFactor = oldFactor;
- }
-
- @Test
- public void denseSerializeBenchmark() throws Exception {
- final int p = 15;
- int m = 1 << p;
- double oldFactor = HyperLogLogPlusCounterNew.overflowFactor;
- HyperLogLogPlusCounterNew.overflowFactor = 0; //keep sparse
- for (int cardinality : getTestDataDivide(m)) {
- System.out.println("----------------------------");
- System.out.println("cardinality : " + cardinality);
- final HyperLogLogPlusCounterOld oldCounter = getRandOldCounter(p, cardinality);
- long oldTime = runTestCase(new TestCase() {
- @Override
- public void run() throws Exception {
- ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
- long totalBytes = 0;
- for (int i = 0; i < testTimes; i++) {
- buf.clear();
- oldCounter.writeRegisters(buf);
- totalBytes += buf.position();
- buf.flip();
- oldCounter.readRegisters(buf);
- }
- System.out.println("old serialize bytes : " + totalBytes / testTimes + "B");
- }
- });
- final HyperLogLogPlusCounterNew newCounter = getRandNewCounter(p, cardinality, RegisterType.DENSE);
- long newTime = runTestCase(new TestCase() {
- @Override
- public void run() throws Exception {
- ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
- long totalBytes = 0;
- for (int i = 0; i < testTimes; i++) {
- buf.clear();
- newCounter.writeRegisters(buf);
- totalBytes += buf.position();
- buf.flip();
- newCounter.readRegisters(buf);
- }
- System.out.println("new serialize bytes : " + totalBytes / testTimes + "B");
- }
- });
- assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
- System.out.println("old serialize time : " + oldTime);
- System.out.println("new serialize time : " + newTime);
- }
- HyperLogLogPlusCounterNew.overflowFactor = oldFactor;
- }
-
- interface TestCase {
- void run() throws Exception;
- }
-
- public long runTestCase(TestCase testCase) throws Exception {
- long startTime = System.currentTimeMillis();
- testCase.run();
- return System.currentTimeMillis() - startTime;
- }
-
- public HyperLogLogPlusCounterOld getRandOldCounter(int p, int num) {
- HyperLogLogPlusCounterOld c = new HyperLogLogPlusCounterOld(p);
- for (int i = 0; i < num; i++)
- c.add(i);
- return c;
- }
-
- public HyperLogLogPlusCounterNew getRandNewCounter(int p, int num) {
- HyperLogLogPlusCounterNew c = new HyperLogLogPlusCounterNew(p);
- for (int i = 0; i < num; i++)
- c.add(i);
- return c;
- }
-
- public HyperLogLogPlusCounterNew getRandNewCounter(int p, int num, RegisterType type) {
- HyperLogLogPlusCounterNew c = new HyperLogLogPlusCounterNew(p, type);
- for (int i = 0; i < num; i++)
- c.add(i);
- return c;
- }
-
- public static int[] getTestDataDivide(int m) {
- return new int[] { 1, 5, 10, 100, m / 200, m / 100, m / 50, m / 20, m / 10, m };
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
new file mode 100644
index 0000000..c4a97cd
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterOldTest.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.measure.hllc;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.measure.hllc.HLLCounterOld;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @author yangli9
+ *
+ */
+@SuppressWarnings("deprecation")
+public class HLLCounterOldTest {
+
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ Random rand1 = new Random(1);
+ Random rand2 = new Random(2);
+ Random rand3 = new Random(3);
+ int errorCount1 = 0;
+ int errorCount2 = 0;
+ int errorCount3 = 0;
+
+ @Test
+ public void testOneAdd() throws IOException {
+ HLLCounterOld hllc = new HLLCounterOld(14);
+ HLLCounterOld one = new HLLCounterOld(14);
+ for (int i = 0; i < 1000000; i++) {
+ one.clear();
+ one.add(rand1.nextInt());
+ hllc.merge(one);
+ }
+ assertTrue(hllc.getCountEstimate() > 1000000 * 0.9);
+ }
+
+ @Test
+ public void testPeekLength() throws IOException {
+ HLLCounterOld hllc = new HLLCounterOld(10);
+ HLLCounterOld copy = new HLLCounterOld(10);
+ byte[] value = new byte[10];
+ for (int i = 0; i < 200000; i++) {
+ rand1.nextBytes(value);
+ hllc.add(value);
+
+ buf.clear();
+ hllc.writeRegisters(buf);
+
+ int len = buf.position();
+ buf.position(0);
+ assertEquals(len, hllc.peekLength(buf));
+
+ copy.readRegisters(buf);
+ assertEquals(len, buf.position());
+ assertEquals(hllc, copy);
+ }
+ buf.clear();
+ }
+
+ private Set<String> generateTestData(int n) {
+ Set<String> testData = new HashSet<String>();
+ for (int i = 0; i < n; i++) {
+ String[] samples = generateSampleData();
+ for (String sample : samples) {
+ testData.add(sample);
+ }
+ }
+ return testData;
+ }
+
+ // simulate the visit (=visitor+id)
+ private String[] generateSampleData() {
+
+ StringBuilder buf = new StringBuilder();
+ for (int i = 0; i < 19; i++) {
+ buf.append(Math.abs(rand1.nextInt()) % 10);
+ }
+ String header = buf.toString();
+
+ int size = Math.abs(rand3.nextInt()) % 9 + 1;
+ String[] samples = new String[size];
+ for (int k = 0; k < size; k++) {
+ buf = new StringBuilder(header);
+ buf.append("-");
+ for (int i = 0; i < 10; i++) {
+ buf.append(Math.abs(rand3.nextInt()) % 10);
+ }
+ samples[k] = buf.toString();
+ }
+
+ return samples;
+ }
+
+ @Test
+ public void countTest() throws IOException {
+ int n = 10;
+ for (int i = 0; i < 5; i++) {
+ count(n);
+ n *= 10;
+ }
+ }
+
+ private void count(int n) throws IOException {
+ Set<String> testSet = generateTestData(n);
+
+ HLLCounterOld hllc = newHLLC();
+ for (String testData : testSet) {
+ hllc.add(Bytes.toBytes(testData));
+ }
+ long estimate = hllc.getCountEstimate();
+ double errorRate = hllc.getErrorRate();
+ double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size();
+ System.out.println(estimate);
+ System.out.println(testSet.size());
+ System.out.println(errorRate);
+ System.out.println("=" + actualError);
+ Assert.assertTrue(actualError < errorRate * 3.0);
+
+ checkSerialize(hllc);
+ }
+
+ private void checkSerialize(HLLCounterOld hllc) throws IOException {
+ long estimate = hllc.getCountEstimate();
+ buf.clear();
+ hllc.writeRegisters(buf);
+ buf.flip();
+ hllc.readRegisters(buf);
+ Assert.assertEquals(estimate, hllc.getCountEstimate());
+ }
+
+ @Test
+ public void mergeTest() throws IOException {
+ double error = 0;
+ int n = 100;
+ for (int i = 0; i < n; i++) {
+ double e = merge(i);
+ error += e;
+ }
+ System.out.println("Total average error is " + error / n);
+
+ System.out.println(" errorRateCount1 is " + errorCount1 + "!");
+ System.out.println(" errorRateCount2 is " + errorCount2 + "!");
+ System.out.println(" errorRateCount3 is " + errorCount3 + "!");
+
+ Assert.assertTrue(errorCount1 <= n * 0.30);
+ Assert.assertTrue(errorCount2 <= n * 0.05);
+ Assert.assertTrue(errorCount3 <= n * 0.02);
+ }
+
+ private double merge(int round) throws IOException {
+ int ln = 20;
+ int dn = 100 * (round + 1);
+ Set<String> testSet = new HashSet<String>();
+ HLLCounterOld[] hllcs = new HLLCounterOld[ln];
+ for (int i = 0; i < ln; i++) {
+ hllcs[i] = newHLLC();
+ for (int k = 0; k < dn; k++) {
+ String[] samples = generateSampleData();
+ for (String data : samples) {
+ testSet.add(data);
+ hllcs[i].add(Bytes.toBytes(data));
+ }
+ }
+ }
+ HLLCounterOld mergeHllc = newHLLC();
+ for (HLLCounterOld hllc : hllcs) {
+ mergeHllc.merge(serDes(hllc));
+ }
+
+ double errorRate = mergeHllc.getErrorRate();
+ long estimate = mergeHllc.getCountEstimate();
+ double actualError = Math.abs((double) (testSet.size() - estimate) / testSet.size());
+
+ System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
+ Assert.assertTrue(actualError < 0.1);
+
+ if (actualError > errorRate) {
+ errorCount1++;
+ }
+ if (actualError > 2 * errorRate) {
+ errorCount2++;
+ }
+ if (actualError > 3 * errorRate) {
+ errorCount3++;
+ }
+
+ return actualError;
+ }
+
+ private HLLCounterOld serDes(HLLCounterOld hllc) throws IOException {
+ buf.clear();
+ hllc.writeRegisters(buf);
+ buf.flip();
+ HLLCounterOld copy = new HLLCounterOld(hllc.getPrecision());
+ copy.readRegisters(buf);
+ Assert.assertEquals(copy.getCountEstimate(), hllc.getCountEstimate());
+ return copy;
+ }
+
+ @Test
+ public void testPerformance() throws IOException {
+ int N = 3; // reduce N HLLC into one
+ int M = 1000; // for M times, use 100000 for real perf test
+
+ HLLCounterOld samples[] = new HLLCounterOld[N];
+ for (int i = 0; i < N; i++) {
+ samples[i] = newHLLC();
+ for (String str : generateTestData(10000))
+ samples[i].add(str);
+ }
+
+ System.out.println("Perf test running ... ");
+ long start = System.currentTimeMillis();
+ HLLCounterOld sum = newHLLC();
+ for (int i = 0; i < M; i++) {
+ sum.clear();
+ for (int j = 0; j < N; j++) {
+ sum.merge(samples[j]);
+ checkSerialize(sum);
+ }
+ }
+ long duration = System.currentTimeMillis() - start;
+ System.out.println("Perf test result: " + duration / 1000 + " seconds");
+ }
+
+ @Test
+ public void testEquivalence() {
+ byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 };
+ byte[] b = new byte[] { 3, 4, 42 };
+ HLLCounterOld ha = new HLLCounterOld();
+ HLLCounterOld hb = new HLLCounterOld();
+ ha.add(a, 1, 3);
+ hb.add(b);
+
+ Assert.assertTrue(ha.getCountEstimate() == hb.getCountEstimate());
+ }
+
+ private HLLCounterOld newHLLC() {
+ return new HLLCounterOld(16);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
new file mode 100644
index 0000000..26ad4a7
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/HLLCounterTest.java
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.measure.hllc;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.measure.hllc.HLLCounterOld;
+import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.measure.hllc.RegisterType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Created by xiefan on 16-12-12.
+ */
+@SuppressWarnings("deprecation")
+public class HLLCounterTest {
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ Random rand1 = new Random(1);
+ Random rand2 = new Random(2);
+ Random rand3 = new Random(3);
+ int errorCount1 = 0;
+ int errorCount2 = 0;
+ int errorCount3 = 0;
+
+ @Test
+ public void testOneAdd() throws IOException {
+ HLLCounter hllc = new HLLCounter(14);
+ HLLCounter one = new HLLCounter(14);
+ for (int i = 0; i < 1000000; i++) {
+ one.clear();
+ one.add(rand1.nextInt());
+ hllc.merge(one);
+ }
+ System.out.println(hllc.getCountEstimate());
+ assertTrue(hllc.getCountEstimate() > 1000000 * 0.9);
+ }
+
+ @Test
+ public void tesSparseEstimate() throws IOException {
+ HLLCounter hllc = new HLLCounter(14);
+ for (int i = 0; i < 10; i++) {
+ hllc.add(i);
+ }
+ System.out.println(hllc.getCountEstimate());
+ assertTrue(hllc.getCountEstimate() > 10 * 0.9);
+ }
+
+ @Test
+ public void countTest() throws IOException {
+ int n = 10;
+ for (int i = 0; i < 5; i++) {
+ count(n);
+ n *= 10;
+ }
+ }
+
+ @Test
+ public void mergeTest() throws IOException {
+ double error = 0;
+ int n = 100;
+ for (int i = 0; i < n; i++) {
+ double e = merge(i);
+ error += e;
+ }
+ System.out.println("Total average error is " + error / n);
+
+ System.out.println(" errorRateCount1 is " + errorCount1 + "!");
+ System.out.println(" errorRateCount2 is " + errorCount2 + "!");
+ System.out.println(" errorRateCount3 is " + errorCount3 + "!");
+
+ Assert.assertTrue(errorCount1 <= n * 0.30);
+ Assert.assertTrue(errorCount2 <= n * 0.05);
+ Assert.assertTrue(errorCount3 <= n * 0.02);
+ }
+
+ /* compare the result of two different hll counter */
+ @Test
+ public void compareResult() throws IOException {
+ int p = 12; //4096
+ int m = 1 << p;
+
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+
+ for (int t = 0; t < 5; t++) {
+ //compare sparse
+ HLLCounterOld oldCounter = new HLLCounterOld(p);
+ HLLCounter newCounter = new HLLCounter(p);
+ HLLCounter newCounter2 = new HLLCounter(p);
+
+ for (int i = 0; i < 20; i++) {
+ int r = rand1.nextInt();
+ oldCounter.add(r);
+ newCounter.add(r);
+ }
+ assertEquals(RegisterType.SPARSE, newCounter.getRegisterType());
+ assertEquals(oldCounter.getCountEstimate(), newCounter.getCountEstimate());
+
+ buf.clear();
+ oldCounter.writeRegisters(buf);
+ buf.flip();
+ newCounter2.readRegisters(buf);
+ assertEquals(oldCounter.getCountEstimate(), newCounter2.getCountEstimate());
+
+ //compare dense
+ for (int i = 0; i < m / 2; i++) {
+ int r = rand1.nextInt();
+ oldCounter.add(r);
+ newCounter.add(r);
+ }
+ assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
+ assertEquals(oldCounter.getCountEstimate(), newCounter.getCountEstimate());
+
+ buf.clear();
+ oldCounter.writeRegisters(buf);
+ buf.flip();
+ newCounter2.readRegisters(buf);
+ assertEquals(oldCounter.getCountEstimate(), newCounter2.getCountEstimate());
+ }
+ }
+
+ @Test
+ public void testPeekLength() throws IOException {
+ HLLCounter hllc = new HLLCounter(10);
+ HLLCounter copy = new HLLCounter(10);
+ byte[] value = new byte[10];
+ for (int i = 0; i < 200000; i++) {
+ rand1.nextBytes(value);
+ hllc.add(value);
+
+ buf.clear();
+ hllc.writeRegisters(buf);
+
+ int len = buf.position();
+ buf.position(0);
+ assertEquals(len, hllc.peekLength(buf));
+
+ copy.readRegisters(buf);
+ assertEquals(len, buf.position());
+ assertEquals(hllc, copy);
+ }
+ buf.clear();
+ }
+
+ @Test
+ public void testEquivalence() {
+ byte[] a = new byte[] { 0, 3, 4, 42, 2, 2 };
+ byte[] b = new byte[] { 3, 4, 42 };
+ HLLCounter ha = new HLLCounter();
+ HLLCounter hb = new HLLCounter();
+ ha.add(a, 1, 3);
+ hb.add(b);
+
+ Assert.assertTrue(ha.getCountEstimate() == hb.getCountEstimate());
+ }
+
+ @Test
+ public void testAutoChangeToSparse() {
+ int p = 15;
+ int m = 1 << p;
+ HLLCounter counter = new HLLCounter(p);
+ assertEquals(RegisterType.SPARSE, counter.getRegisterType());
+ double over = HLLCounter.OVERFLOW_FACTOR * m;
+ int overFlow = (int) over + 1000;
+ for (int i = 0; i < overFlow; i++)
+ counter.add(i);
+ assertEquals(RegisterType.DENSE, counter.getRegisterType());
+ }
+
+ @Test
+ public void testSerialilze() throws Exception {
+ //test sparse serialize
+ int p = 15;
+ int m = 1 << p;
+ HLLCounter counter = new HLLCounter(p);
+ counter.add(123);
+ assertEquals(RegisterType.SPARSE, counter.getRegisterType());
+ checkSerialize(counter);
+ //test dense serialize
+ double over = HLLCounter.OVERFLOW_FACTOR * m;
+ int overFlow = (int) over + 1000;
+ for (int i = 0; i < overFlow; i++)
+ counter.add(i);
+ assertEquals(RegisterType.DENSE, counter.getRegisterType());
+ checkSerialize(counter);
+ }
+
+ private Set<String> generateTestData(int n) {
+ Set<String> testData = new HashSet<String>();
+ for (int i = 0; i < n; i++) {
+ String[] samples = generateSampleData();
+ for (String sample : samples) {
+ testData.add(sample);
+ }
+ }
+ return testData;
+ }
+
+ // simulate the visit (=visitor+id)
+ private String[] generateSampleData() {
+
+ StringBuilder buf = new StringBuilder();
+ for (int i = 0; i < 19; i++) {
+ buf.append(Math.abs(rand1.nextInt()) % 10);
+ }
+ String header = buf.toString();
+
+ int size = Math.abs(rand3.nextInt()) % 9 + 1;
+ String[] samples = new String[size];
+ for (int k = 0; k < size; k++) {
+ buf = new StringBuilder(header);
+ buf.append("-");
+ for (int i = 0; i < 10; i++) {
+ buf.append(Math.abs(rand3.nextInt()) % 10);
+ }
+ samples[k] = buf.toString();
+ }
+
+ return samples;
+ }
+
+ private double merge(int round) throws IOException {
+ int ln = 20;
+ int dn = 100 * (round + 1);
+ Set<String> testSet = new HashSet<String>();
+ HLLCounter[] hllcs = new HLLCounter[ln];
+ for (int i = 0; i < ln; i++) {
+ hllcs[i] = newHLLC();
+ for (int k = 0; k < dn; k++) {
+ String[] samples = generateSampleData();
+ for (String data : samples) {
+ testSet.add(data);
+ hllcs[i].add(Bytes.toBytes(data));
+ }
+ }
+ }
+ HLLCounter mergeHllc = newHLLC();
+ for (HLLCounter hllc : hllcs) {
+ mergeHllc.merge(hllc);
+ }
+
+ double errorRate = mergeHllc.getErrorRate();
+ long estimate = mergeHllc.getCountEstimate();
+ double actualError = Math.abs((double) (testSet.size() - estimate) / testSet.size());
+
+ System.out.println(testSet.size() + "-" + estimate + " ~ " + actualError);
+ Assert.assertTrue(actualError < 0.1);
+
+ if (actualError > errorRate) {
+ errorCount1++;
+ }
+ if (actualError > 2 * errorRate) {
+ errorCount2++;
+ }
+ if (actualError > 3 * errorRate) {
+ errorCount3++;
+ }
+
+ return actualError;
+ }
+
+ private HLLCounter newHLLC() {
+ return new HLLCounter(16);
+ }
+
+ private void count(int n) throws IOException {
+ Set<String> testSet = generateTestData(n);
+
+ HLLCounter hllc = newHLLC();
+ for (String testData : testSet) {
+ hllc.add(Bytes.toBytes(testData));
+ }
+ long estimate = hllc.getCountEstimate();
+ double errorRate = hllc.getErrorRate();
+ double actualError = (double) Math.abs(testSet.size() - estimate) / testSet.size();
+ System.out.println(estimate);
+ System.out.println(testSet.size());
+ System.out.println(errorRate);
+ System.out.println("=" + actualError);
+ Assert.assertTrue(actualError < errorRate * 3.0);
+
+ checkSerialize(hllc);
+ }
+
+ private void checkSerialize(HLLCounter hllc) throws IOException {
+ long estimate = hllc.getCountEstimate();
+ buf.clear();
+ hllc.writeRegisters(buf);
+ buf.flip();
+ hllc.readRegisters(buf);
+ Assert.assertEquals(estimate, hllc.getCountEstimate());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/core-metadata/src/test/java/org/apache/kylin/measure/hllc/NewHyperLogLogBenchmarkTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/hllc/NewHyperLogLogBenchmarkTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/NewHyperLogLogBenchmarkTest.java
new file mode 100644
index 0000000..586c007
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/hllc/NewHyperLogLogBenchmarkTest.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.measure.hllc;
+
+import org.apache.kylin.measure.hllc.HLLCounterOld;
+import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.measure.hllc.RegisterType;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Created by xiefan on 16-12-12.
+ */
+@SuppressWarnings("deprecation")
+public class NewHyperLogLogBenchmarkTest {
+
+ public static final Random rand = new Random(1);
+
+ final int testTimes = 10000;
+
+ @Test
+ public void denseToDenseRegisterMergeBenchmark() throws Exception {
+ final int p = 15;
+ int m = 1 << p;
+
+ System.out.println("denseToDenseRegisterMergeBenchmark(), m : " + m);
+ double oldFactor = HLLCounter.OVERFLOW_FACTOR;
+ HLLCounter.OVERFLOW_FACTOR = 1.1; //keep sparse
+ for (int cardinality : getTestDataDivide(m)) {
+ final HLLCounterOld oldCounter = new HLLCounterOld(p);
+ final HLLCounterOld oldCounter2 = getRandOldCounter(p, cardinality);
+ long oldTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+
+ for (int i = 0; i < testTimes; i++) {
+ oldCounter.merge(oldCounter2);
+ }
+ }
+ });
+ final HLLCounter newCounter = new HLLCounter(p, RegisterType.DENSE);
+ final HLLCounter newCounter2 = new HLLCounter(p, RegisterType.DENSE);
+ for (int i = 0; i < testTimes; i++)
+ newCounter2.add(i);
+ long newTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+ for (int i = 0; i < testTimes; i++) {
+ newCounter.merge(newCounter2);
+ }
+ }
+ });
+ assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
+ assertEquals(RegisterType.DENSE, newCounter2.getRegisterType());
+ System.out.println("----------------------------");
+ System.out.println("cardinality : " + cardinality);
+ System.out.println("old time : " + oldTime);
+ System.out.println("new time : " + newTime);
+ }
+ HLLCounter.OVERFLOW_FACTOR = oldFactor;
+ }
+
+ @Test
+ public void sparseToSparseMergeBenchmark() throws Exception {
+ final int p = 15;
+ int m = 1 << p;
+ System.out.println("sparseToSparseMergeBenchmark(), m : " + m);
+ double oldFactor = HLLCounter.OVERFLOW_FACTOR;
+ HLLCounter.OVERFLOW_FACTOR = 1.1; //keep sparse
+ for (int cardinality : getTestDataDivide(m)) {
+ final HLLCounterOld oldCounter = new HLLCounterOld(p);
+ final HLLCounterOld oldCounter2 = getRandOldCounter(p, cardinality);
+ long oldTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+
+ for (int i = 0; i < testTimes; i++) {
+ oldCounter.merge(oldCounter2);
+ }
+ }
+ });
+ final HLLCounter newCounter = new HLLCounter(p);
+ final HLLCounter newCounter2 = getRandNewCounter(p, cardinality);
+ long newTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+ for (int i = 0; i < testTimes; i++) {
+ newCounter.merge(newCounter2);
+ }
+ }
+ });
+ assertEquals(RegisterType.SPARSE, newCounter.getRegisterType());
+ assertEquals(RegisterType.SPARSE, newCounter2.getRegisterType());
+ System.out.println("----------------------------");
+ System.out.println("cardinality : " + cardinality);
+ System.out.println("old time : " + oldTime);
+ System.out.println("new time : " + newTime);
+ }
+ HLLCounter.OVERFLOW_FACTOR = oldFactor;
+ }
+
+ @Test
+ public void sparseToDenseRegisterMergeBenchmark() throws Exception {
+ final int p = 15;
+ int m = 1 << p;
+ System.out.println("sparseToDenseRegisterMergeBenchmark(), m : " + m);
+ double oldFactor = HLLCounter.OVERFLOW_FACTOR;
+ HLLCounter.OVERFLOW_FACTOR = 1.1; //keep sparse
+ for (int cardinality : getTestDataDivide(m)) {
+ System.out.println("----------------------------");
+ System.out.println("cardinality : " + cardinality);
+ final HLLCounterOld oldCounter = new HLLCounterOld(p);
+ final HLLCounterOld oldCounter2 = getRandOldCounter(p, cardinality);
+ long oldTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+ for (int i = 0; i < testTimes; i++) {
+ oldCounter.merge(oldCounter2);
+ }
+ }
+ });
+ final HLLCounter newCounter = new HLLCounter(p, RegisterType.DENSE);
+ final HLLCounter newCounter2 = getRandNewCounter(p, cardinality);
+ long newTime = runTestCase(new TestCase() {
+ @Override
+ public void run() {
+ for (int i = 0; i < testTimes; i++) {
+ newCounter.merge(newCounter2);
+ }
+ }
+ });
+ assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
+ assertEquals(RegisterType.SPARSE, newCounter2.getRegisterType());
+ System.out.println("old time : " + oldTime);
+ System.out.println("new time : " + newTime);
+ }
+ HLLCounter.OVERFLOW_FACTOR = oldFactor;
+ }
+
+ @Test
+ public void sparseSerializeBenchmark() throws Exception {
+ final int p = 15;
+ int m = 1 << p;
+ double oldFactor = HLLCounter.OVERFLOW_FACTOR;
+ HLLCounter.OVERFLOW_FACTOR = 1.1; //keep sparse
+ System.out.println("sparseSerializeBenchmark()");
+ for (int cardinality : getTestDataDivide(m)) {
+ System.out.println("----------------------------");
+ System.out.println("cardinality : " + cardinality);
+ final HLLCounterOld oldCounter = getRandOldCounter(p, cardinality);
+ long oldTime = runTestCase(new TestCase() {
+ @Override
+ public void run() throws Exception {
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ long totalBytes = 0;
+ for (int i = 0; i < testTimes; i++) {
+ buf.clear();
+ oldCounter.writeRegisters(buf);
+ totalBytes += buf.position();
+ buf.flip();
+ oldCounter.readRegisters(buf);
+ }
+ System.out.println("old serialize bytes : " + totalBytes / testTimes + "B");
+ }
+ });
+ final HLLCounter newCounter = getRandNewCounter(p, cardinality);
+ long newTime = runTestCase(new TestCase() {
+ @Override
+ public void run() throws Exception {
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ long totalBytes = 0;
+ for (int i = 0; i < testTimes; i++) {
+ buf.clear();
+ newCounter.writeRegisters(buf);
+ totalBytes += buf.position();
+ buf.flip();
+ newCounter.readRegisters(buf);
+ }
+ System.out.println("new serialize bytes : " + totalBytes / testTimes + "B");
+ }
+ });
+ assertEquals(RegisterType.SPARSE, newCounter.getRegisterType());
+ System.out.println("old serialize time : " + oldTime);
+ System.out.println("new serialize time : " + newTime);
+ }
+ HLLCounter.OVERFLOW_FACTOR = oldFactor;
+ }
+
+ @Test
+ public void denseSerializeBenchmark() throws Exception {
+ final int p = 15;
+ final int m = 1 << p;
+ double oldFactor = HLLCounter.OVERFLOW_FACTOR;
+ HLLCounter.OVERFLOW_FACTOR = 0; //keep sparse
+ System.out.println("denseSerializeBenchmark()");
+ for (int cardinality : getTestDataDivide(m)) {
+ System.out.println("----------------------------");
+ System.out.println("cardinality : " + cardinality);
+ final HLLCounterOld oldCounter = getRandOldCounter(p, cardinality);
+ long oldTime = runTestCase(new TestCase() {
+ @Override
+ public void run() throws Exception {
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ long totalBytes = 0;
+ for (int i = 0; i < testTimes; i++) {
+ buf.clear();
+ oldCounter.writeRegisters(buf);
+ totalBytes += buf.position();
+ buf.flip();
+ oldCounter.readRegisters(buf);
+ }
+ System.out.println("old serialize bytes : " + totalBytes / testTimes + "B");
+ }
+ });
+ final HLLCounter newCounter = getRandNewCounter(p, cardinality, RegisterType.DENSE);
+ long newTime = runTestCase(new TestCase() {
+ @Override
+ public void run() throws Exception {
+ ByteBuffer buf = ByteBuffer.allocate(1024 * 1024);
+ long totalBytes = 0;
+ for (int i = 0; i < testTimes; i++) {
+ buf.clear();
+ newCounter.writeRegisters(buf);
+ totalBytes += buf.position();
+ buf.flip();
+ newCounter.readRegisters(buf);
+ }
+ System.out.println("new serialize bytes : " + totalBytes / testTimes + "B");
+ }
+ });
+ assertEquals(RegisterType.DENSE, newCounter.getRegisterType());
+ System.out.println("old serialize time : " + oldTime);
+ System.out.println("new serialize time : " + newTime);
+ }
+ HLLCounter.OVERFLOW_FACTOR = oldFactor;
+ }
+
+ interface TestCase {
+ void run() throws Exception;
+ }
+
+ public long runTestCase(TestCase testCase) throws Exception {
+ long startTime = System.currentTimeMillis();
+ testCase.run();
+ return System.currentTimeMillis() - startTime;
+ }
+
+ public HLLCounterOld getRandOldCounter(int p, int num) {
+ HLLCounterOld c = new HLLCounterOld(p);
+ for (int i = 0; i < num; i++)
+ c.add(i);
+ return c;
+ }
+
+ public HLLCounter getRandNewCounter(int p, int num) {
+ HLLCounter c = new HLLCounter(p);
+ for (int i = 0; i < num; i++)
+ c.add(i);
+ return c;
+ }
+
+ public HLLCounter getRandNewCounter(int p, int num, RegisterType type) {
+ HLLCounter c = new HLLCounter(p, type);
+ for (int i = 0; i < num; i++)
+ c.add(i);
+ return c;
+ }
+
+ public static int[] getTestDataDivide(int m) {
+ return new int[] { 1, 5, 10, 100, m / 200, m / 100, m / 50, m / 20, m / 10, m };
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 5445491..ffba181 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -53,7 +53,7 @@ import org.apache.kylin.cube.kv.CubeDimEncMap;
import org.apache.kylin.cube.kv.RowKeyEncoder;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -76,7 +76,7 @@ public class CubeStatsReader {
final int samplingPercentage;
final int mapperNumberOfFirstBuild; // becomes meaningless after merge
final double mapperOverlapRatioOfFirstBuild; // becomes meaningless after merge
- final Map<Long, HyperLogLogPlusCounterNew> cuboidRowEstimatesHLL;
+ final Map<Long, HLLCounter> cuboidRowEstimatesHLL;
final CuboidScheduler cuboidScheduler;
public CubeStatsReader(CubeSegment cubeSegment, KylinConfig kylinConfig) throws IOException {
@@ -96,7 +96,7 @@ public class CubeStatsReader {
int percentage = 100;
int mapperNumber = 0;
double mapperOverlapRatio = 0;
- Map<Long, HyperLogLogPlusCounterNew> counterMap = Maps.newHashMap();
+ Map<Long, HLLCounter> counterMap = Maps.newHashMap();
LongWritable key = (LongWritable) ReflectionUtils.newInstance(reader.getKeyClass(), hadoopConf);
BytesWritable value = (BytesWritable) ReflectionUtils.newInstance(reader.getValueClass(), hadoopConf);
@@ -108,7 +108,7 @@ public class CubeStatsReader {
} else if (key.get() == -2) {
mapperNumber = Bytes.toInt(value.getBytes());
} else if (key.get() > 0) {
- HyperLogLogPlusCounterNew hll = new HyperLogLogPlusCounterNew(kylinConfig.getCubeStatsHLLPrecision());
+ HLLCounter hll = new HLLCounter(kylinConfig.getCubeStatsHLLPrecision());
ByteArray byteArray = new ByteArray(value.getBytes());
hll.readRegisters(byteArray.asBuffer());
counterMap.put(key.get(), hll);
@@ -161,9 +161,9 @@ public class CubeStatsReader {
return mapperOverlapRatioOfFirstBuild;
}
- public static Map<Long, Long> getCuboidRowCountMapFromSampling(Map<Long, HyperLogLogPlusCounterNew> hllcMap, int samplingPercentage) {
+ public static Map<Long, Long> getCuboidRowCountMapFromSampling(Map<Long, HLLCounter> hllcMap, int samplingPercentage) {
Map<Long, Long> cuboidRowCountMap = Maps.newHashMap();
- for (Map.Entry<Long, HyperLogLogPlusCounterNew> entry : hllcMap.entrySet()) {
+ for (Map.Entry<Long, HLLCounter> entry : hllcMap.entrySet()) {
// No need to adjust according sampling percentage. Assumption is that data set is far
// more than cardinality. Even a percentage of the data should already see all cardinalities.
cuboidRowCountMap.put(entry.getKey(), entry.getValue().getCountEstimate());
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
index 219cdf2..8f400c3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
@@ -33,17 +33,17 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
public class CubeStatsWriter {
public static void writeCuboidStatistics(Configuration conf, Path outputPath, //
- Map<Long, HyperLogLogPlusCounterNew> cuboidHLLMap, int samplingPercentage) throws IOException {
+ Map<Long, HLLCounter> cuboidHLLMap, int samplingPercentage) throws IOException {
writeCuboidStatistics(conf, outputPath, cuboidHLLMap, samplingPercentage, 0, 0);
}
public static void writeCuboidStatistics(Configuration conf, Path outputPath, //
- Map<Long, HyperLogLogPlusCounterNew> cuboidHLLMap, int samplingPercentage, int mapperNumber, double mapperOverlapRatio) throws IOException {
+ Map<Long, HLLCounter> cuboidHLLMap, int samplingPercentage, int mapperNumber, double mapperOverlapRatio) throws IOException {
Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
List<Long> allCuboids = new ArrayList<Long>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 0d388c7..3115fe4 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -47,7 +47,7 @@ import org.apache.kylin.engine.mr.KylinReducer;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.common.CubeStatsWriter;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,7 +64,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private List<TblColRef> columnList;
private String statisticsOutput = null;
private List<Long> baseCuboidRowCountInMappers;
- protected Map<Long, HyperLogLogPlusCounterNew> cuboidHLLMap = null;
+ protected Map<Long, HLLCounter> cuboidHLLMap = null;
protected long baseCuboidId;
protected CubeDesc cubeDesc;
private long totalRowsBeforeMerge = 0;
@@ -156,7 +156,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
// for hll
long cuboidId = Bytes.toLong(key.getBytes(), 1, Bytes.SIZEOF_LONG);
for (Text value : values) {
- HyperLogLogPlusCounterNew hll = new HyperLogLogPlusCounterNew(cubeConfig.getCubeStatsHLLPrecision());
+ HLLCounter hll = new HLLCounter(cubeConfig.getCubeStatsHLLPrecision());
ByteBuffer bf = ByteBuffer.wrap(value.getBytes(), 0, value.getLength());
hll.readRegisters(bf);
@@ -270,7 +270,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
if (isStatistics) {
// output the hll info
long grandTotal = 0;
- for (HyperLogLogPlusCounterNew hll : cuboidHLLMap.values()) {
+ for (HLLCounter hll : cuboidHLLMap.values()) {
grandTotal += hll.getCountEstimate();
}
double mapperOverlapRatio = grandTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grandTotal;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
index c0575f1..5692c76 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
@@ -29,7 +29,7 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.model.TblColRef;
import com.google.common.collect.Lists;
@@ -45,7 +45,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
protected CuboidScheduler cuboidScheduler = null;
protected int nRowKey;
private Integer[][] allCuboidsBitSet = null;
- private HyperLogLogPlusCounterNew[] allCuboidsHLL = null;
+ private HLLCounter[] allCuboidsHLL = null;
private Long[] cuboidIds;
private HashFunction hf = null;
private int rowCount = 0;
@@ -76,9 +76,9 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
allCuboidsBitSet = allCuboidsBitSetList.toArray(new Integer[cuboidIdList.size()][]);
cuboidIds = cuboidIdList.toArray(new Long[cuboidIdList.size()]);
- allCuboidsHLL = new HyperLogLogPlusCounterNew[cuboidIds.length];
+ allCuboidsHLL = new HLLCounter[cuboidIds.length];
for (int i = 0; i < cuboidIds.length; i++) {
- allCuboidsHLL[i] = new HyperLogLogPlusCounterNew(cubeDesc.getConfig().getCubeStatsHLLPrecision());
+ allCuboidsHLL[i] = new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision());
}
hf = Hashing.murmur3_32();
@@ -207,7 +207,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
if (collectStatistics) {
ByteBuffer hllBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);
// output each cuboid's hll to reducer, key is 0 - cuboidId
- HyperLogLogPlusCounterNew hll;
+ HLLCounter hll;
for (int i = 0; i < cuboidIds.length; i++) {
hll = allCuboidsHLL[i];
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index e839989..811fc24 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -47,7 +47,7 @@ import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,7 +56,7 @@ import com.google.common.collect.Maps;
public class MergeStatisticsStep extends AbstractExecutable {
private static final Logger logger = LoggerFactory.getLogger(MergeStatisticsStep.class);
- protected Map<Long, HyperLogLogPlusCounterNew> cuboidHLLMap = Maps.newHashMap();
+ protected Map<Long, HLLCounter> cuboidHLLMap = Maps.newHashMap();
public MergeStatisticsStep() {
super();
@@ -100,7 +100,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
// sampling percentage;
averageSamplingPercentage += Bytes.toInt(value.getBytes());
} else if (key.get() > 0) {
- HyperLogLogPlusCounterNew hll = new HyperLogLogPlusCounterNew(kylinConf.getCubeStatsHLLPrecision());
+ HLLCounter hll = new HLLCounter(kylinConf.getCubeStatsHLLPrecision());
ByteArray byteArray = new ByteArray(value.getBytes());
hll.readRegisters(byteArray.asBuffer());
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
index cae3b62..beec00f 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeSamplingTest.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.junit.Before;
import org.junit.Test;
@@ -45,7 +45,7 @@ public class CubeSamplingTest {
private Integer[][] allCuboidsBitSet;
private HashFunction hf = null;
private long baseCuboidId;
- private HyperLogLogPlusCounterNew[] allCuboidsHLL = null;
+ private HLLCounter[] allCuboidsHLL = null;
private final byte[] seperator = Bytes.toBytes(",");
@Before
@@ -61,9 +61,9 @@ public class CubeSamplingTest {
allCuboidsBitSet = allCuboidsBitSetList.toArray(new Integer[allCuboidsBitSetList.size()][]);
System.out.println("Totally have " + allCuboidsBitSet.length + " cuboids.");
- allCuboidsHLL = new HyperLogLogPlusCounterNew[allCuboids.size()];
+ allCuboidsHLL = new HLLCounter[allCuboids.size()];
for (int i = 0; i < allCuboids.size(); i++) {
- allCuboidsHLL[i] = new HyperLogLogPlusCounterNew(14);
+ allCuboidsHLL[i] = new HLLCounter(14);
}
// hf = Hashing.goodFastHash(32);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6e330a8/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
index a00db94..f6f790e 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.CubeStatsWriter;
-import org.apache.kylin.measure.hllc.HyperLogLogPlusCounterNew;
+import org.apache.kylin.measure.hllc.HLLCounter;
import org.junit.Test;
import com.google.common.collect.Maps;
@@ -48,7 +48,7 @@ public class FactDistinctColumnsReducerTest {
}
System.out.println(outputPath);
- Map<Long, HyperLogLogPlusCounterNew> cuboidHLLMap = Maps.newHashMap();
+ Map<Long, HLLCounter> cuboidHLLMap = Maps.newHashMap();
CubeStatsWriter.writeCuboidStatistics(conf, outputPath, cuboidHLLMap, 100);
FileSystem.getLocal(conf).delete(outputPath, true);
[16/50] [abbrv] kylin git commit: KYLIN-2277 return correct columns
for select *
Posted by li...@apache.org.
KYLIN-2277 return correct columns for select *
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/654144f4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/654144f4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/654144f4
Branch: refs/heads/master-cdh5.7
Commit: 654144f47ff647bc4b7eac9793d2fa5f88dec372
Parents: e6e330a
Author: Yang Li <li...@apache.org>
Authored: Tue Dec 13 23:45:55 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Dec 14 21:18:21 2016 +0800
----------------------------------------------------------------------
.../calcite/sql2rel/SqlToRelConverter.java | 62 +++++++++++++++++++-
.../kylin/metadata/model/FunctionDesc.java | 4 +-
.../apache/kylin/query/ITKylinQueryTest.java | 6 ++
.../org/apache/kylin/query/KylinTestBase.java | 21 +++++++
.../resources/query/sql_timeout/query01.sql | 2 +-
5 files changed, 91 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/654144f4/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
index d223cdf..cf36f61 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java
@@ -67,6 +67,7 @@ import org.apache.calcite.rel.stream.Delta;
import org.apache.calcite.rel.stream.LogicalDelta;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
@@ -194,6 +195,7 @@ import static org.apache.calcite.util.Static.RESOURCE;
* - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE`
* - isTrimUnusedFields(), override to false
* - AggConverter.translateAgg(...), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104
+ * - convertQuery(), call hackSelectStar() at the end
*/
/**
@@ -529,6 +531,8 @@ public class SqlToRelConverter {
* the query will be part of a view.
*/
public RelRoot convertQuery(SqlNode query, final boolean needsValidation, final boolean top) {
+ SqlNode origQuery = query; /* OVERRIDE POINT */
+
if (needsValidation) {
query = validator.validate(query);
}
@@ -553,7 +557,63 @@ public class SqlToRelConverter {
}
final RelDataType validatedRowType = validator.getValidatedNodeType(query);
- return RelRoot.of(result, validatedRowType, query.getKind()).withCollation(collation);
+ return hackSelectStar(origQuery, RelRoot.of(result, validatedRowType, query.getKind()).withCollation(collation));
+ }
+
+ /* OVERRIDE POINT */
+ private RelRoot hackSelectStar(SqlNode query, RelRoot root) {
+ /*
+ * Rel tree is like:
+ *
+ * LogicalSort (optional)
+ * |- LogicalProject
+ * |- OLAPTableScan
+ */
+ LogicalProject rootPrj = null;
+ LogicalSort rootSort = null;
+ if (root.rel instanceof LogicalProject) {
+ rootPrj = (LogicalProject) root.rel;
+ } else if (root.rel instanceof LogicalSort && root.rel.getInput(0) instanceof LogicalProject) {
+ rootPrj = (LogicalProject) root.rel.getInput(0);
+ rootSort = (LogicalSort) root.rel;
+ } else {
+ return root;
+ }
+
+ if (!rootPrj.getInput().getClass().getSimpleName().equals("OLAPTableScan"))
+ return root;
+
+ RelNode scan = rootPrj.getInput();
+ if (rootPrj.getRowType().getFieldCount() < scan.getRowType().getFieldCount())
+ return root;
+
+ RelDataType inType = rootPrj.getRowType();
+ List<String> inFields = inType.getFieldNames();
+ List<RexNode> projExp = new ArrayList<>();
+ List<Pair<Integer, String>> projFields = new ArrayList<>();
+ FieldInfoBuilder projTypeBuilder = getCluster().getTypeFactory().builder();
+ FieldInfoBuilder validTypeBuilder = getCluster().getTypeFactory().builder();
+ for (int i = 0; i < inFields.size(); i++) {
+ if (!inFields.get(i).startsWith("_KY_")) {
+ projExp.add(rootPrj.getProjects().get(i));
+ projFields.add(Pair.of(projFields.size(), inFields.get(i)));
+ projTypeBuilder.add(inType.getFieldList().get(i));
+ validTypeBuilder.add(root.validatedRowType.getFieldList().get(i));
+ }
+ }
+
+ RelDataType projRowType = getCluster().getTypeFactory().createStructType(projTypeBuilder);
+ rootPrj = LogicalProject.create(scan, projExp, projRowType);
+ if (rootSort != null) {
+ rootSort = (LogicalSort) rootSort.copy(rootSort.getTraitSet(), rootPrj, rootSort.collation, rootSort.offset, rootSort.fetch);
+ }
+
+ RelDataType validRowType = getCluster().getTypeFactory().createStructType(validTypeBuilder);
+ root = new RelRoot(rootSort == null ? rootPrj : rootSort, validRowType, root.kind, projFields, root.collation);
+
+ validator.setValidatedNodeType(query, validRowType);
+
+ return root;
}
private static boolean isStream(SqlNode query) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/654144f4/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index ac13f40..4d89e1a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -138,9 +138,9 @@ public class FunctionDesc {
if (isSum()) {
return getParameter().getValue();
} else if (isCount()) {
- return "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same
+ return "_KY_" + "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same
} else {
- return getFullExpression().replaceAll("[(),. ]", "_");
+ return "_KY_" + getFullExpression().replaceAll("[(),. ]", "_");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/654144f4/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index d485955..7f04bbb 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -379,4 +379,10 @@ public class ITKylinQueryTest extends KylinTestBase {
// compare the result
Assert.assertEquals(expectVersion, queriedVersion);
}
+
+ @Test
+ public void testSelectStarColumnCount() throws Exception {
+ execAndCompColumnCount("select * from test_kylin_fact limit 10", 9);
+ execAndCompColumnCount("select * from test_kylin_fact", 9);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/654144f4/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index bcf55e5..ddb996c 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -67,6 +67,7 @@ import org.dbunit.ext.h2.H2Connection;
import org.dbunit.ext.h2.H2DataTypeFactory;
import org.junit.Assert;
+import com.google.common.collect.ImmutableSet;
import com.google.common.io.Files;
/**
@@ -422,6 +423,26 @@ public class KylinTestBase {
}
}
+ protected void execAndCompColumnCount(String input, int expectedColumnCount) throws Exception {
+ printInfo("---------- test column count: " + input);
+ Set<String> sqlSet = ImmutableSet.of(input);
+
+ for (String sql : sqlSet) {
+ // execute Kylin
+ printInfo("Query Result from Kylin - " + sql);
+ IDatabaseConnection kylinConn = new DatabaseConnection(cubeConnection);
+ ITable kylinTable = executeQuery(kylinConn, sql, sql, false);
+
+ try {
+ // compare the result
+ Assert.assertEquals(expectedColumnCount, kylinTable.getTableMetaData().getColumns().length);
+ } catch (Throwable t) {
+ printInfo("execAndCompColumnCount failed on: " + sql);
+ throw t;
+ }
+ }
+ }
+
protected void execLimitAndValidate(String queryFolder) throws Exception {
printInfo("---------- test folder: " + new File(queryFolder).getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/kylin/blob/654144f4/kylin-it/src/test/resources/query/sql_timeout/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_timeout/query01.sql b/kylin-it/src/test/resources/query/sql_timeout/query01.sql
index 3b9a837..eaff396 100644
--- a/kylin-it/src/test/resources/query/sql_timeout/query01.sql
+++ b/kylin-it/src/test/resources/query/sql_timeout/query01.sql
@@ -16,4 +16,4 @@
-- limitations under the License.
--
-select * from test_kylin_fact limit 1200
+select seller_id,lstg_format_name from test_kylin_fact limit 1200
[19/50] [abbrv] kylin git commit: KYLIN-2275,
Remove dimensions cause wrong remove in advance settings
Posted by li...@apache.org.
KYLIN-2275,Remove dimensions cause wrong remove in advance settings
Signed-off-by: zhongjian <ji...@163.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/076fbcfb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/076fbcfb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/076fbcfb
Branch: refs/heads/master-cdh5.7
Commit: 076fbcfb616cee4b4696cae20178ad6dc8365b3c
Parents: 36a42d8
Author: luguosheng <55...@qq.com>
Authored: Tue Dec 13 14:41:31 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 15 18:57:36 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/076fbcfb/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index 85bd4b1..b901e48 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -662,14 +662,18 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
var mandatory = group.select_rule.mandatory_dims;
if(mandatory && mandatory.length){
var columnIndex = mandatory.indexOf(deprecatedItem);
- group.select_rule.mandatory_dims.splice(columnIndex,1);
+ if(columnIndex>=0){
+ group.select_rule.mandatory_dims.splice(columnIndex,1);
+ }
}
var hierarchys = group.select_rule.hierarchy_dims;
if(hierarchys && hierarchys.length){
for(var i=0;i<hierarchys.length;i++){
var hierarchysIndex = hierarchys[i].indexOf(deprecatedItem);
- group.select_rule.hierarchy_dims[i].splice(hierarchysIndex,1);
+ if(hierarchysIndex>=0) {
+ group.select_rule.hierarchy_dims[i].splice(hierarchysIndex, 1);
+ }
}
}
@@ -678,7 +682,9 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
if(joints && joints.length){
for(var i=0;i<joints.length;i++){
var jointIndex = joints[i].indexOf(deprecatedItem);
- group.select_rule.joint_dims[i].splice(jointIndex,1);
+ if(jointIndex>=0) {
+ group.select_rule.joint_dims[i].splice(jointIndex, 1);
+ }
}
}
[02/50] [abbrv] kylin git commit: KYLIN-2266 Reduce memory usage for
building global dict
Posted by li...@apache.org.
KYLIN-2266 Reduce memory usage for building global dict
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/65c58990
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/65c58990
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/65c58990
Branch: refs/heads/master-cdh5.7
Commit: 65c58990802c644ea35005c301ba2951a9b7e6a5
Parents: 5d6bfba
Author: kangkaisen <ka...@live.com>
Authored: Sun Dec 11 20:49:28 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 12 10:55:32 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 9 ---------
.../java/org/apache/kylin/dict/AppendTrieDictionary.java | 6 ++----
.../org/apache/kylin/dict/AppendTrieDictionaryTest.java | 1 -
3 files changed, 2 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c58990/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f46c185..d4272f9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -245,15 +245,6 @@ abstract public class KylinConfigBase implements Serializable {
setProperty("kylin.dictionary.append-entry-size", String.valueOf(entrySize));
}
- public int getAppendDictCacheSize() {
- return Integer.parseInt(getOptional("kylin.dictionary.append-cache-size", "20"));
- }
-
- // for test
- public void setAppendDictCacheSize(int cacheSize) {
- setProperty("kylin.dictionary.append-cache-size", String.valueOf(cacheSize));
- }
-
public int getCachedSnapshotMaxEntrySize() {
return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", "500"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c58990/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 35aa9a5..32bfde6 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -115,10 +115,9 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
}
public void initDictSliceMap(CachedTreeMap dictMap) throws IOException {
- int cacheSize = KylinConfig.getInstanceFromEnv().getAppendDictCacheSize();
int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions();
long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL();
- CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir)
+ CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir)
.immutable(true).maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build();
newDictSliceMap.loadEntry(dictMap);
this.dictSliceMap = newDictSliceMap;
@@ -930,11 +929,10 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
this.bytesConverter = bytesConverter;
MAX_ENTRY_IN_SLICE = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
- int cacheSize = KylinConfig.getInstanceFromEnv().getAppendDictCacheSize();
int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions();
long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL();
// create a new cached map with baseDir
- mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir)
+ mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir)
.maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictNode.class).immutable(false).build();
if (dictMapBytes != null) {
((Writable) mutableDictSliceMap).readFields(new DataInputStream(new ByteArrayInputStream(dictMapBytes)));
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c58990/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index a7e8152..0776599 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -63,7 +63,6 @@ public class AppendTrieDictionaryTest {
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/localmeta");
KylinConfig config = KylinConfig.getInstanceFromEnv();
config.setAppendDictEntrySize(50000);
- config.setAppendDictCacheSize(3);
config.setProperty("kylin.env.hdfs-working-dir", BASE_DIR);
}