You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/05/27 03:04:09 UTC
[kylin] branch pr140 created (now cbdb348)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch pr140
in repository https://gitbox.apache.org/repos/asf/kylin.git.
at cbdb348 Minor, make query response with "Unknown error" if exception without message occurs
This branch includes the following new commits:
new e4ae5e5 KYLIN-3315 fix table init
new 8563820 KYLIN-3348 fix 'missing LastBuildJobID' error
new 1d62824 KYLIN-3352 better filter transform for better seg pruning
new 4917466 KYLIN-3354 escape double-quoted defaultCatalog keyword
new a3d0517 KYLIN-3315 fix table init to avoid deadlock on project manager
new 56bc503 Minor, fix wrong method name of cube's input record size
new 2baa8e7 Minor, add member "displayCubeName" for job instance
new cbdb348 Minor, make query response with "Unknown error" if exception without message occurs
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 02/08: KYLIN-3348 fix 'missing LastBuildJobID' error
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch pr140
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 85638207f49b117f6cc019720303f5f1786f1754
Author: Li Yang <li...@apache.org>
AuthorDate: Wed Apr 25 15:28:47 2018 +0800
KYLIN-3348 fix 'missing LastBuildJobID' error
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../java/org/apache/kylin/cube/CubeInstance.java | 23 ++++++++++++++++++++++
.../java/org/apache/kylin/cube/CubeManager.java | 10 ++++++----
.../org/apache/kylin/cube/CubeManagerTest.java | 3 +++
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 2 +-
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 2 +-
examples/test_case_data/localmeta/kylin.properties | 2 +-
6 files changed, 35 insertions(+), 7 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 035cf7b..b4f712b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -237,6 +237,29 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return getCanonicalName();
}
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (getClass() != obj.getClass())
+ return false;
+ CubeInstance other = (CubeInstance) obj;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ return true;
+ }
+
// ============================================================================
@Override
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index fc2ad3d..84fc920 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -803,10 +803,12 @@ public class CubeManager implements IRealizationProvider {
return segment;
}
- public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment newSegment) throws IOException {
- // work on copy instead of cached objects
- CubeInstance cubeCopy = cube.latestCopyForWrite(); // get a latest copy
- CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());
+ public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment newSegCopy) throws IOException {
+ // double check the updating objects are not on cache
+ if (newSegCopy.getCubeInstance().isCachedAndShared())
+ throw new IllegalStateException();
+
+ CubeInstance cubeCopy = getCube(cube.getName()).latestCopyForWrite();
if (StringUtils.isBlank(newSegCopy.getStorageLocationIdentifier()))
throw new IllegalStateException(
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index 8953868..cad5094 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -211,6 +211,9 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
CubeSegment seg4 = mgr.appendSegment(cube, null, new SegmentRange(3000L, 4000L), m3, m4);
mgr.updateCubeSegStatus(seg4, SegmentStatusEnum.READY);
+ cube = mgr.getCube(cube.getName());
+ assertTrue(cube.getSegments().size() == 4);
+
CubeSegment merge1 = mgr.mergeSegments(cube, null, new SegmentRange(0L, 2000L), true);
merge1.setStatus(SegmentStatusEnum.NEW);
merge1.setLastBuildJobID("test");
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index d085a77..7262383 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -56,7 +56,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
- final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite();
final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
index 8b478aa..e7aec8c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
@@ -45,7 +45,7 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
- final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite();
CubeSegment mergedSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
if (mergedSegment == null) {
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 805432b..8ae8081 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -26,7 +26,7 @@ kylin.env=DEV
kylin.storage.hbase.owner-tag=whoami@kylin.apache.org
# List of web servers in use, this enables one web server instance to sync up with other servers.
-#kylin.server.cluster-servers=localhost:7070
+kylin.server.cluster-servers=
### SOURCE ###
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 05/08: KYLIN-3315 fix table init to avoid deadlock on
project manager
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch pr140
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a3d0517e02e4928e1feb3166090a8e15bb0addb5
Author: lidongsjtu <li...@apache.org>
AuthorDate: Tue May 1 00:19:28 2018 +0800
KYLIN-3315 fix table init to avoid deadlock on project manager
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../org/apache/kylin/metadata/model/TableDesc.java | 23 +++++++++++-----------
1 file changed, 12 insertions(+), 11 deletions(-)
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 123fedb..f69b05d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -53,7 +53,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
public static String concatRawResourcePath(String nameOnPath) {
return ResourceStore.TABLE_RESOURCE_ROOT + "/" + nameOnPath + ".json";
}
-
+
public static String makeResourceName(String tableIdentity, String prj) {
return prj == null ? tableIdentity : tableIdentity + "--" + prj;
}
@@ -134,7 +134,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
public String resourceName() {
return makeResourceName(getIdentity(), getProject());
}
-
+
public TableDesc appendColumns(ColumnDesc[] computedColumns, boolean makeCopy) {
if (computedColumns == null || computedColumns.length == 0) {
return this;
@@ -195,7 +195,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
if (isBorrowedFromGlobal()) {
return concatResourcePath(getIdentity(), null);
}
-
+
return concatResourcePath(getIdentity(), project);
}
@@ -294,13 +294,8 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
public void init(KylinConfig config, String project) {
this.project = project;
- if (project == null) {
- this.config = config;
- } else {
- ProjectInstance projInstance = ProjectManager.getInstance(config).getProject(project);
- this.config = projInstance == null ? config : projInstance.getConfig();
- }
-
+ this.config = config;
+
if (name != null)
name = name.toUpperCase();
@@ -383,9 +378,15 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
return sourceType;
}
+ // cannot set config in init() due to cascade lock() on projectManager
@Override
public KylinConfig getConfig() {
- return config;
+ if (project == null) {
+ return config;
+ } else {
+ ProjectInstance projInstance = ProjectManager.getInstance(config).getProject(project);
+ return projInstance == null ? config : projInstance.getConfig();
+ }
}
public void setSourceType(int sourceType) {
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 07/08: Minor, add member "displayCubeName" for job instance
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch pr140
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2baa8e705454bb8268bb7f8a27cdf1d1f3c64a12
Author: nichunen <ch...@kyligence.io>
AuthorDate: Mon May 14 13:17:53 2018 +0800
Minor, add member "displayCubeName" for job instance
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../src/main/java/org/apache/kylin/job/JobInstance.java | 15 +++++++++++++++
.../apache/kylin/engine/mr/common/JobInfoConverter.java | 4 +++-
.../java/org/apache/kylin/rest/service/JobService.java | 8 ++++++--
.../java/org/apache/kylin/tool/JobInstanceExtractor.java | 7 +++++--
4 files changed, 29 insertions(+), 5 deletions(-)
diff --git a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
index e7c0aa0..a2c3459 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JobInstance.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.job.constant.JobStatusEnum;
@@ -52,6 +53,8 @@ public class JobInstance extends RootPersistentEntity implements Comparable<JobI
private long duration;
@JsonProperty("related_cube")
private String relatedCube;
+ @JsonProperty("display_cube_name")
+ private String displayCubeName;
@JsonProperty("related_segment")
private String relatedSegment;
@JsonProperty("exec_start_time")
@@ -176,6 +179,18 @@ public class JobInstance extends RootPersistentEntity implements Comparable<JobI
this.relatedCube = relatedCube;
}
+ public String getDisplayCubeName() {
+ if (StringUtils.isBlank(displayCubeName)) {
+ return relatedCube;
+ } else {
+ return displayCubeName;
+ }
+ }
+
+ public void setDisplayCubeName(String displayCubeName) {
+ this.displayCubeName = displayCubeName;
+ }
+
public String getRelatedSegment() {
return relatedSegment;
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
index ad8b954..862d7e5 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
@@ -76,7 +76,8 @@ public class JobInfoConverter {
final JobInstance result = new JobInstance();
result.setName(job.getName());
- result.setRelatedCube(cube != null ? cube.getDisplayName() : CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+ result.setRelatedCube(cube != null ? cube.getName() : CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+ result.setDisplayCubeName(cube != null ? cube.getDisplayName() : CubingExecutableUtil.getCubeName(cubeJob.getParams()));
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setLastModified(output.getLastModified());
result.setSubmitter(job.getSubmitter());
@@ -111,6 +112,7 @@ public class JobInfoConverter {
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
+ result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
result.setLastModified(output.getLastModified());
result.setSubmitter(job.getSubmitter());
result.setUuid(job.getId());
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 4317ed5..e120d43 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -461,9 +461,12 @@ public class JobService extends BasicService implements InitializingBean {
final JobInstance result = new JobInstance();
result.setName(job.getName());
if (cube != null) {
- result.setRelatedCube(cube.getDisplayName());
+ result.setRelatedCube(cube.getName());
+ result.setDisplayCubeName(cube.getDisplayName());
} else {
- result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+ String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
+ result.setRelatedCube(cubeName);
+ result.setDisplayCubeName(cubeName);
}
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setLastModified(cubeJob.getLastModified());
@@ -494,6 +497,7 @@ public class JobService extends BasicService implements InitializingBean {
final JobInstance result = new JobInstance();
result.setName(job.getName());
result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
+ result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
result.setLastModified(job.getLastModified());
result.setSubmitter(job.getSubmitter());
result.setUuid(job.getId());
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java b/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
index 6153b7f..bc61f94 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobInstanceExtractor.java
@@ -131,9 +131,12 @@ public class JobInstanceExtractor extends AbstractInfoExtractor {
final JobInstance result = new JobInstance();
result.setName(cubeJob.getName());
if (cube != null) {
- result.setRelatedCube(cube.getDisplayName());
+ result.setRelatedCube(cube.getName());
+ result.setDisplayCubeName(cube.getDisplayName());
} else {
- result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
+ String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
+ result.setRelatedCube(cubeName);
+ result.setDisplayCubeName(cubeName);
}
result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
result.setLastModified(output.getLastModified());
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 04/08: KYLIN-3354 escape double-quoted defaultCatalog
keyword
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch pr140
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4917466bf102f6234d2738ad749ca39d5757c89e
Author: lidongsjtu <li...@apache.org>
AuthorDate: Sat Apr 28 15:30:25 2018 +0800
KYLIN-3354 escape double-quoted defaultCatalog keyword
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../org/apache/kylin/query/util/KeywordDefaultDirtyHack.java | 1 +
.../apache/kylin/query/util/KeywordDefaultDirtyHackTest.java | 10 ++++++++++
2 files changed, 11 insertions(+)
diff --git a/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java b/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
index 939f5bc..08b982c 100644
--- a/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
+++ b/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
@@ -30,6 +30,7 @@ public class KeywordDefaultDirtyHack implements QueryUtil.IQueryTransformer {
// KYLIN-2108, DEFAULT is hive default database, but a sql keyword too, needs quote
sql = sql.replaceAll("(?i)default\\.", "\"DEFAULT\".");
sql = sql.replace("defaultCatalog.", "");
+ sql = sql.replace("\"defaultCatalog\".", "");
return sql;
}
diff --git a/query/src/test/java/org/apache/kylin/query/util/KeywordDefaultDirtyHackTest.java b/query/src/test/java/org/apache/kylin/query/util/KeywordDefaultDirtyHackTest.java
index afd775d..ab4e6db 100644
--- a/query/src/test/java/org/apache/kylin/query/util/KeywordDefaultDirtyHackTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/KeywordDefaultDirtyHackTest.java
@@ -56,5 +56,15 @@ public class KeywordDefaultDirtyHackTest extends LocalFileMetadataTestCase {
String s = kwDefaultHack.transform(sql, null, "DEFAULT");
Assert.assertEquals("select count(*) from \"DEFAULT\".test_kylin_fact", s);
}
+ {
+ String sql = "select count(*) from defaultCatalog.default.test_kylin_fact";
+ String s = kwDefaultHack.transform(sql, null, "DEFAULT");
+ Assert.assertEquals("select count(*) from \"DEFAULT\".test_kylin_fact", s);
+ }
+ {
+ String sql = "select count(*) from \"defaultCatalog\".default.test_kylin_fact";
+ String s = kwDefaultHack.transform(sql, null, "DEFAULT");
+ Assert.assertEquals("select count(*) from \"DEFAULT\".test_kylin_fact", s);
+ }
}
}
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 08/08: Minor,
make query response with "Unknown error" if exception without
message occurs
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch pr140
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit cbdb34855e3298d8b77931e13bc5bde51eb56e67
Author: nichunen <ch...@kyligence.io>
AuthorDate: Tue May 15 21:08:16 2018 +0800
Minor, make query response with "Unknown error" if exception without message occurs
Signed-off-by: shaofengshi <sh...@apache.org>
---
query/src/main/java/org/apache/kylin/query/util/QueryUtil.java | 3 +++
query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java | 6 ++++++
2 files changed, 9 insertions(+)
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 0552895..424a172 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -123,6 +123,9 @@ public class QueryUtil {
}
public static String makeErrorMsgUserFriendly(String errorMsg) {
+ if (errorMsg == null) {
+ return "Unknown error.";
+ }
try {
// make one line
errorMsg = errorMsg.replaceAll("\\s", " ");
diff --git a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index 4d766b9..30f4934 100644
--- a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -225,4 +225,10 @@ public class QueryUtilTest extends LocalFileMetadataTestCase {
Assert.assertEquals(originSql, QueryUtil.removeCommentInSql(sqlWithComment));
}
}
+
+ @Test
+ public void testUnknownErrorResponseMessage() {
+ String msg = QueryUtil.makeErrorMsgUserFriendly(new NullPointerException());
+ Assert.assertEquals("Unknown error.", msg);
+ }
}
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 01/08: KYLIN-3315 fix table init
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch pr140
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit e4ae5e50c534eff1eee427decb66c139330c2be2
Author: lidongsjtu <li...@apache.org>
AuthorDate: Sun Apr 22 23:30:25 2018 +0800
KYLIN-3315 fix table init
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../java/org/apache/kylin/metadata/model/TableDesc.java | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index a9e9877..123fedb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -28,6 +28,8 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringSplitter;
import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +53,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
public static String concatRawResourcePath(String nameOnPath) {
return ResourceStore.TABLE_RESOURCE_ROOT + "/" + nameOnPath + ".json";
}
-
+
public static String makeResourceName(String tableIdentity, String prj) {
return prj == null ? tableIdentity : tableIdentity + "--" + prj;
}
@@ -132,7 +134,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
public String resourceName() {
return makeResourceName(getIdentity(), getProject());
}
-
+
public TableDesc appendColumns(ColumnDesc[] computedColumns, boolean makeCopy) {
if (computedColumns == null || computedColumns.length == 0) {
return this;
@@ -193,7 +195,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
if (isBorrowedFromGlobal()) {
return concatResourcePath(getIdentity(), null);
}
-
+
return concatResourcePath(getIdentity(), project);
}
@@ -292,8 +294,13 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
public void init(KylinConfig config, String project) {
this.project = project;
- this.config = config;
-
+ if (project == null) {
+ this.config = config;
+ } else {
+ ProjectInstance projInstance = ProjectManager.getInstance(config).getProject(project);
+ this.config = projInstance == null ? config : projInstance.getConfig();
+ }
+
if (name != null)
name = name.toUpperCase();
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 06/08: Minor,
fix wrong method name of cube's input record size
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch pr140
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 56bc50364c6b3079fc3bbcc576390d789f7381ab
Author: nichunen <ch...@kyligence.io>
AuthorDate: Fri May 4 15:18:02 2018 +0800
Minor, fix wrong method name of cube's input record size
Signed-off-by: shaofengshi <sh...@apache.org>
---
core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java | 2 +-
.../java/org/apache/kylin/rest/response/CubeInstanceResponse.java | 8 ++++----
.../main/java/org/apache/kylin/rest/service/DashboardService.java | 6 +++---
3 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index b4f712b..bf966d4 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -642,7 +642,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
// For JSON serialization of this attribute, use CubeInstanceResponse
- public long getInputRecordSizeMB() {
+ public long getInputRecordSizeBytes() {
long sizeRecordSize = 0L;
for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java
index 63c305f..19a5ef5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java
@@ -45,7 +45,7 @@ public class CubeInstanceResponse extends CubeInstance {
@JsonProperty("input_records_count")
private long inputRecordCnt;
@JsonProperty("input_records_size")
- private long inputRecordSizeMB;
+ private long inputRecordSizeBytes;
public CubeInstanceResponse(CubeInstance cube, String project) {
@@ -79,7 +79,7 @@ public class CubeInstanceResponse extends CubeInstance {
initSizeKB();
initInputRecordCount();
- initInputRecordSizeMB();
+ initInputRecordSizeBytes();
}
protected void setModel(String model) {
@@ -94,8 +94,8 @@ public class CubeInstanceResponse extends CubeInstance {
this.inputRecordCnt = super.getInputRecordCount();
}
- protected void initInputRecordSizeMB() {
- this.inputRecordSizeMB = super.getInputRecordSizeMB();
+ protected void initInputRecordSizeBytes() {
+ this.inputRecordSizeBytes = super.getInputRecordSizeBytes();
}
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
index 63bc4fe..e548693 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
@@ -139,10 +139,10 @@ public class DashboardService extends BasicService {
Float maxCubeExpansion = Float.NEGATIVE_INFINITY;
cubeMetrics.increase("totalCube", totalCube.floatValue());
for (CubeInstance cubeInstance : cubeInstances) {
- if (cubeInstance.getInputRecordSizeMB() > 0) {
+ if (cubeInstance.getInputRecordSizeBytes() > 0) {
totalCubeSize += cubeInstance.getSizeKB();
- totalRecoadSize += cubeInstance.getInputRecordSizeMB();
- Float cubeExpansion = new Float(cubeInstance.getSizeKB()) * 1024 / cubeInstance.getInputRecordSizeMB();
+ totalRecoadSize += cubeInstance.getInputRecordSizeBytes();
+ Float cubeExpansion = new Float(cubeInstance.getSizeKB()) * 1024 / cubeInstance.getInputRecordSizeBytes();
if (cubeExpansion > maxCubeExpansion) {
maxCubeExpansion = cubeExpansion;
}
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 03/08: KYLIN-3352 better filter transform for better seg
pruning
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch pr140
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1d62824fd75d3ed6e17c3ce6df9802c6e39ca9b4
Author: Li Yang <li...@apache.org>
AuthorDate: Fri Apr 27 14:39:21 2018 +0800
KYLIN-3352 better filter transform for better seg pruning
Signed-off-by: shaofengshi <sh...@apache.org>
---
.../java/org/apache/kylin/gridtable/GTUtil.java | 28 +++--
.../kylin/metadata/filter/CompareTupleFilter.java | 1 -
.../kylin/storage/gtrecord/DictGridTableTest.java | 127 +++++++++++++++++----
3 files changed, 126 insertions(+), 30 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index 5c9dfe3..65704d5 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -33,6 +33,7 @@ import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -154,8 +155,7 @@ public class GTUtil {
// In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
// Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter)
// will always return FALSE.
- if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT
- && !TupleFilter.isEvaluableRecursively(filter)) {
+ if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
return ConstantTupleFilter.TRUE;
}
@@ -181,7 +181,6 @@ public class GTUtil {
return filter;
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
protected TupleFilter encodeConstants(CompareTupleFilter oldCompareFilter) {
// extract ColumnFilter & ConstantFilter
TblColRef externalCol = oldCompareFilter.getColumn();
@@ -249,9 +248,13 @@ public class GTUtil {
}
break;
case LT:
- code = translate(col, firstValue, 1);
+ code = translate(col, firstValue, 0);
if (code == null) {
- result = ConstantTupleFilter.TRUE;
+ code = translate(col, firstValue, -1);
+ if (code == null)
+ result = ConstantTupleFilter.FALSE;
+ else
+ result = newCompareFilter(FilterOperatorEnum.LTE, externalCol, code);
} else {
newCompareFilter.addChild(new ConstantTupleFilter(code));
result = newCompareFilter;
@@ -267,9 +270,13 @@ public class GTUtil {
}
break;
case GT:
- code = translate(col, firstValue, -1);
+ code = translate(col, firstValue, 0);
if (code == null) {
- result = ConstantTupleFilter.TRUE;
+ code = translate(col, firstValue, 1);
+ if (code == null)
+ result = ConstantTupleFilter.FALSE;
+ else
+ result = newCompareFilter(FilterOperatorEnum.GTE, externalCol, code);
} else {
newCompareFilter.addChild(new ConstantTupleFilter(code));
result = newCompareFilter;
@@ -290,6 +297,13 @@ public class GTUtil {
return result;
}
+ private TupleFilter newCompareFilter(FilterOperatorEnum op, TblColRef col, ByteArray code) {
+ CompareTupleFilter r = new CompareTupleFilter(op);
+ r.addChild(new ColumnTupleFilter(col));
+ r.addChild(new ConstantTupleFilter(code));
+ return r;
+ }
+
transient ByteBuffer buf;
protected ByteArray translate(int col, Object value, int roundingFlag) {
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 4875217..298477f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -154,7 +154,6 @@ public class CompareTupleFilter extends TupleFilter implements IOptimizeableTupl
// TODO requires generalize, currently only evaluates COLUMN {op} CONST
@Override
- @SuppressWarnings({ "unchecked", "rawtypes" })
public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
// extract tuple value
Object tupleValue = null;
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
index e80a67c..073c12c 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -34,7 +34,7 @@ import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
-import org.apache.kylin.dict.NumberDictionaryBuilder;
+import org.apache.kylin.dict.NumberDictionaryForestBuilder;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionaryBuilder;
import org.apache.kylin.dimension.DictionaryDimEnc;
@@ -441,19 +441,61 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp");
TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer");
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LT, "9");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
List<TblColRef> colMapping = Lists.newArrayList();
colMapping.add(extColA);
colMapping.add(extColB);
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+
+ // $1<"9" round down to FALSE
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "9"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(ConstantTupleFilter.FALSE, newFilter);
+ }
- // $1<"9" round up to $1<"10"
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals(
- "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]",
- newFilter.toString());
+ // $1<"10" needs no rounding
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "10"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]",
+ newFilter.toString());
+ }
+
+ // $1<"11" round down to <="10"
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "11"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LTE [\\x00]]",
+ newFilter.toString());
+ }
+
+ // $1<="9" round down to FALSE
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "9"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(ConstantTupleFilter.FALSE, newFilter);
+ }
+
+ // $1<="10" needs no rounding
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "10"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LTE [\\x00]]",
+ newFilter.toString());
+ }
+
+ // $1<="11" round down to <="10"
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "11"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LTE [\\x00]]",
+ newFilter.toString());
+ }
}
@Test
@@ -464,17 +506,61 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp");
TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer");
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LTE, "9");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
List<TblColRef> colMapping = Lists.newArrayList();
colMapping.add(extColA);
colMapping.add(extColB);
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+
+ // $1>"101" round up to FALSE
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "101"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(ConstantTupleFilter.FALSE, newFilter);
+ }
- // $1<="9" round down to FALSE
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals(ConstantTupleFilter.FALSE, newFilter);
+ // $1>"100" needs no rounding
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "100"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GT [\\x09]]",
+ newFilter.toString());
+ }
+
+ // $1>"99" round up to >="100"
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "99"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GTE [\\x09]]",
+ newFilter.toString());
+ }
+
+ // $1>="101" round up to FALSE
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "101"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(ConstantTupleFilter.FALSE, newFilter);
+ }
+
+ // $1>="100" needs no rounding
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "100"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GTE [\\x09]]",
+ newFilter.toString());
+ }
+
+ // $1>="99" round up to >="100"
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "99"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GTE [\\x09]]",
+ newFilter.toString());
+ }
}
@Test
@@ -639,7 +725,6 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
return info;
}
- @SuppressWarnings("unchecked")
private static CubeCodeSystem newDictCodeSystem() {
DimensionEncoding[] dimEncs = new DimensionEncoding[3];
dimEncs[1] = new DictionaryDimEnc(newDictionaryOfInteger());
@@ -647,7 +732,6 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
return new CubeCodeSystem(dimEncs);
}
- @SuppressWarnings("rawtypes")
private static Dictionary newDictionaryOfString() {
TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter());
builder.addValue("Dong");
@@ -663,9 +747,8 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
return builder.build(0);
}
- @SuppressWarnings("rawtypes")
private static Dictionary newDictionaryOfInteger() {
- NumberDictionaryBuilder builder = new NumberDictionaryBuilder();
+ NumberDictionaryForestBuilder builder = new NumberDictionaryForestBuilder();
builder.addValue("10");
builder.addValue("20");
builder.addValue("30");
@@ -676,7 +759,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
builder.addValue("80");
builder.addValue("90");
builder.addValue("100");
- return builder.build(0);
+ return builder.build();
}
public static ImmutableBitSet setOf(int... values) {
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.