You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/05/15 07:22:45 UTC
[1/5] incubator-kylin git commit: Fix null value decode issue in
mergecuboidmapper
Repository: incubator-kylin
Updated Branches:
refs/heads/0.6 d5bb214de -> 531ed58b6
Fix null value decode issue in mergecuboidmapper
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/359e291b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/359e291b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/359e291b
Branch: refs/heads/0.6
Commit: 359e291b4c19cdd4d5e9abbad92beb58572f2b22
Parents: d5bb214
Author: honma <ho...@ebay.com>
Authored: Mon Apr 27 15:56:52 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Apr 27 15:56:52 2015 +0800
----------------------------------------------------------------------
.../src/main/java/com/kylinolap/dict/Dictionary.java | 8 +++++---
.../java/com/kylinolap/dict/DateStrDictionaryTest.java | 2 +-
.../test/java/com/kylinolap/dict/TrieDictionaryTest.java | 2 +-
.../com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java | 11 +++++++++--
4 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/359e291b/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java b/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java
index b54c6a3..51c2c90 100644
--- a/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java
+++ b/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java
@@ -59,7 +59,7 @@ abstract public class Dictionary<T> implements Writable {
* value of column
*/
abstract public int getSizeOfValue();
-
+
/**
* Convenient form of <code>getIdFromValue(value, 0)</code>
*/
@@ -131,7 +131,7 @@ abstract public class Dictionary<T> implements Writable {
else
return getIdFromValueBytesImpl(value, offset, len, roundingFlag);
}
-
+
protected boolean isNullByteForm(byte[] value, int offset, int len) {
return value == null;
}
@@ -142,13 +142,15 @@ abstract public class Dictionary<T> implements Writable {
* A lower level API, get byte values from ID, return the number of bytes
* written. Bypassing the cache layer, this could be significantly slower
* than getIdFromValue(T value).
+ *
+ * @return size of value bytes, 0 if empty string, -1 if null
*
* @throws IllegalArgumentException
* if ID is not found in dictionary
*/
final public int getValueBytesFromId(int id, byte[] returnValue, int offset) {
if (isNullId(id))
- return 0;
+ return -1;
else
return getValueBytesFromIdImpl(id, returnValue, offset);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/359e291b/dictionary/src/test/java/com/kylinolap/dict/DateStrDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/com/kylinolap/dict/DateStrDictionaryTest.java b/dictionary/src/test/java/com/kylinolap/dict/DateStrDictionaryTest.java
index c577138..41f7f84 100644
--- a/dictionary/src/test/java/com/kylinolap/dict/DateStrDictionaryTest.java
+++ b/dictionary/src/test/java/com/kylinolap/dict/DateStrDictionaryTest.java
@@ -34,7 +34,7 @@ public class DateStrDictionaryTest {
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), 0);
+ assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
assertEquals(nullId, nullId2);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/359e291b/dictionary/src/test/java/com/kylinolap/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/dictionary/src/test/java/com/kylinolap/dict/TrieDictionaryTest.java b/dictionary/src/test/java/com/kylinolap/dict/TrieDictionaryTest.java
index 81287ad..0a86d0c 100644
--- a/dictionary/src/test/java/com/kylinolap/dict/TrieDictionaryTest.java
+++ b/dictionary/src/test/java/com/kylinolap/dict/TrieDictionaryTest.java
@@ -271,7 +271,7 @@ public class TrieDictionaryTest {
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), 0);
+ assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
assertEquals(nullId, nullId2);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/359e291b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java
index c2ca802..28c7baf 100644
--- a/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java
+++ b/job/src/main/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapper.java
@@ -151,11 +151,18 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
}
int idInSourceDict = BytesUtil.readUnsigned(splittedByteses[i + 1].value, 0, splittedByteses[i + 1].length);
+ int idInMergedDict;
+
int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBuf, bufOffset);
- int idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBuf, bufOffset, size);
- BytesUtil.writeUnsigned(idInMergedDict, newKeyBuf, bufOffset, mergedDict.getSizeOfId());
+ if (size < 0) {
+ idInMergedDict = mergedDict.nullId();
+ } else {
+ idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBuf, bufOffset, size);
+ }
+ BytesUtil.writeUnsigned(idInMergedDict, newKeyBuf, bufOffset, mergedDict.getSizeOfId());
bufOffset += mergedDict.getSizeOfId();
+
} else {
// keep as it is
while (splittedByteses[i + 1].length > newKeyBuf.length - bufOffset) {
[2/5] incubator-kylin git commit: KYLIN-739,
Dimension as metrics does not work with PK-FK derived column
Posted by ma...@apache.org.
KYLIN-739, Dimension as metrics does not work with PK-FK derived column
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ca6c7e22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ca6c7e22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ca6c7e22
Branch: refs/heads/0.6
Commit: ca6c7e227ed3689430aef7ae90c2d4d8b9307b35
Parents: 359e291
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Apr 29 10:36:43 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Apr 29 10:36:43 2015 +0800
----------------------------------------------------------------------
.../kylinolap/query/routing/QueryRouter.java | 2 +-
query/src/test/resources/query/sql/query78.sql | 1 +
.../java/com/kylinolap/rest/DebugTomcat.java | 40 +++++++++++++++-----
.../com/kylinolap/storage/StorageContext.java | 12 ------
.../storage/hbase/HBaseStorageEngine.java | 1 -
5 files changed, 33 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca6c7e22/query/src/main/java/com/kylinolap/query/routing/QueryRouter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/com/kylinolap/query/routing/QueryRouter.java b/query/src/main/java/com/kylinolap/query/routing/QueryRouter.java
index 7745c2a..f786681 100644
--- a/query/src/main/java/com/kylinolap/query/routing/QueryRouter.java
+++ b/query/src/main/java/com/kylinolap/query/routing/QueryRouter.java
@@ -284,7 +284,7 @@ public class QueryRouter {
if (col != null) {
metricColumns.remove(col);
dimensionColumns.add(col);
- olapContext.storageContext.addOtherMandatoryColumns(col);
+ olapContext.groupByColumns.add(col);
}
logger.info("Adjust OLAPContext for " + functionDesc);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca6c7e22/query/src/test/resources/query/sql/query78.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query78.sql b/query/src/test/resources/query/sql/query78.sql
new file mode 100644
index 0000000..3764593
--- /dev/null
+++ b/query/src/test/resources/query/sql/query78.sql
@@ -0,0 +1 @@
+select max(cal_dt) from test_kylin_fact
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca6c7e22/server/src/main/java/com/kylinolap/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/DebugTomcat.java b/server/src/main/java/com/kylinolap/rest/DebugTomcat.java
index 1e79c7e..40e66e7 100644
--- a/server/src/main/java/com/kylinolap/rest/DebugTomcat.java
+++ b/server/src/main/java/com/kylinolap/rest/DebugTomcat.java
@@ -16,31 +16,54 @@
package com.kylinolap.rest;
import java.io.File;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
import org.apache.catalina.Context;
import org.apache.catalina.core.AprLifecycleListener;
import org.apache.catalina.core.StandardServer;
import org.apache.catalina.deploy.ErrorPage;
import org.apache.catalina.startup.Tomcat;
+import org.apache.hadoop.util.Shell;
-import com.kylinolap.rest.util.ClasspathUtil;
+import com.kylinolap.common.KylinConfig;
+import com.kylinolap.common.util.ClasspathUtil;
public class DebugTomcat {
public static void main(String[] args) throws Exception {
- if (args.length >= 1) {
- System.setProperty("kylin.metadata.url", args[0]);
- }
int port = 7070;
- if (args.length >= 2) {
- port = Integer.parseInt(args[1]);
+ if (args.length >= 1) {
+ port = Integer.parseInt(args[0]);
}
-
+
+ // test_case_data/sandbox/ contains HDP 2.2 site xmls which is dev sandbox
ClasspathUtil.addClasspath(new File("../examples/test_case_data/sandbox").getAbsolutePath());
+ System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
+ // workaround for job submission from win to linux -- https://issues.apache.org/jira/browse/MAPREDUCE-4052
+ if (Shell.WINDOWS) {
+ {
+ Field field = Shell.class.getDeclaredField("WINDOWS");
+ field.setAccessible(true);
+ Field modifiersField = Field.class.getDeclaredField("modifiers");
+ modifiersField.setAccessible(true);
+ modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+ field.set(null, false);
+ }
+ {
+ Field field = java.io.File.class.getDeclaredField("pathSeparator");
+ field.setAccessible(true);
+ Field modifiersField = Field.class.getDeclaredField("modifiers");
+ modifiersField.setAccessible(true);
+ modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+ field.set(null, ":");
+ }
+ }
+
+ System.setProperty("spring.profiles.active", "testing");
String webBase = new File("../webapp/app").getAbsolutePath();
-// String apiBase = new File("src/main/webapp").getAbsolutePath();
if (new File(webBase, "WEB-INF").exists() == false) {
throw new RuntimeException("In order to launch Kylin web app from IDE, please make a symblink from webapp/app/WEB-INF to server/src/main/webapp/WEB-INF");
}
@@ -54,7 +77,6 @@ public class DebugTomcat {
AprLifecycleListener listener = new AprLifecycleListener();
server.addLifecycleListener(listener);
-// tomcat.addWebapp("/kylin", apiBase);
Context webContext = tomcat.addWebapp("/kylin", webBase);
ErrorPage notFound = new ErrorPage();
notFound.setErrorCode(404);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca6c7e22/storage/src/main/java/com/kylinolap/storage/StorageContext.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/StorageContext.java b/storage/src/main/java/com/kylinolap/storage/StorageContext.java
index d4e01ca..79a997a 100644
--- a/storage/src/main/java/com/kylinolap/storage/StorageContext.java
+++ b/storage/src/main/java/com/kylinolap/storage/StorageContext.java
@@ -16,9 +16,7 @@
package com.kylinolap.storage;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
@@ -50,7 +48,6 @@ public class StorageContext {
// To hint records shall be returned at most granular level, avoid aggregation (coprocessor) wherever possible.
private boolean avoidAggregation;
private boolean exactAggregation;
- private Set<TblColRef> otherMandatoryColumns;
private boolean enableLimit;
private boolean enableCoprocessor;
@@ -70,7 +67,6 @@ public class StorageContext {
this.avoidAggregation = false;
this.exactAggregation = false;
- this.otherMandatoryColumns = new HashSet<TblColRef>();
this.enableLimit = false;
this.enableCoprocessor = false;
@@ -193,14 +189,6 @@ public class StorageContext {
return this.exactAggregation;
}
- public void addOtherMandatoryColumns(TblColRef col) {
- this.otherMandatoryColumns.add(col);
- }
-
- public Set<TblColRef> getOtherMandatoryColumns() {
- return this.otherMandatoryColumns;
- }
-
public void enableCoprocessor() {
this.enableCoprocessor = true;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ca6c7e22/storage/src/main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java b/storage/src/main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java
index b47ae5f..6f214eb 100644
--- a/storage/src/main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java
+++ b/storage/src/main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java
@@ -109,7 +109,6 @@ public class HBaseStorageEngine implements IStorageEngine {
// - columns on non-evaluatable filter have to return
// - columns on loosened filter (due to derived translation) have to return
Set<TblColRef> groupsCopD = Sets.newHashSet(groupsD);
- groupsCopD.addAll(context.getOtherMandatoryColumns()); // TODO: this is tricky, to generalize
collectNonEvaluable(filter, groupsCopD);
TupleFilter filterD = translateDerived(filter, groupsCopD);
[3/5] incubator-kylin git commit: KYLIN-740,
increase scan range merge threshold to 100
Posted by ma...@apache.org.
KYLIN-740, increase scan range merge threshold to 100
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d4e9b433
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d4e9b433
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d4e9b433
Branch: refs/heads/0.6
Commit: d4e9b433432b9c0bd696145fcf8b2ab0bdbd869d
Parents: ca6c7e2
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Apr 29 11:12:09 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Apr 29 11:12:09 2015 +0800
----------------------------------------------------------------------
.../main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d4e9b433/storage/src/main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java b/storage/src/main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java
index 6f214eb..e2320af 100644
--- a/storage/src/main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java
+++ b/storage/src/main/java/com/kylinolap/storage/hbase/HBaseStorageEngine.java
@@ -67,7 +67,7 @@ public class HBaseStorageEngine implements IStorageEngine {
private static final Logger logger = LoggerFactory.getLogger(HBaseStorageEngine.class);
- private static final int MERGE_KEYRANGE_THRESHOLD = 7;
+ private static final int MERGE_KEYRANGE_THRESHOLD = 100;
private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G
private final CubeInstance cubeInstance;
[4/5] incubator-kylin git commit: KYLIN-739, fix test case
Posted by ma...@apache.org.
KYLIN-739, fix test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/bbdadcb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/bbdadcb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/bbdadcb7
Branch: refs/heads/0.6
Commit: bbdadcb72bd9d16b9c420b128451d1a71300542f
Parents: d4e9b43
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Apr 29 11:18:07 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Apr 29 11:18:07 2015 +0800
----------------------------------------------------------------------
query/src/test/resources/query/sql/query78.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/bbdadcb7/query/src/test/resources/query/sql/query78.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql/query78.sql b/query/src/test/resources/query/sql/query78.sql
index 3764593..d7b7506 100644
--- a/query/src/test/resources/query/sql/query78.sql
+++ b/query/src/test/resources/query/sql/query78.sql
@@ -1 +1 @@
-select max(cal_dt) from test_kylin_fact
+select max(cal_dt) as cnt from test_kylin_fact
[5/5] incubator-kylin git commit: KYLIN-765 When a cube job is failed,
still be possible to submit a new job
Posted by ma...@apache.org.
KYLIN-765 When a cube job is failed, still be possible to submit a new job
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/531ed58b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/531ed58b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/531ed58b
Branch: refs/heads/0.6
Commit: 531ed58b6cf811679aecfdd5acdc8cb42b589256
Parents: bbdadcb
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Thu May 14 15:17:13 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Thu May 14 15:17:13 2015 +0800
----------------------------------------------------------------------
server/src/main/java/com/kylinolap/rest/service/JobService.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/531ed58b/server/src/main/java/com/kylinolap/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/com/kylinolap/rest/service/JobService.java b/server/src/main/java/com/kylinolap/rest/service/JobService.java
index af4f16f..9d15ea0 100644
--- a/server/src/main/java/com/kylinolap/rest/service/JobService.java
+++ b/server/src/main/java/com/kylinolap/rest/service/JobService.java
@@ -100,7 +100,7 @@ public class JobService extends BasicService {
List<JobInstance> jobInstances = this.getJobManager().listJobs(cube.getName(), null);
for (JobInstance jobInstance : jobInstances) {
- if (jobInstance.getStatus() == JobStatusEnum.PENDING || jobInstance.getStatus() == JobStatusEnum.RUNNING) {
+ if (jobInstance.getStatus() == JobStatusEnum.PENDING || jobInstance.getStatus() == JobStatusEnum.RUNNING || jobInstance.getStatus() == JobStatusEnum.ERROR) {
throw new JobException("The cube " + cube.getName() + " has running job(" + jobInstance.getUuid() + ") please discard it and try again.");
}
}
@@ -142,7 +142,7 @@ public class JobService extends BasicService {
CubeInstance cube = this.getCubeManager().getCube(job.getRelatedCube());
List<JobInstance> jobs = this.getJobManager().listJobs(cube.getName(), null);
for (JobInstance jobInstance : jobs) {
- if (jobInstance.getStatus() != JobStatusEnum.DISCARDED && jobInstance.getStatus() != JobStatusEnum.FINISHED) {
+ if (jobInstance.getStatus() != JobStatusEnum.DISCARDED && jobInstance.getStatus() != JobStatusEnum.FINISHED && jobInstance.getUuid().equals(job.getUuid())) {
this.getJobManager().discardJob(jobInstance.getUuid());
}
}